Page Speed Optimization Libraries  1.9.32.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
serf_url_async_fetcher.h
Go to the documentation of this file.
1 // Copyright 2010 Google Inc.
17 
18 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
19 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
20 
21 #include <vector>
22 
25 #include "net/instaweb/util/public/gtest_prod.h"
26 #include "net/instaweb/util/public/pool.h"
30 
41 #ifndef SERF_HTTPS_FETCHING
42 #define SERF_HTTPS_FETCHING 1
43 #endif
44 
45 struct apr_pool_t;
46 struct apr_uri_t;
47 struct serf_context_t;
48 
49 namespace net_instaweb {
50 
51 class AsyncFetch;
52 class MessageHandler;
53 class Statistics;
54 class SerfFetch;
55 class SerfThreadedFetcher;
56 class Timer;
57 class UpDownCounter;
58 class Variable;
59 
60 struct SerfStats {
61  static const char kSerfFetchRequestCount[];
62  static const char kSerfFetchByteCount[];
63  static const char kSerfFetchTimeDurationMs[];
64  static const char kSerfFetchCancelCount[];
65  static const char kSerfFetchActiveCount[];
66  static const char kSerfFetchTimeoutCount[];
67  static const char kSerfFetchFailureCount[];
68  static const char kSerfFetchCertErrors[];
69 };
70 
75 #define SERF_HTTPS_KEYWORDS \
76  "enable,disable,allow_self_signed," \
77  "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
78 
86  public:
87  enum WaitChoice {
88  kThreadedOnly,
89  kMainlineOnly,
90  kThreadedAndMainline
91  };
92 
93  SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
94  ThreadSystem* thread_system,
95  Statistics* statistics, Timer* timer, int64 timeout_ms,
96  MessageHandler* handler);
97  SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
98  virtual ~SerfUrlAsyncFetcher();
99 
100  static void InitStats(Statistics* statistics);
101 
104  virtual void ShutDown();
105 
106  virtual bool SupportsHttps() const;
107 
108  virtual void Fetch(const GoogleString& url,
109  MessageHandler* message_handler,
110  AsyncFetch* callback);
112  int Poll(int64 max_wait_ms);
113 
114  bool WaitForActiveFetches(int64 max_milliseconds,
115  MessageHandler* message_handler,
116  WaitChoice wait_choice);
117 
120  void FetchComplete(SerfFetch* fetch);
121 
124 
125  apr_pool_t* pool() const { return pool_; }
126  serf_context_t* serf_context() const { return serf_context_; }
127 
128  void PrintActiveFetches(MessageHandler* handler) const;
129  virtual int64 timeout_ms() { return timeout_ms_; }
130  ThreadSystem* thread_system() { return thread_system_; }
131 
135 
139  return track_original_content_length_;
140  }
141  void set_track_original_content_length(bool x);
142 
152  bool SetHttpsOptions(StringPiece directive);
153 
156  static bool ValidateHttpsOptions(StringPiece directive,
157  GoogleString* error_message) {
158  uint32 options;
159  return ParseHttpsOptions(directive, &options, error_message);
160  }
161 
162  void SetSslCertificatesDir(StringPiece dir);
163  const GoogleString& ssl_certificates_dir() const {
164  return ssl_certificates_dir_;
165  }
166 
167  void SetSslCertificatesFile(StringPiece file);
168  const GoogleString& ssl_certificates_file() const {
169  return ssl_certificates_file_;
170  }
171 
172  protected:
173  typedef Pool<SerfFetch> SerfFetchPool;
174 
176  inline bool allow_https() const;
177  inline bool allow_self_signed() const;
178  inline bool allow_unknown_certificate_authority() const;
179  inline bool allow_certificate_not_yet_valid() const;
180 
181  void set_https_options(uint32 https_options) {
182  https_options_ = https_options;
183  }
184 
185  void Init(apr_pool_t* parent_pool, const char* proxy);
186  bool SetupProxy(const char* proxy);
187 
192  bool StartFetch(SerfFetch* fetch);
193 
198  virtual bool AnyPendingFetches();
202 
203  void CancelActiveFetches();
204  void CancelActiveFetchesMutexHeld();
205  bool WaitForActiveFetchesHelper(int64 max_ms,
206  MessageHandler* message_handler);
207 
212 
214  bool shutdown() const { return shutdown_; }
215  void set_shutdown(bool s) { shutdown_ = s; }
216 
217  apr_pool_t* pool_;
218  ThreadSystem* thread_system_;
219  Timer* timer_;
220 
222  ThreadSystem::CondvarCapableMutex* mutex_;
223  serf_context_t* serf_context_;
224  SerfFetchPool active_fetches_;
225 
226  typedef std::vector<SerfFetch*> FetchVector;
227  SerfFetchPool completed_fetches_;
228  SerfThreadedFetcher* threaded_fetcher_;
229 
232  UpDownCounter* active_count_;
233 
234  private:
235  friend class SerfFetch;
236 
238  static const char* ExtractHostHeader(const apr_uri_t& uri,
239  apr_pool_t* pool);
240  FRIEND_TEST(SerfUrlAsyncFetcherTest, TestHostConstruction);
241 
244  static GoogleString RemovePortFromHostHeader(const GoogleString& in);
245  FRIEND_TEST(SerfUrlAsyncFetcherTest, TestPortRemoval);
246 
247  static bool ParseHttpsOptions(StringPiece directive, uint32* options,
248  GoogleString* error_message);
249 
250  Variable* request_count_;
251  Variable* byte_count_;
252  Variable* time_duration_ms_;
253  Variable* cancel_count_;
254  Variable* timeout_count_;
255  Variable* failure_count_;
256  Variable* cert_errors_;
257  const int64 timeout_ms_;
258  bool shutdown_;
259  bool list_outstanding_urls_on_error_;
260  bool track_original_content_length_;
261  uint32 https_options_;
262  MessageHandler* message_handler_;
263  GoogleString ssl_certificates_dir_;
264  GoogleString ssl_certificates_file_;
265 
266  DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
267 };
268 
269 }
270 
271 #endif
int Poll(int64 max_wait_ms)
bool track_original_content_length() const
Definition: serf_url_async_fetcher.h:138
virtual void Fetch(const GoogleString &url, MessageHandler *message_handler, AsyncFetch *callback)
Definition: serf_url_async_fetcher.h:85
bool SetHttpsOptions(StringPiece directive)
static bool ValidateHttpsOptions(StringPiece directive, GoogleString *error_message)
Definition: serf_url_async_fetcher.h:156
bool allow_https() const
Determines whether https is allowed in the current configuration.
void FetchComplete(SerfFetch *fetch)
Definition: async_fetch.h:53
bool StartFetch(SerfFetch *fetch)
UpDownCounter * active_count_
Definition: serf_url_async_fetcher.h:232
bool shutdown() const
These must be accessed with mutex_ held.
Definition: serf_url_async_fetcher.h:214
virtual bool SupportsHttps() const
friend class SerfFetch
To access stats variables below.
Definition: serf_url_async_fetcher.h:235
ThreadSystem::CondvarCapableMutex * mutex_
mutex_ protects serf_context_ and active_fetches_.
Definition: serf_url_async_fetcher.h:222
Definition: serf_url_async_fetcher.h:60
virtual int64 timeout_ms()
Definition: serf_url_async_fetcher.h:129
void set_list_outstanding_urls_on_error(bool x)
Definition: url_async_fetcher.h:33
void ReportCompletedFetchStats(SerfFetch *fetch)
Update the statistics object with results of the (completed) fetch.