18 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
19 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
24 #include "pagespeed/kernel/base/basictypes.h"
25 #include "pagespeed/kernel/base/gtest_prod.h"
26 #include "pagespeed/kernel/base/pool.h"
27 #include "pagespeed/kernel/base/string.h"
28 #include "pagespeed/kernel/base/string_util.h"
29 #include "pagespeed/kernel/base/thread_system.h"
41 #ifndef SERF_HTTPS_FETCHING
42 #define SERF_HTTPS_FETCHING 1
47 struct serf_context_t;
49 namespace net_instaweb {
55 class SerfThreadedFetcher;
61 static const char kSerfFetchRequestCount[];
62 static const char kSerfFetchByteCount[];
63 static const char kSerfFetchTimeDurationMs[];
64 static const char kSerfFetchCancelCount[];
65 static const char kSerfFetchActiveCount[];
66 static const char kSerfFetchTimeoutCount[];
67 static const char kSerfFetchFailureCount[];
68 static const char kSerfFetchCertErrors[];
75 #define SERF_HTTPS_KEYWORDS \
76 "enable,disable,allow_self_signed," \
77 "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
94 ThreadSystem* thread_system,
95 Statistics* statistics, Timer* timer, int64
timeout_ms,
96 MessageHandler* handler);
100 static void InitStats(Statistics* statistics);
108 virtual void Fetch(
const GoogleString& url,
109 MessageHandler* message_handler,
112 int Poll(int64 max_wait_ms);
114 bool WaitForActiveFetches(int64 max_milliseconds,
115 MessageHandler* message_handler,
116 WaitChoice wait_choice);
125 apr_pool_t* pool()
const {
return pool_; }
126 serf_context_t* serf_context()
const {
return serf_context_; }
128 void PrintActiveFetches(MessageHandler* handler)
const;
130 ThreadSystem* thread_system() {
return thread_system_; }
139 return track_original_content_length_;
141 void set_track_original_content_length(
bool x);
157 GoogleString* error_message) {
159 return ParseHttpsOptions(directive, &options, error_message);
162 void SetSslCertificatesDir(StringPiece dir);
163 const GoogleString& ssl_certificates_dir()
const {
164 return ssl_certificates_dir_;
167 void SetSslCertificatesFile(StringPiece file);
168 const GoogleString& ssl_certificates_file()
const {
169 return ssl_certificates_file_;
173 typedef Pool<SerfFetch> SerfFetchPool;
177 inline bool allow_self_signed()
const;
178 inline bool allow_unknown_certificate_authority()
const;
179 inline bool allow_certificate_not_yet_valid()
const;
181 void set_https_options(uint32 https_options) {
182 https_options_ = https_options;
185 void Init(apr_pool_t* parent_pool,
const char* proxy);
186 bool SetupProxy(
const char* proxy);
203 void CancelActiveFetches();
204 void CancelActiveFetchesMutexHeld();
205 bool WaitForActiveFetchesHelper(int64 max_ms,
206 MessageHandler* message_handler);
215 void set_shutdown(
bool s) { shutdown_ = s; }
218 ThreadSystem* thread_system_;
222 ThreadSystem::CondvarCapableMutex*
mutex_;
223 serf_context_t* serf_context_;
224 SerfFetchPool active_fetches_;
226 typedef std::vector<SerfFetch*> FetchVector;
227 SerfFetchPool completed_fetches_;
228 SerfThreadedFetcher* threaded_fetcher_;
238 static const char* ExtractHostHeader(
const apr_uri_t& uri,
240 FRIEND_TEST(SerfUrlAsyncFetcherTest, TestHostConstruction);
244 static GoogleString RemovePortFromHostHeader(
const GoogleString& in);
245 FRIEND_TEST(SerfUrlAsyncFetcherTest, TestPortRemoval);
247 static bool ParseHttpsOptions(StringPiece directive, uint32* options,
248 GoogleString* error_message);
250 Variable* request_count_;
251 Variable* byte_count_;
252 Variable* time_duration_ms_;
253 Variable* cancel_count_;
254 Variable* timeout_count_;
255 Variable* failure_count_;
256 Variable* cert_errors_;
257 const int64 timeout_ms_;
259 bool list_outstanding_urls_on_error_;
260 bool track_original_content_length_;
261 uint32 https_options_;
262 MessageHandler* message_handler_;
263 GoogleString ssl_certificates_dir_;
264 GoogleString ssl_certificates_file_;
int Poll(int64 max_wait_ms)
bool track_original_content_length() const
Definition: serf_url_async_fetcher.h:138
virtual void Fetch(const GoogleString &url, MessageHandler *message_handler, AsyncFetch *callback)
Definition: serf_url_async_fetcher.h:85
bool SetHttpsOptions(StringPiece directive)
void CleanupFetchesWithErrors()
static bool ValidateHttpsOptions(StringPiece directive, GoogleString *error_message)
Definition: serf_url_async_fetcher.h:156
int ApproximateNumActiveFetches()
bool allow_https() const
Determines whether https is allowed in the current configuration.
void FetchComplete(SerfFetch *fetch)
Definition: async_fetch.h:53
virtual bool AnyPendingFetches()
bool StartFetch(SerfFetch *fetch)
UpDownCounter * active_count_
Definition: serf_url_async_fetcher.h:232
bool shutdown() const
These must be accessed with mutex_ held.
Definition: serf_url_async_fetcher.h:214
virtual bool SupportsHttps() const
friend class SerfFetch
To access stats variables below.
Definition: serf_url_async_fetcher.h:235
ThreadSystem::CondvarCapableMutex * mutex_
mutex_ protects serf_context_ and active_fetches_.
Definition: serf_url_async_fetcher.h:222
Definition: serf_url_async_fetcher.h:60
virtual int64 timeout_ms()
Definition: serf_url_async_fetcher.h:129
void set_list_outstanding_urls_on_error(bool x)
Definition: url_async_fetcher.h:33
void ReportCompletedFetchStats(SerfFetch *fetch)
Update the statistics object with results of the (completed) fetch.