Page Speed Optimization Libraries
1.7.30.2
|
00001 // Copyright 2010 Google Inc. 00017 00018 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_ 00019 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_ 00020 00021 #include <set> 00022 #include <vector> 00023 00024 #include "net/instaweb/http/public/url_async_fetcher.h" 00025 #include "net/instaweb/util/public/basictypes.h" 00026 #include "net/instaweb/util/public/pool.h" 00027 #include "net/instaweb/util/public/string.h" 00028 #include "net/instaweb/util/public/string_util.h" 00029 #include "net/instaweb/util/public/thread_system.h" 00030 00041 #ifndef SERF_HTTPS_FETCHING 00042 #define SERF_HTTPS_FETCHING 1 00043 #endif 00044 00045 struct apr_pool_t; 00046 struct serf_context_t; 00047 00048 namespace net_instaweb { 00049 00050 class AsyncFetch; 00051 class MessageHandler; 00052 class Statistics; 00053 class SerfFetch; 00054 class SerfThreadedFetcher; 00055 class Timer; 00056 class Variable; 00057 struct ContentType; 00058 00059 struct SerfStats { 00060 static const char kSerfFetchRequestCount[]; 00061 static const char kSerfFetchByteCount[]; 00062 static const char kSerfFetchTimeDurationMs[]; 00063 static const char kSerfFetchCancelCount[]; 00064 static const char kSerfFetchActiveCount[]; 00065 static const char kSerfFetchTimeoutCount[]; 00066 static const char kSerfFetchFailureCount[]; 00067 static const char kSerfFetchCertErrors[]; 00068 }; 00069 00074 #define SERF_HTTPS_KEYWORDS \ 00075 "enable,disable,allow_self_signed," \ 00076 "allow_unknown_certificate_authority,allow_certificate_not_yet_valid" 00077 00084 class SerfUrlAsyncFetcher : public UrlAsyncFetcher { 00085 public: 00086 enum WaitChoice { 00087 kThreadedOnly, 00088 kMainlineOnly, 00089 kThreadedAndMainline 00090 }; 00091 00092 SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool, 00093 ThreadSystem* thread_system, 00094 Statistics* statistics, Timer* timer, int64 timeout_ms, 00095 MessageHandler* handler); 00096 SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy); 00097 virtual ~SerfUrlAsyncFetcher(); 00098 00099 static void InitStats(Statistics* statistics); 00100 00103 virtual void ShutDown(); 00104 00105 virtual bool SupportsHttps() const; 00106 00107 virtual void Fetch(const GoogleString& url, 00108 MessageHandler* message_handler, 00109 AsyncFetch* callback); 00111 int Poll(int64 max_wait_ms); 00112 00113 bool WaitForActiveFetches(int64 max_milliseconds, 00114 MessageHandler* message_handler, 00115 WaitChoice wait_choice); 00116 00119 void FetchComplete(SerfFetch* fetch); 00120 00122 void ReportCompletedFetchStats(SerfFetch* fetch); 00123 00124 apr_pool_t* pool() const { return pool_; } 00125 serf_context_t* serf_context() const { return serf_context_; } 00126 00127 void PrintActiveFetches(MessageHandler* handler) const; 00128 virtual int64 timeout_ms() { return timeout_ms_; } 00129 ThreadSystem* thread_system() { return thread_system_; } 00130 00133 void set_list_outstanding_urls_on_error(bool x); 00134 00137 bool track_original_content_length() const { 00138 return track_original_content_length_; 00139 } 00140 void set_track_original_content_length(bool x); 00141 00142 void set_inflation_content_type_blacklist( 00143 const std::set<const ContentType*>& bypass_set) { 00144 inflation_content_type_blacklist_ = bypass_set; 00145 } 00146 00156 bool SetHttpsOptions(StringPiece directive); 00157 00160 static bool ValidateHttpsOptions(StringPiece directive, 00161 GoogleString* error_message) { 00162 uint32 options; 00163 return ParseHttpsOptions(directive, &options, error_message); 00164 } 00165 00166 void SetSslCertificatesDir(StringPiece dir); 00167 const GoogleString& ssl_certificates_dir() const { 00168 return ssl_certificates_dir_; 00169 } 00170 00171 void SetSslCertificatesFile(StringPiece file); 00172 const GoogleString& ssl_certificates_file() const { 00173 return ssl_certificates_file_; 00174 } 00175 00176 protected: 00177 typedef Pool<SerfFetch> SerfFetchPool; 00178 00180 inline bool allow_https() const; 00181 inline bool allow_self_signed() const; 00182 inline bool allow_unknown_certificate_authority() const; 00183 inline bool allow_certificate_not_yet_valid() const; 00184 00185 void set_https_options(uint32 https_options) { 00186 https_options_ = https_options; 00187 } 00188 00189 void Init(apr_pool_t* parent_pool, const char* proxy); 00190 bool SetupProxy(const char* proxy); 00191 00196 bool StartFetch(SerfFetch* fetch); 00197 00202 virtual bool AnyPendingFetches(); 00205 int ApproximateNumActiveFetches(); 00206 00207 void CancelActiveFetches(); 00208 void CancelActiveFetchesMutexHeld(); 00209 bool WaitForActiveFetchesHelper(int64 max_ms, 00210 MessageHandler* message_handler); 00211 00215 void CleanupFetchesWithErrors(); 00216 00218 bool shutdown() const { return shutdown_; } 00219 void set_shutdown(bool s) { shutdown_ = s; } 00220 00221 apr_pool_t* pool_; 00222 ThreadSystem* thread_system_; 00223 Timer* timer_; 00224 00226 ThreadSystem::CondvarCapableMutex* mutex_; 00227 serf_context_t* serf_context_; 00228 SerfFetchPool active_fetches_; 00229 00230 typedef std::vector<SerfFetch*> FetchVector; 00231 SerfFetchPool completed_fetches_; 00232 SerfThreadedFetcher* threaded_fetcher_; 00233 00236 Variable* active_count_; 00237 00238 private: 00239 friend class SerfFetch; 00240 00241 static bool ParseHttpsOptions(StringPiece directive, uint32* options, 00242 GoogleString* error_message); 00243 00244 Variable* request_count_; 00245 Variable* byte_count_; 00246 Variable* time_duration_ms_; 00247 Variable* cancel_count_; 00248 Variable* timeout_count_; 00249 Variable* failure_count_; 00250 Variable* cert_errors_; 00251 const int64 timeout_ms_; 00252 bool shutdown_; 00253 bool list_outstanding_urls_on_error_; 00254 bool track_original_content_length_; 00255 uint32 https_options_; 00256 MessageHandler* message_handler_; 00257 GoogleString ssl_certificates_dir_; 00258 GoogleString ssl_certificates_file_; 00259 00262 std::set<const ContentType*> inflation_content_type_blacklist_; 00263 00264 DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher); 00265 }; 00266 00267 } 00268 00269 #endif ///< NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_