Page Speed Optimization Libraries
1.4.26.1
|
00001 // Copyright 2010 Google Inc. 00017 00018 #ifndef NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_ 00019 #define NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_ 00020 00021 #include <set> 00022 #include <vector> 00023 00024 #include "net/instaweb/http/public/url_pollable_async_fetcher.h" 00025 #include "net/instaweb/util/public/basictypes.h" 00026 #include "net/instaweb/util/public/pool.h" 00027 #include "net/instaweb/util/public/string.h" 00028 #include "net/instaweb/util/public/string_util.h" 00029 #include "net/instaweb/util/public/thread_system.h" 00030 00041 #ifndef SERF_HTTPS_FETCHING 00042 #define SERF_HTTPS_FETCHING 0 00043 #endif 00044 00045 struct apr_pool_t; 00046 struct serf_context_t; 00047 00048 namespace net_instaweb { 00049 00050 class AsyncFetch; 00051 class MessageHandler; 00052 class Statistics; 00053 class SerfFetch; 00054 class SerfThreadedFetcher; 00055 class Timer; 00056 class Variable; 00057 00058 struct SerfStats { 00059 static const char kSerfFetchRequestCount[]; 00060 static const char kSerfFetchByteCount[]; 00061 static const char kSerfFetchTimeDurationMs[]; 00062 static const char kSerfFetchCancelCount[]; 00063 static const char kSerfFetchActiveCount[]; 00064 static const char kSerfFetchTimeoutCount[]; 00065 static const char kSerfFetchFailureCount[]; 00066 static const char kSerfFetchCertErrors[]; 00067 }; 00068 00073 #define SERF_HTTPS_KEYWORDS \ 00074 "enable,disable,allow_self_signed," \ 00075 "allow_unknown_certificate_authority,allow_certificate_not_yet_valid" 00076 00083 class SerfUrlAsyncFetcher : public UrlPollableAsyncFetcher { 00084 public: 00085 enum WaitChoice { 00086 kThreadedOnly, 00087 kMainlineOnly, 00088 kThreadedAndMainline 00089 }; 00090 00091 SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool, 00092 ThreadSystem* thread_system, 00093 Statistics* statistics, Timer* timer, int64 timeout_ms, 00094 MessageHandler* handler); 00095 SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy); 00096 virtual ~SerfUrlAsyncFetcher(); 00097 00098 static void InitStats(Statistics* statistics); 00099 00102 virtual void ShutDown(); 00103 00104 virtual bool SupportsHttps() const; 00105 00106 virtual void Fetch(const GoogleString& url, 00107 MessageHandler* message_handler, 00108 AsyncFetch* callback); 00109 00110 virtual int Poll(int64 max_wait_ms); 00111 00112 bool WaitForActiveFetches(int64 max_milliseconds, 00113 MessageHandler* message_handler, 00114 WaitChoice wait_choice); 00115 00118 void FetchComplete(SerfFetch* fetch); 00119 apr_pool_t* pool() const { return pool_; } 00120 serf_context_t* serf_context() const { return serf_context_; } 00121 00122 void PrintActiveFetches(MessageHandler* handler) const; 00123 virtual int64 timeout_ms() { return timeout_ms_; } 00124 ThreadSystem* thread_system() { return thread_system_; } 00125 00132 void set_force_threaded(bool x) { force_threaded_ = x; } 00133 00136 void set_list_outstanding_urls_on_error(bool x); 00137 00140 bool track_original_content_length() const { 00141 return track_original_content_length_; 00142 } 00143 void set_track_original_content_length(bool x); 00144 00145 void set_inflation_content_type_blacklist( 00146 const std::set<const ContentType*>& bypass_set) { 00147 inflation_content_type_blacklist_ = bypass_set; 00148 } 00149 00159 bool SetHttpsOptions(StringPiece directive); 00160 00163 static bool ValidateHttpsOptions(StringPiece directive, 00164 GoogleString* error_message) { 00165 uint32 options; 00166 return ParseHttpsOptions(directive, &options, error_message); 00167 } 00168 00169 protected: 00170 typedef Pool<SerfFetch> SerfFetchPool; 00171 00173 inline bool allow_https() const; 00174 inline bool allow_self_signed() const; 00175 inline bool allow_unknown_certificate_authority() const; 00176 inline bool allow_certificate_not_yet_valid() const; 00177 00178 void set_https_options(uint32 https_options) { 00179 https_options_ = https_options; 00180 } 00181 00182 void Init(apr_pool_t* parent_pool, const char* proxy); 00183 bool SetupProxy(const char* proxy); 00184 00189 bool StartFetch(SerfFetch* fetch); 00190 00195 virtual bool AnyPendingFetches(); 00198 int ApproximateNumActiveFetches(); 00199 00200 void CancelActiveFetches(); 00201 void CancelActiveFetchesMutexHeld(); 00202 bool WaitForActiveFetchesHelper(int64 max_ms, 00203 MessageHandler* message_handler); 00204 00208 void CleanupFetchesWithErrors(); 00209 00211 bool shutdown() const { return shutdown_; } 00212 void set_shutdown(bool s) { shutdown_ = s; } 00213 00214 apr_pool_t* pool_; 00215 ThreadSystem* thread_system_; 00216 Timer* timer_; 00217 00219 ThreadSystem::CondvarCapableMutex* mutex_; 00220 serf_context_t* serf_context_; 00221 SerfFetchPool active_fetches_; 00222 00223 typedef std::vector<SerfFetch*> FetchVector; 00224 SerfFetchPool completed_fetches_; 00225 SerfThreadedFetcher* threaded_fetcher_; 00226 00229 Variable* active_count_; 00230 00231 private: 00232 friend class SerfFetch; 00233 00234 static bool ParseHttpsOptions(StringPiece directive, uint32* options, 00235 GoogleString* error_message); 00236 00237 Variable* request_count_; 00238 Variable* byte_count_; 00239 Variable* time_duration_ms_; 00240 Variable* cancel_count_; 00241 Variable* timeout_count_; 00242 Variable* failure_count_; 00243 Variable* cert_errors_; 00244 const int64 timeout_ms_; 00245 bool force_threaded_; 00246 bool shutdown_; 00247 bool list_outstanding_urls_on_error_; 00248 bool track_original_content_length_; 00249 uint32 https_options_; 00250 MessageHandler* message_handler_; 00251 00254 std::set<const ContentType*> inflation_content_type_blacklist_; 00255 00256 DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher); 00257 }; 00258 00259 } 00260 00261 #endif ///< NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_