Page Speed Optimization Libraries
1.7.30.4
|
00001 // Copyright 2010 Google Inc. 00017 00018 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_ 00019 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_ 00020 00021 #include <vector> 00022 00023 #include "net/instaweb/http/public/url_async_fetcher.h" 00024 #include "net/instaweb/util/public/basictypes.h" 00025 #include "net/instaweb/util/public/gtest_prod.h" 00026 #include "net/instaweb/util/public/pool.h" 00027 #include "net/instaweb/util/public/string.h" 00028 #include "net/instaweb/util/public/string_util.h" 00029 #include "net/instaweb/util/public/thread_system.h" 00030 00041 #ifndef SERF_HTTPS_FETCHING 00042 #define SERF_HTTPS_FETCHING 1 00043 #endif 00044 00045 struct apr_pool_t; 00046 struct apr_uri_t; 00047 struct serf_context_t; 00048 00049 namespace net_instaweb { 00050 00051 class AsyncFetch; 00052 class MessageHandler; 00053 class Statistics; 00054 class SerfFetch; 00055 class SerfThreadedFetcher; 00056 class Timer; 00057 class Variable; 00058 00059 struct SerfStats { 00060 static const char kSerfFetchRequestCount[]; 00061 static const char kSerfFetchByteCount[]; 00062 static const char kSerfFetchTimeDurationMs[]; 00063 static const char kSerfFetchCancelCount[]; 00064 static const char kSerfFetchActiveCount[]; 00065 static const char kSerfFetchTimeoutCount[]; 00066 static const char kSerfFetchFailureCount[]; 00067 static const char kSerfFetchCertErrors[]; 00068 }; 00069 00074 #define SERF_HTTPS_KEYWORDS \ 00075 "enable,disable,allow_self_signed," \ 00076 "allow_unknown_certificate_authority,allow_certificate_not_yet_valid" 00077 00084 class SerfUrlAsyncFetcher : public UrlAsyncFetcher { 00085 public: 00086 enum WaitChoice { 00087 kThreadedOnly, 00088 kMainlineOnly, 00089 kThreadedAndMainline 00090 }; 00091 00092 SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool, 00093 ThreadSystem* thread_system, 00094 Statistics* statistics, Timer* timer, int64 timeout_ms, 00095 MessageHandler* handler); 00096 SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy); 00097 virtual ~SerfUrlAsyncFetcher(); 00098 00099 static void InitStats(Statistics* statistics); 00100 00103 virtual void ShutDown(); 00104 00105 virtual bool SupportsHttps() const; 00106 00107 virtual void Fetch(const GoogleString& url, 00108 MessageHandler* message_handler, 00109 AsyncFetch* callback); 00111 int Poll(int64 max_wait_ms); 00112 00113 bool WaitForActiveFetches(int64 max_milliseconds, 00114 MessageHandler* message_handler, 00115 WaitChoice wait_choice); 00116 00119 void FetchComplete(SerfFetch* fetch); 00120 00122 void ReportCompletedFetchStats(SerfFetch* fetch); 00123 00124 apr_pool_t* pool() const { return pool_; } 00125 serf_context_t* serf_context() const { return serf_context_; } 00126 00127 void PrintActiveFetches(MessageHandler* handler) const; 00128 virtual int64 timeout_ms() { return timeout_ms_; } 00129 ThreadSystem* thread_system() { return thread_system_; } 00130 00133 void set_list_outstanding_urls_on_error(bool x); 00134 00137 bool track_original_content_length() const { 00138 return track_original_content_length_; 00139 } 00140 void set_track_original_content_length(bool x); 00141 00151 bool SetHttpsOptions(StringPiece directive); 00152 00155 static bool ValidateHttpsOptions(StringPiece directive, 00156 GoogleString* error_message) { 00157 uint32 options; 00158 return ParseHttpsOptions(directive, &options, error_message); 00159 } 00160 00161 void SetSslCertificatesDir(StringPiece dir); 00162 const GoogleString& ssl_certificates_dir() const { 00163 return ssl_certificates_dir_; 00164 } 00165 00166 void SetSslCertificatesFile(StringPiece file); 00167 const GoogleString& ssl_certificates_file() const { 00168 return ssl_certificates_file_; 00169 } 00170 00171 protected: 00172 typedef Pool<SerfFetch> SerfFetchPool; 00173 00175 inline bool allow_https() const; 00176 inline bool allow_self_signed() const; 00177 inline bool allow_unknown_certificate_authority() const; 00178 inline bool allow_certificate_not_yet_valid() const; 00179 00180 void set_https_options(uint32 https_options) { 00181 https_options_ = https_options; 00182 } 00183 00184 void Init(apr_pool_t* parent_pool, const char* proxy); 00185 bool SetupProxy(const char* proxy); 00186 00191 bool StartFetch(SerfFetch* fetch); 00192 00197 virtual bool AnyPendingFetches(); 00200 int ApproximateNumActiveFetches(); 00201 00202 void CancelActiveFetches(); 00203 void CancelActiveFetchesMutexHeld(); 00204 bool WaitForActiveFetchesHelper(int64 max_ms, 00205 MessageHandler* message_handler); 00206 00210 void CleanupFetchesWithErrors(); 00211 00213 bool shutdown() const { return shutdown_; } 00214 void set_shutdown(bool s) { shutdown_ = s; } 00215 00216 apr_pool_t* pool_; 00217 ThreadSystem* thread_system_; 00218 Timer* timer_; 00219 00221 ThreadSystem::CondvarCapableMutex* mutex_; 00222 serf_context_t* serf_context_; 00223 SerfFetchPool active_fetches_; 00224 00225 typedef std::vector<SerfFetch*> FetchVector; 00226 SerfFetchPool completed_fetches_; 00227 SerfThreadedFetcher* threaded_fetcher_; 00228 00231 Variable* active_count_; 00232 00233 private: 00234 friend class SerfFetch; 00235 00237 static const char* ExtractHostHeader(const apr_uri_t& uri, 00238 apr_pool_t* pool); 00239 FRIEND_TEST(SerfUrlAsyncFetcherTest, TestHostConstruction); 00240 00243 static GoogleString RemovePortFromHostHeader(const GoogleString& in); 00244 FRIEND_TEST(SerfUrlAsyncFetcherTest, TestPortRemoval); 00245 00246 static bool ParseHttpsOptions(StringPiece directive, uint32* options, 00247 GoogleString* error_message); 00248 00249 Variable* request_count_; 00250 Variable* byte_count_; 00251 Variable* time_duration_ms_; 00252 Variable* cancel_count_; 00253 Variable* timeout_count_; 00254 Variable* failure_count_; 00255 Variable* cert_errors_; 00256 const int64 timeout_ms_; 00257 bool shutdown_; 00258 bool list_outstanding_urls_on_error_; 00259 bool track_original_content_length_; 00260 uint32 https_options_; 00261 MessageHandler* message_handler_; 00262 GoogleString ssl_certificates_dir_; 00263 GoogleString ssl_certificates_file_; 00264 00265 DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher); 00266 }; 00267 00268 } 00269 00270 #endif ///< NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_