Page Speed Optimization Libraries  1.7.30.2
net/instaweb/system/public/serf_url_async_fetcher.h
Go to the documentation of this file.
00001 // Copyright 2010 Google Inc.
00017 
00018 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
00019 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
00020 
00021 #include <set>
00022 #include <vector>
00023 
00024 #include "net/instaweb/http/public/url_async_fetcher.h"
00025 #include "net/instaweb/util/public/basictypes.h"
00026 #include "net/instaweb/util/public/pool.h"
00027 #include "net/instaweb/util/public/string.h"
00028 #include "net/instaweb/util/public/string_util.h"
00029 #include "net/instaweb/util/public/thread_system.h"
00030 
00041 #ifndef SERF_HTTPS_FETCHING
00042 #define SERF_HTTPS_FETCHING 1
00043 #endif
00044 
00045 struct apr_pool_t;
00046 struct serf_context_t;
00047 
00048 namespace net_instaweb {
00049 
00050 class AsyncFetch;
00051 class MessageHandler;
00052 class Statistics;
00053 class SerfFetch;
00054 class SerfThreadedFetcher;
00055 class Timer;
00056 class Variable;
00057 struct ContentType;
00058 
00059 struct SerfStats {
00060   static const char kSerfFetchRequestCount[];
00061   static const char kSerfFetchByteCount[];
00062   static const char kSerfFetchTimeDurationMs[];
00063   static const char kSerfFetchCancelCount[];
00064   static const char kSerfFetchActiveCount[];
00065   static const char kSerfFetchTimeoutCount[];
00066   static const char kSerfFetchFailureCount[];
00067   static const char kSerfFetchCertErrors[];
00068 };
00069 
00074 #define SERF_HTTPS_KEYWORDS \
00075   "enable,disable,allow_self_signed," \
00076   "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
00077 
00084 class SerfUrlAsyncFetcher : public UrlAsyncFetcher {
00085  public:
00086   enum WaitChoice {
00087     kThreadedOnly,
00088     kMainlineOnly,
00089     kThreadedAndMainline
00090   };
00091 
00092   SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
00093                       ThreadSystem* thread_system,
00094                       Statistics* statistics, Timer* timer, int64 timeout_ms,
00095                       MessageHandler* handler);
00096   SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
00097   virtual ~SerfUrlAsyncFetcher();
00098 
00099   static void InitStats(Statistics* statistics);
00100 
00103   virtual void ShutDown();
00104 
00105   virtual bool SupportsHttps() const;
00106 
00107   virtual void Fetch(const GoogleString& url,
00108                      MessageHandler* message_handler,
00109                      AsyncFetch* callback);
00111   int Poll(int64 max_wait_ms);
00112 
00113   bool WaitForActiveFetches(int64 max_milliseconds,
00114                             MessageHandler* message_handler,
00115                             WaitChoice wait_choice);
00116 
00119   void FetchComplete(SerfFetch* fetch);
00120 
00122   void ReportCompletedFetchStats(SerfFetch* fetch);
00123 
00124   apr_pool_t* pool() const { return pool_; }
00125   serf_context_t* serf_context() const { return serf_context_; }
00126 
00127   void PrintActiveFetches(MessageHandler* handler) const;
00128   virtual int64 timeout_ms() { return timeout_ms_; }
00129   ThreadSystem* thread_system() { return thread_system_; }
00130 
00133   void set_list_outstanding_urls_on_error(bool x);
00134 
00137   bool track_original_content_length() const {
00138     return track_original_content_length_;
00139   }
00140   void set_track_original_content_length(bool x);
00141 
00142   void set_inflation_content_type_blacklist(
00143       const std::set<const ContentType*>& bypass_set) {
00144     inflation_content_type_blacklist_ = bypass_set;
00145   }
00146 
00156   bool SetHttpsOptions(StringPiece directive);
00157 
00160   static bool ValidateHttpsOptions(StringPiece directive,
00161                                    GoogleString* error_message) {
00162     uint32 options;
00163     return ParseHttpsOptions(directive, &options, error_message);
00164   }
00165 
00166   void SetSslCertificatesDir(StringPiece dir);
00167   const GoogleString& ssl_certificates_dir() const {
00168     return ssl_certificates_dir_;
00169   }
00170 
00171   void SetSslCertificatesFile(StringPiece file);
00172   const GoogleString& ssl_certificates_file() const {
00173     return ssl_certificates_file_;
00174   }
00175 
00176  protected:
00177   typedef Pool<SerfFetch> SerfFetchPool;
00178 
00180   inline bool allow_https() const;
00181   inline bool allow_self_signed() const;
00182   inline bool allow_unknown_certificate_authority() const;
00183   inline bool allow_certificate_not_yet_valid() const;
00184 
00185   void set_https_options(uint32 https_options) {
00186     https_options_ = https_options;
00187   }
00188 
00189   void Init(apr_pool_t* parent_pool, const char* proxy);
00190   bool SetupProxy(const char* proxy);
00191 
00196   bool StartFetch(SerfFetch* fetch);
00197 
00202   virtual bool AnyPendingFetches();
00205   int ApproximateNumActiveFetches();
00206 
00207   void CancelActiveFetches();
00208   void CancelActiveFetchesMutexHeld();
00209   bool WaitForActiveFetchesHelper(int64 max_ms,
00210                                   MessageHandler* message_handler);
00211 
00215   void CleanupFetchesWithErrors();
00216 
00218   bool shutdown() const { return shutdown_; }
00219   void set_shutdown(bool s) { shutdown_ = s; }
00220 
00221   apr_pool_t* pool_;
00222   ThreadSystem* thread_system_;
00223   Timer* timer_;
00224 
00226   ThreadSystem::CondvarCapableMutex* mutex_;
00227   serf_context_t* serf_context_;
00228   SerfFetchPool active_fetches_;
00229 
00230   typedef std::vector<SerfFetch*> FetchVector;
00231   SerfFetchPool completed_fetches_;
00232   SerfThreadedFetcher* threaded_fetcher_;
00233 
00236   Variable* active_count_;
00237 
00238  private:
00239   friend class SerfFetch; 
00240 
00241   static bool ParseHttpsOptions(StringPiece directive, uint32* options,
00242                                 GoogleString* error_message);
00243 
00244   Variable* request_count_;
00245   Variable* byte_count_;
00246   Variable* time_duration_ms_;
00247   Variable* cancel_count_;
00248   Variable* timeout_count_;
00249   Variable* failure_count_;
00250   Variable* cert_errors_;
00251   const int64 timeout_ms_;
00252   bool shutdown_;
00253   bool list_outstanding_urls_on_error_;
00254   bool track_original_content_length_;
00255   uint32 https_options_; 
00256   MessageHandler* message_handler_;
00257   GoogleString ssl_certificates_dir_;
00258   GoogleString ssl_certificates_file_;
00259 
00262   std::set<const ContentType*> inflation_content_type_blacklist_;
00263 
00264   DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
00265 };
00266 
00267 }  
00268 
00269 #endif  ///< NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines