Page Speed Optimization Libraries  1.3.25.1
net/instaweb/apache/serf_url_async_fetcher.h
Go to the documentation of this file.
00001 // Copyright 2010 Google Inc.
00017 
00018 #ifndef NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00019 #define NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00020 
00021 #include <vector>
00022 
00023 #include "net/instaweb/http/public/url_pollable_async_fetcher.h"
00024 #include "net/instaweb/util/public/basictypes.h"
00025 #include "net/instaweb/util/public/pool.h"
00026 #include "net/instaweb/util/public/string.h"
00027 #include "net/instaweb/util/public/string_util.h"
00028 #include "net/instaweb/util/public/thread_system.h"
00029 
00040 #ifndef SERF_HTTPS_FETCHING
00041 #define SERF_HTTPS_FETCHING 0
00042 #endif
00043 
00044 struct apr_pool_t;
00045 struct serf_context_t;
00046 
00047 namespace net_instaweb {
00048 
00049 class AsyncFetch;
00050 class MessageHandler;
00051 class Statistics;
00052 class SerfFetch;
00053 class SerfThreadedFetcher;
00054 class Timer;
00055 class Variable;
00056 
00057 struct SerfStats {
00058   static const char kSerfFetchRequestCount[];
00059   static const char kSerfFetchByteCount[];
00060   static const char kSerfFetchTimeDurationMs[];
00061   static const char kSerfFetchCancelCount[];
00062   static const char kSerfFetchActiveCount[];
00063   static const char kSerfFetchTimeoutCount[];
00064   static const char kSerfFetchFailureCount[];
00065   static const char kSerfFetchCertErrors[];
00066 };
00067 
00072 #define SERF_HTTPS_KEYWORDS \
00073   "enable,disable,allow_self_signed," \
00074   "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
00075 
00082 class SerfUrlAsyncFetcher : public UrlPollableAsyncFetcher {
00083  public:
00084   enum WaitChoice {
00085     kThreadedOnly,
00086     kMainlineOnly,
00087     kThreadedAndMainline
00088   };
00089 
00090   SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
00091                       ThreadSystem* thread_system,
00092                       Statistics* statistics, Timer* timer, int64 timeout_ms,
00093                       MessageHandler* handler);
00094   SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
00095   virtual ~SerfUrlAsyncFetcher();
00096 
00097   static void InitStats(Statistics* statistics);
00098 
00101   virtual void ShutDown();
00102 
00103   virtual bool SupportsHttps() const;
00104 
00105   virtual void Fetch(const GoogleString& url,
00106                      MessageHandler* message_handler,
00107                      AsyncFetch* callback);
00108 
00109   virtual int Poll(int64 max_wait_ms);
00110 
00111   bool WaitForActiveFetches(int64 max_milliseconds,
00112                             MessageHandler* message_handler,
00113                             WaitChoice wait_choice);
00114 
00117   void FetchComplete(SerfFetch* fetch);
00118   apr_pool_t* pool() const { return pool_; }
00119   serf_context_t* serf_context() const { return serf_context_; }
00120 
00121   void PrintActiveFetches(MessageHandler* handler) const;
00122   virtual int64 timeout_ms() { return timeout_ms_; }
00123   ThreadSystem* thread_system() { return thread_system_; }
00124 
00131   void set_force_threaded(bool x) { force_threaded_ = x; }
00132 
00135   void set_list_outstanding_urls_on_error(bool x);
00136 
00139   bool track_original_content_length() const {
00140     return track_original_content_length_;
00141   }
00142   void set_track_original_content_length(bool x);
00143 
00153   bool SetHttpsOptions(StringPiece directive);
00154 
00157   static bool ValidateHttpsOptions(StringPiece directive,
00158                                    GoogleString* error_message) {
00159     uint32 options;
00160     return ParseHttpsOptions(directive, &options, error_message);
00161   }
00162 
00163  protected:
00164   typedef Pool<SerfFetch> SerfFetchPool;
00165 
00167   inline bool allow_https() const;
00168   inline bool allow_self_signed() const;
00169   inline bool allow_unknown_certificate_authority() const;
00170   inline bool allow_certificate_not_yet_valid() const;
00171 
00172   void set_https_options(uint32 https_options) {
00173     https_options_ = https_options;
00174   }
00175 
00176   void Init(apr_pool_t* parent_pool, const char* proxy);
00177   bool SetupProxy(const char* proxy);
00178 
00183   bool StartFetch(SerfFetch* fetch);
00184 
00189   virtual bool AnyPendingFetches();
00192   int ApproximateNumActiveFetches();
00193 
00194   void CancelActiveFetches();
00195   void CancelActiveFetchesMutexHeld();
00196   bool WaitForActiveFetchesHelper(int64 max_ms,
00197                                   MessageHandler* message_handler);
00198 
00202   void CleanupFetchesWithErrors();
00203 
00205   bool shutdown() const { return shutdown_; }
00206   void set_shutdown(bool s) { shutdown_ = s; }
00207 
00208   apr_pool_t* pool_;
00209   ThreadSystem* thread_system_;
00210   Timer* timer_;
00211 
00213   ThreadSystem::CondvarCapableMutex* mutex_;
00214   serf_context_t* serf_context_;
00215   SerfFetchPool active_fetches_;
00216 
00217   typedef std::vector<SerfFetch*> FetchVector;
00218   SerfFetchPool completed_fetches_;
00219   SerfThreadedFetcher* threaded_fetcher_;
00220 
00223   Variable* active_count_;
00224 
00225  private:
00226   friend class SerfFetch; 
00227 
00228   static bool ParseHttpsOptions(StringPiece directive, uint32* options,
00229                                 GoogleString* error_message);
00230 
00231   Variable* request_count_;
00232   Variable* byte_count_;
00233   Variable* time_duration_ms_;
00234   Variable* cancel_count_;
00235   Variable* timeout_count_;
00236   Variable* failure_count_;
00237   Variable* cert_errors_;
00238   const int64 timeout_ms_;
00239   bool force_threaded_;
00240   bool shutdown_;
00241   bool list_outstanding_urls_on_error_;
00242   bool track_original_content_length_;
00243   uint32 https_options_; 
00244   MessageHandler* message_handler_;
00245 
00246   DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
00247 };
00248 
00249 }  
00250 
00251 #endif  ///< NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines