Page Speed Optimization Libraries  1.4.26.1
net/instaweb/apache/serf_url_async_fetcher.h
Go to the documentation of this file.
00001 // Copyright 2010 Google Inc.
00017 
00018 #ifndef NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00019 #define NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00020 
00021 #include <set>
00022 #include <vector>
00023 
00024 #include "net/instaweb/http/public/url_pollable_async_fetcher.h"
00025 #include "net/instaweb/util/public/basictypes.h"
00026 #include "net/instaweb/util/public/pool.h"
00027 #include "net/instaweb/util/public/string.h"
00028 #include "net/instaweb/util/public/string_util.h"
00029 #include "net/instaweb/util/public/thread_system.h"
00030 
00041 #ifndef SERF_HTTPS_FETCHING
00042 #define SERF_HTTPS_FETCHING 0
00043 #endif
00044 
00045 struct apr_pool_t;
00046 struct serf_context_t;
00047 
00048 namespace net_instaweb {
00049 
00050 class AsyncFetch;
00051 class MessageHandler;
00052 class Statistics;
00053 class SerfFetch;
00054 class SerfThreadedFetcher;
00055 class Timer;
00056 class Variable;
00057 
00058 struct SerfStats {
00059   static const char kSerfFetchRequestCount[];
00060   static const char kSerfFetchByteCount[];
00061   static const char kSerfFetchTimeDurationMs[];
00062   static const char kSerfFetchCancelCount[];
00063   static const char kSerfFetchActiveCount[];
00064   static const char kSerfFetchTimeoutCount[];
00065   static const char kSerfFetchFailureCount[];
00066   static const char kSerfFetchCertErrors[];
00067 };
00068 
00073 #define SERF_HTTPS_KEYWORDS \
00074   "enable,disable,allow_self_signed," \
00075   "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
00076 
00083 class SerfUrlAsyncFetcher : public UrlPollableAsyncFetcher {
00084  public:
00085   enum WaitChoice {
00086     kThreadedOnly,
00087     kMainlineOnly,
00088     kThreadedAndMainline
00089   };
00090 
00091   SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
00092                       ThreadSystem* thread_system,
00093                       Statistics* statistics, Timer* timer, int64 timeout_ms,
00094                       MessageHandler* handler);
00095   SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
00096   virtual ~SerfUrlAsyncFetcher();
00097 
00098   static void InitStats(Statistics* statistics);
00099 
00102   virtual void ShutDown();
00103 
00104   virtual bool SupportsHttps() const;
00105 
00106   virtual void Fetch(const GoogleString& url,
00107                      MessageHandler* message_handler,
00108                      AsyncFetch* callback);
00109 
00110   virtual int Poll(int64 max_wait_ms);
00111 
00112   bool WaitForActiveFetches(int64 max_milliseconds,
00113                             MessageHandler* message_handler,
00114                             WaitChoice wait_choice);
00115 
00118   void FetchComplete(SerfFetch* fetch);
00119   apr_pool_t* pool() const { return pool_; }
00120   serf_context_t* serf_context() const { return serf_context_; }
00121 
00122   void PrintActiveFetches(MessageHandler* handler) const;
00123   virtual int64 timeout_ms() { return timeout_ms_; }
00124   ThreadSystem* thread_system() { return thread_system_; }
00125 
00132   void set_force_threaded(bool x) { force_threaded_ = x; }
00133 
00136   void set_list_outstanding_urls_on_error(bool x);
00137 
00140   bool track_original_content_length() const {
00141     return track_original_content_length_;
00142   }
00143   void set_track_original_content_length(bool x);
00144 
00145   void set_inflation_content_type_blacklist(
00146       const std::set<const ContentType*>& bypass_set) {
00147     inflation_content_type_blacklist_ = bypass_set;
00148   }
00149 
00159   bool SetHttpsOptions(StringPiece directive);
00160 
00163   static bool ValidateHttpsOptions(StringPiece directive,
00164                                    GoogleString* error_message) {
00165     uint32 options;
00166     return ParseHttpsOptions(directive, &options, error_message);
00167   }
00168 
00169  protected:
00170   typedef Pool<SerfFetch> SerfFetchPool;
00171 
00173   inline bool allow_https() const;
00174   inline bool allow_self_signed() const;
00175   inline bool allow_unknown_certificate_authority() const;
00176   inline bool allow_certificate_not_yet_valid() const;
00177 
00178   void set_https_options(uint32 https_options) {
00179     https_options_ = https_options;
00180   }
00181 
00182   void Init(apr_pool_t* parent_pool, const char* proxy);
00183   bool SetupProxy(const char* proxy);
00184 
00189   bool StartFetch(SerfFetch* fetch);
00190 
00195   virtual bool AnyPendingFetches();
00198   int ApproximateNumActiveFetches();
00199 
00200   void CancelActiveFetches();
00201   void CancelActiveFetchesMutexHeld();
00202   bool WaitForActiveFetchesHelper(int64 max_ms,
00203                                   MessageHandler* message_handler);
00204 
00208   void CleanupFetchesWithErrors();
00209 
00211   bool shutdown() const { return shutdown_; }
00212   void set_shutdown(bool s) { shutdown_ = s; }
00213 
00214   apr_pool_t* pool_;
00215   ThreadSystem* thread_system_;
00216   Timer* timer_;
00217 
00219   ThreadSystem::CondvarCapableMutex* mutex_;
00220   serf_context_t* serf_context_;
00221   SerfFetchPool active_fetches_;
00222 
00223   typedef std::vector<SerfFetch*> FetchVector;
00224   SerfFetchPool completed_fetches_;
00225   SerfThreadedFetcher* threaded_fetcher_;
00226 
00229   Variable* active_count_;
00230 
00231  private:
00232   friend class SerfFetch; 
00233 
00234   static bool ParseHttpsOptions(StringPiece directive, uint32* options,
00235                                 GoogleString* error_message);
00236 
00237   Variable* request_count_;
00238   Variable* byte_count_;
00239   Variable* time_duration_ms_;
00240   Variable* cancel_count_;
00241   Variable* timeout_count_;
00242   Variable* failure_count_;
00243   Variable* cert_errors_;
00244   const int64 timeout_ms_;
00245   bool force_threaded_;
00246   bool shutdown_;
00247   bool list_outstanding_urls_on_error_;
00248   bool track_original_content_length_;
00249   uint32 https_options_; 
00250   MessageHandler* message_handler_;
00251 
00254   std::set<const ContentType*> inflation_content_type_blacklist_;
00255 
00256   DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
00257 };
00258 
00259 }  
00260 
00261 #endif  ///< NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines