Page Speed Optimization Libraries  1.7.30.4
net/instaweb/system/public/serf_url_async_fetcher.h
Go to the documentation of this file.
00001 // Copyright 2010 Google Inc.
00017 
00018 #ifndef NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
00019 #define NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
00020 
00021 #include <vector>
00022 
00023 #include "net/instaweb/http/public/url_async_fetcher.h"
00024 #include "net/instaweb/util/public/basictypes.h"
00025 #include "net/instaweb/util/public/gtest_prod.h"
00026 #include "net/instaweb/util/public/pool.h"
00027 #include "net/instaweb/util/public/string.h"
00028 #include "net/instaweb/util/public/string_util.h"
00029 #include "net/instaweb/util/public/thread_system.h"
00030 
00041 #ifndef SERF_HTTPS_FETCHING
00042 #define SERF_HTTPS_FETCHING 1
00043 #endif
00044 
00045 struct apr_pool_t;
00046 struct apr_uri_t;
00047 struct serf_context_t;
00048 
00049 namespace net_instaweb {
00050 
00051 class AsyncFetch;
00052 class MessageHandler;
00053 class Statistics;
00054 class SerfFetch;
00055 class SerfThreadedFetcher;
00056 class Timer;
00057 class Variable;
00058 
00059 struct SerfStats {
00060   static const char kSerfFetchRequestCount[];
00061   static const char kSerfFetchByteCount[];
00062   static const char kSerfFetchTimeDurationMs[];
00063   static const char kSerfFetchCancelCount[];
00064   static const char kSerfFetchActiveCount[];
00065   static const char kSerfFetchTimeoutCount[];
00066   static const char kSerfFetchFailureCount[];
00067   static const char kSerfFetchCertErrors[];
00068 };
00069 
00074 #define SERF_HTTPS_KEYWORDS \
00075   "enable,disable,allow_self_signed," \
00076   "allow_unknown_certificate_authority,allow_certificate_not_yet_valid"
00077 
00084 class SerfUrlAsyncFetcher : public UrlAsyncFetcher {
00085  public:
00086   enum WaitChoice {
00087     kThreadedOnly,
00088     kMainlineOnly,
00089     kThreadedAndMainline
00090   };
00091 
00092   SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
00093                       ThreadSystem* thread_system,
00094                       Statistics* statistics, Timer* timer, int64 timeout_ms,
00095                       MessageHandler* handler);
00096   SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
00097   virtual ~SerfUrlAsyncFetcher();
00098 
00099   static void InitStats(Statistics* statistics);
00100 
00103   virtual void ShutDown();
00104 
00105   virtual bool SupportsHttps() const;
00106 
00107   virtual void Fetch(const GoogleString& url,
00108                      MessageHandler* message_handler,
00109                      AsyncFetch* callback);
00111   int Poll(int64 max_wait_ms);
00112 
00113   bool WaitForActiveFetches(int64 max_milliseconds,
00114                             MessageHandler* message_handler,
00115                             WaitChoice wait_choice);
00116 
00119   void FetchComplete(SerfFetch* fetch);
00120 
00122   void ReportCompletedFetchStats(SerfFetch* fetch);
00123 
00124   apr_pool_t* pool() const { return pool_; }
00125   serf_context_t* serf_context() const { return serf_context_; }
00126 
00127   void PrintActiveFetches(MessageHandler* handler) const;
00128   virtual int64 timeout_ms() { return timeout_ms_; }
00129   ThreadSystem* thread_system() { return thread_system_; }
00130 
00133   void set_list_outstanding_urls_on_error(bool x);
00134 
00137   bool track_original_content_length() const {
00138     return track_original_content_length_;
00139   }
00140   void set_track_original_content_length(bool x);
00141 
00151   bool SetHttpsOptions(StringPiece directive);
00152 
00155   static bool ValidateHttpsOptions(StringPiece directive,
00156                                    GoogleString* error_message) {
00157     uint32 options;
00158     return ParseHttpsOptions(directive, &options, error_message);
00159   }
00160 
00161   void SetSslCertificatesDir(StringPiece dir);
00162   const GoogleString& ssl_certificates_dir() const {
00163     return ssl_certificates_dir_;
00164   }
00165 
00166   void SetSslCertificatesFile(StringPiece file);
00167   const GoogleString& ssl_certificates_file() const {
00168     return ssl_certificates_file_;
00169   }
00170 
00171  protected:
00172   typedef Pool<SerfFetch> SerfFetchPool;
00173 
00175   inline bool allow_https() const;
00176   inline bool allow_self_signed() const;
00177   inline bool allow_unknown_certificate_authority() const;
00178   inline bool allow_certificate_not_yet_valid() const;
00179 
00180   void set_https_options(uint32 https_options) {
00181     https_options_ = https_options;
00182   }
00183 
00184   void Init(apr_pool_t* parent_pool, const char* proxy);
00185   bool SetupProxy(const char* proxy);
00186 
00191   bool StartFetch(SerfFetch* fetch);
00192 
00197   virtual bool AnyPendingFetches();
00200   int ApproximateNumActiveFetches();
00201 
00202   void CancelActiveFetches();
00203   void CancelActiveFetchesMutexHeld();
00204   bool WaitForActiveFetchesHelper(int64 max_ms,
00205                                   MessageHandler* message_handler);
00206 
00210   void CleanupFetchesWithErrors();
00211 
00213   bool shutdown() const { return shutdown_; }
00214   void set_shutdown(bool s) { shutdown_ = s; }
00215 
00216   apr_pool_t* pool_;
00217   ThreadSystem* thread_system_;
00218   Timer* timer_;
00219 
00221   ThreadSystem::CondvarCapableMutex* mutex_;
00222   serf_context_t* serf_context_;
00223   SerfFetchPool active_fetches_;
00224 
00225   typedef std::vector<SerfFetch*> FetchVector;
00226   SerfFetchPool completed_fetches_;
00227   SerfThreadedFetcher* threaded_fetcher_;
00228 
00231   Variable* active_count_;
00232 
00233  private:
00234   friend class SerfFetch; 
00235 
00237   static const char* ExtractHostHeader(const apr_uri_t& uri,
00238                                        apr_pool_t* pool);
00239   FRIEND_TEST(SerfUrlAsyncFetcherTest, TestHostConstruction);
00240 
00243   static GoogleString RemovePortFromHostHeader(const GoogleString& in);
00244   FRIEND_TEST(SerfUrlAsyncFetcherTest, TestPortRemoval);
00245 
00246   static bool ParseHttpsOptions(StringPiece directive, uint32* options,
00247                                 GoogleString* error_message);
00248 
00249   Variable* request_count_;
00250   Variable* byte_count_;
00251   Variable* time_duration_ms_;
00252   Variable* cancel_count_;
00253   Variable* timeout_count_;
00254   Variable* failure_count_;
00255   Variable* cert_errors_;
00256   const int64 timeout_ms_;
00257   bool shutdown_;
00258   bool list_outstanding_urls_on_error_;
00259   bool track_original_content_length_;
00260   uint32 https_options_; 
00261   MessageHandler* message_handler_;
00262   GoogleString ssl_certificates_dir_;
00263   GoogleString ssl_certificates_file_;
00264 
00265   DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
00266 };
00267 
00268 }  
00269 
00270 #endif  ///< NET_INSTAWEB_SYSTEM_PUBLIC_SERF_URL_ASYNC_FETCHER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines