00001
00017
00018 #ifndef NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00019 #define NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_
00020
00021 #include <set>
00022 #include <string>
00023 #include <vector>
00024 #include <list>
00025 #include <map>
00026 #include "net/instaweb/util/public/basictypes.h"
00027 #include "net/instaweb/util/public/message_handler.h"
00028 #include "net/instaweb/util/public/pool.h"
00029 #include "net/instaweb/util/public/pool_element.h"
00030 #include "net/instaweb/util/public/thread_system.h"
00031 #include "net/instaweb/http/public/response_headers.h"
00032 #include "net/instaweb/http/public/url_pollable_async_fetcher.h"
00033
00034 struct apr_pool_t;
00035 struct serf_context_t;
00036 struct apr_thread_mutex_t;
00037
00038 namespace net_instaweb {
00039
00040 class Statistics;
00041 class SerfFetch;
00042 class SerfThreadedFetcher;
00043 class Timer;
00044 class Variable;
00045
00046 struct SerfStats {
00047 static const char kSerfFetchRequestCount[];
00048 static const char kSerfFetchByteCount[];
00049 static const char kSerfFetchTimeDurationMs[];
00050 static const char kSerfFetchCancelCount[];
00051 static const char kSerfFetchActiveCount[];
00052 static const char kSerfFetchTimeoutCount[];
00053 static const char kSerfFetchFailureCount[];
00054 };
00055
00062 class SerfUrlAsyncFetcher : public UrlPollableAsyncFetcher {
00063 public:
00064 SerfUrlAsyncFetcher(const char* proxy, apr_pool_t* pool,
00065 ThreadSystem* thread_system,
00066 Statistics* statistics, Timer* timer, int64 timeout_ms,
00067 MessageHandler* handler);
00068 SerfUrlAsyncFetcher(SerfUrlAsyncFetcher* parent, const char* proxy);
00069 virtual ~SerfUrlAsyncFetcher();
00070
00071 static void Initialize(Statistics* statistics);
00072
00075 virtual void ShutDown();
00076
00077 virtual bool SupportsHttps() const { return false; }
00078
00079 virtual bool Fetch(const GoogleString& url,
00080 MessageHandler* message_handler,
00081 AsyncFetch* callback);
00082
00083 virtual int Poll(int64 max_wait_ms);
00084
00085 enum WaitChoice {
00086 kThreadedOnly,
00087 kMainlineOnly,
00088 kThreadedAndMainline
00089 };
00090
00091 bool WaitForActiveFetches(int64 max_milliseconds,
00092 MessageHandler* message_handler,
00093 WaitChoice wait_choice);
00094
00097 void FetchComplete(SerfFetch* fetch);
00098 apr_pool_t* pool() const { return pool_; }
00099 serf_context_t* serf_context() const { return serf_context_; }
00100
00101 void PrintActiveFetches(MessageHandler* handler) const;
00102 virtual int64 timeout_ms() { return timeout_ms_; }
00103 ThreadSystem* thread_system() { return thread_system_; }
00104
00111 void set_force_threaded(bool x) { force_threaded_ = x; }
00112
00115 void set_list_outstanding_urls_on_error(bool x);
00116
00117 protected:
00118 typedef Pool<SerfFetch> SerfFetchPool;
00119
00120 void Init(apr_pool_t* parent_pool, const char* proxy);
00121 bool SetupProxy(const char* proxy);
00122
00127 bool StartFetch(SerfFetch* fetch);
00128
00133 virtual bool AnyPendingFetches();
00136 int ApproximateNumActiveFetches();
00137
00138 void CancelActiveFetches();
00139 void CancelActiveFetchesMutexHeld();
00140 bool WaitForActiveFetchesHelper(int64 max_ms,
00141 MessageHandler* message_handler);
00142
00146 void CleanupFetchesWithErrors();
00147
00149 bool shutdown() const { return shutdown_; }
00150 void set_shutdown(bool s) { shutdown_ = s; }
00151
00152 apr_pool_t* pool_;
00153 ThreadSystem* thread_system_;
00154 Timer* timer_;
00155
00157 ThreadSystem::CondvarCapableMutex* mutex_;
00158 serf_context_t* serf_context_;
00159 SerfFetchPool active_fetches_;
00160
00161 typedef std::vector<SerfFetch*> FetchVector;
00162 SerfFetchPool completed_fetches_;
00163 SerfThreadedFetcher* threaded_fetcher_;
00164
00167 Variable* active_count_;
00168
00169 private:
00170 friend class SerfFetch;
00171
00172 Variable* request_count_;
00173 Variable* byte_count_;
00174 Variable* time_duration_ms_;
00175 Variable* cancel_count_;
00176 Variable* timeout_count_;
00177 Variable* failure_count_;
00178 const int64 timeout_ms_;
00179 bool force_threaded_;
00180 bool shutdown_;
00181 bool list_outstanding_urls_on_error_;
00182 MessageHandler* message_handler_;
00183
00184 DISALLOW_COPY_AND_ASSIGN(SerfUrlAsyncFetcher);
00185 };
00186
00187 }
00188
00189 #endif ///< NET_INSTAWEB_APACHE_SERF_URL_ASYNC_FETCHER_H_