00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00018
00019 #ifndef NET_INSTAWEB_HTTP_PUBLIC_RATE_CONTROLLING_URL_ASYNC_FETCHER_H_
00020 #define NET_INSTAWEB_HTTP_PUBLIC_RATE_CONTROLLING_URL_ASYNC_FETCHER_H_
00021
00022 #include <map>
00023
00024 #include "base/scoped_ptr.h"
00025 #include "net/instaweb/http/public/url_async_fetcher.h"
00026 #include "net/instaweb/util/public/basictypes.h"
00027 #include "net/instaweb/util/public/ref_counted_ptr.h"
00028 #include "net/instaweb/util/public/string.h"
00029
00030 namespace net_instaweb {
00031
00032 class AbstractMutex;
00033 class AsyncFetch;
00034 class MessageHandler;
00035 class Statistics;
00036 class ThreadSystem;
00037 class TimedVariable;
00038 class Variable;
00039
00048 class RateControllingUrlAsyncFetcher : public UrlAsyncFetcher {
00049 public:
00050 static const char kQueuedFetchCount[];
00051 static const char kDroppedFetchCount[];
00052 static const char kCurrentGlobalFetchQueueSize[];
00053
00054 RateControllingUrlAsyncFetcher(UrlAsyncFetcher* fetcher,
00055 int max_global_queue_size,
00056 int per_host_outgoing_request_threshold,
00057 int per_host_queued_request_threshold,
00058 ThreadSystem* thread_system,
00059 Statistics* statistics);
00060
00061 virtual ~RateControllingUrlAsyncFetcher();
00062
00063 virtual bool SupportsHttps() const {
00064 return base_fetcher_->SupportsHttps();
00065 }
00066
00067 virtual bool Fetch(const GoogleString& url,
00068 MessageHandler* message_handler,
00069 AsyncFetch* fetch);
00070
00072 static void Initialize(Statistics* statistics);
00073
00074 private:
00075 class HostFetchInfo;
00076 class CustomFetch;
00077 friend class CustomFetch;
00078
00079 typedef RefCountedPtr<HostFetchInfo> HostFetchInfoPtr;
00080
00081 typedef std::map<GoogleString, HostFetchInfoPtr*> HostFetchInfoMap;
00082
00084 void DeleteFetchInfoIfPossible(const HostFetchInfoPtr& fetch_info);
00085
00087 UrlAsyncFetcher* base_fetcher_;
00089 const int max_global_queue_size_;
00091 const int per_host_outgoing_request_threshold_;
00093 const int per_host_queued_request_threshold_;
00094 ThreadSystem* thread_system_;
00095
00097 HostFetchInfoMap fetch_info_map_;
00098 scoped_ptr<AbstractMutex> mutex_;
00099
00100 TimedVariable* queued_fetch_count_;
00101 TimedVariable* dropped_fetch_count_;
00104 Variable* current_global_fetch_queue_size_;
00105
00106 DISALLOW_COPY_AND_ASSIGN(RateControllingUrlAsyncFetcher);
00107 };
00108
00109 }
00110
00111 #endif ///< NET_INSTAWEB_HTTP_PUBLIC_RATE_CONTROLLING_URL_ASYNC_FETCHER_H_