Page Speed Optimization Libraries  1.4.26.1
net/instaweb/automatic/public/proxy_fetch.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00022 
00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00025 
00026 #include <map>
00027 #include <set>
00028 #include <vector>
00029 
00030 #include "net/instaweb/automatic/public/html_detector.h"
00031 #include "net/instaweb/http/public/async_fetch.h"
00032 #include "net/instaweb/http/public/meta_data.h"
00033 #include "net/instaweb/http/public/request_context.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00035 #include "net/instaweb/util/public/queued_worker_pool.h"
00036 #include "net/instaweb/util/public/basictypes.h"
00037 #include "net/instaweb/util/public/gtest_prod.h"
00038 #include "net/instaweb/util/public/property_cache.h"
00039 #include "net/instaweb/util/public/scoped_ptr.h"
00040 #include "net/instaweb/util/public/string.h"
00041 #include "net/instaweb/util/public/string_util.h"
00042 
00043 namespace net_instaweb {
00044 
00045 class AbstractClientState;
00046 class AbstractMutex;
00047 class CacheUrlAsyncFetcher;
00048 class Function;
00049 class LogRecord;
00050 class MessageHandler;
00051 class ProxyFetch;
00052 class ProxyFetchPropertyCallbackCollector;
00053 class QueuedAlarm;
00054 class ServerContext;
00055 class ResponseHeaders;
00056 class RewriteDriver;
00057 class RewriteOptions;
00058 class Timer;
00059 
00062 class ProxyFetchFactory {
00063  public:
00064   explicit ProxyFetchFactory(ServerContext* manager);
00065   ~ProxyFetchFactory();
00066 
00069   void StartNewProxyFetch(
00070       const GoogleString& url,
00071       AsyncFetch* async_fetch,
00072       RewriteDriver* driver,
00073       ProxyFetchPropertyCallbackCollector* property_callback,
00074       AsyncFetch* original_content_fetch);
00075 
00085   ProxyFetch* CreateNewProxyFetch(
00086       const GoogleString& url,
00087       AsyncFetch* async_fetch,
00088       RewriteDriver* driver,
00089       ProxyFetchPropertyCallbackCollector* property_callback,
00090       AsyncFetch* original_content_fetch);
00091 
00092   MessageHandler* message_handler() const { return handler_; }
00093 
00094  private:
00095   friend class ProxyFetch;
00096 
00101   void RegisterNewFetch(ProxyFetch* proxy_fetch);
00102   void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
00103 
00104   ServerContext* manager_;
00105   Timer* timer_;
00106   MessageHandler* handler_;
00107 
00108   scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
00109   std::set<ProxyFetch*> outstanding_proxy_fetches_;
00110 
00111   DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
00112 };
00113 
00122 class ProxyFetchPropertyCallback : public PropertyPage {
00123  public:
00125   enum CacheType {
00126     kPagePropertyCache,
00127     kClientPropertyCache,
00128     kDevicePropertyCache
00129   };
00130 
00131   ProxyFetchPropertyCallback(CacheType cache_type,
00132                              PropertyCache* property_cache,
00133                              const StringPiece& key,
00134                              UserAgentMatcher::DeviceType device_type,
00135                              ProxyFetchPropertyCallbackCollector* collector,
00136                              AbstractMutex* mutex);
00137 
00138   CacheType cache_type() const { return cache_type_; }
00139 
00140   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00141 
00143   virtual bool IsCacheValid(int64 write_timestamp_ms) const;
00144 
00145   virtual void Done(bool success);
00146 
00148   virtual void LogPageCohortInfo(LogRecord* log_record, int cohort_index);
00149 
00150  private:
00151   CacheType cache_type_;
00152   UserAgentMatcher::DeviceType device_type_;
00153   ProxyFetchPropertyCallbackCollector* collector_;
00154   GoogleString url_;
00155   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
00156 };
00157 
00159 class ProxyFetchPropertyCallbackCollector {
00160  public:
00161   ProxyFetchPropertyCallbackCollector(ServerContext* manager,
00162                                       const StringPiece& url,
00163                                       const RequestContextPtr& req_ctx,
00164                                       const RewriteOptions* options,
00165                                       UserAgentMatcher::DeviceType device_type);
00166   virtual ~ProxyFetchPropertyCallbackCollector();
00167 
00170   void AddCallback(ProxyFetchPropertyCallback* callback);
00171 
00178   void ConnectProxyFetch(ProxyFetch* proxy_fetch);
00179 
00186   void Detach(HttpStatus::Code status_code);
00187 
00190   PropertyPage* GetPropertyPage(
00191       ProxyFetchPropertyCallback::CacheType cache_type);
00192 
00195   PropertyPage* GetPropertyPageWithoutOwnership(
00196       ProxyFetchPropertyCallback::CacheType cache_type);
00197 
00207   void AddPostLookupTask(Function* func);
00208 
00212   bool IsCacheValid(int64 write_timestamp_ms) const;
00213 
00215   void Done(ProxyFetchPropertyCallback* callback, bool success);
00216 
00218   void UpdateStatusCodeInPropertyCache();
00219 
00220   const RequestContextPtr& request_context() { return request_context_; }
00221 
00223   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00224 
00225  private:
00226   std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
00227   std::map<ProxyFetchPropertyCallback::CacheType, PropertyPage*>
00228   property_pages_;
00229   scoped_ptr<AbstractMutex> mutex_;
00230   ServerContext* server_context_;
00231   GoogleString url_;
00232   RequestContextPtr request_context_;
00233   UserAgentMatcher::DeviceType device_type_;
00234   bool detached_; 
00235   bool done_; 
00236   bool success_; 
00237   ProxyFetch* proxy_fetch_; 
00238 
00239   scoped_ptr<std::vector<Function*> > post_lookup_task_vector_;
00240   const RewriteOptions* options_; 
00241   HttpStatus::Code status_code_; 
00242 
00243   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
00244 };
00245 
00266 class ProxyFetch : public SharedAsyncFetch {
00267  public:
00270   static const char kCollectorDone[];
00271   static const char kCollectorPrefix[];
00272   static const char kCollectorReady[];
00273   static const char kCollectorDelete[];
00274   static const char kCollectorDetach[];
00275   static const char kCollectorDoneDelete[];
00276 
00279   static const char kHeadersSetupRaceAlarmQueued[];
00280   static const char kHeadersSetupRaceDone[];
00281   static const char kHeadersSetupRaceFlush[];
00282   static const char kHeadersSetupRacePrefix[];
00283   static const char kHeadersSetupRaceWait[];
00284 
00291   static const int kTestSignalTimeoutMs = 200;
00292 
00293  protected:
00295   virtual void HandleHeadersComplete();
00296   virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
00297   virtual bool HandleFlush(MessageHandler* handler);
00298   virtual void HandleDone(bool success);
00299   virtual bool IsCachedResultValid(const ResponseHeaders& headers);
00300 
00301  private:
00302   friend class ProxyFetchFactory;
00303   friend class ProxyFetchPropertyCallbackCollector;
00304   friend class MockProxyFetch;
00305   FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
00306 
00309   virtual void PropertyCacheComplete(
00310       bool success, ProxyFetchPropertyCallbackCollector* collector);
00311 
00316   AbstractClientState* GetClientState(
00317       ProxyFetchPropertyCallbackCollector* collector);
00318 
00321   ProxyFetch(const GoogleString& url,
00322              bool cross_domain,
00323              ProxyFetchPropertyCallbackCollector* property_cache_callback,
00324              AsyncFetch* async_fetch,
00325              AsyncFetch* original_content_fetch,
00326              RewriteDriver* driver,
00327              ServerContext* manager,
00328              Timer* timer,
00329              ProxyFetchFactory* factory);
00330   virtual ~ProxyFetch();
00331 
00332   const RewriteOptions* Options();
00333 
00335   void SetupForHtml();
00336 
00338   void AddPagespeedHeader();
00339 
00342   bool StartParse();
00343 
00345   void StartFetch();
00346 
00348   void DoFetch();
00349 
00352   void ExecuteQueued();
00353 
00356   void ScheduleQueueExecutionIfNeeded();
00357 
00362   void Finish(bool success);
00363 
00365   void CompleteFinishParse(bool success);
00366 
00369   void FlushDone();
00370 
00373 
00375   void CancelIdleAlarm();
00376 
00378   void QueueIdleAlarm();
00379 
00381   void HandleIdleAlarm();
00382 
00383   GoogleString url_;
00384   ServerContext* server_context_;
00385   Timer* timer_;
00386 
00387   scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
00388 
00391   bool cross_domain_;
00392 
00394   bool claims_html_;
00395 
00398   bool started_parse_;
00399 
00401   bool parse_text_called_;
00402 
00404   bool done_called_;
00405 
00406   HtmlDetector html_detector_;
00407 
00412   ProxyFetchPropertyCallbackCollector* property_cache_callback_;
00413 
00417   AsyncFetch* original_content_fetch_;
00418 
00421   RewriteDriver* driver_;
00422 
00425   bool queue_run_job_created_;
00426 
00438   scoped_ptr<AbstractMutex> mutex_;
00439   StringStarVector text_queue_;
00440   bool network_flush_outstanding_;
00441   QueuedWorkerPool::Sequence* sequence_;
00442 
00445   bool done_outstanding_;
00446 
00449   bool finishing_;
00450 
00453   bool done_result_;
00454 
00458   bool waiting_for_flush_to_finish_;
00459 
00462   QueuedAlarm* idle_alarm_;
00463 
00464   ProxyFetchFactory* factory_;
00465 
00467   bool prepare_success_;
00468 
00469   DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
00470 };
00471 
00472 }  
00473 
00474 #endif  ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines