Page Speed Optimization Libraries  1.5.27.2
net/instaweb/automatic/public/proxy_fetch.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00022 
00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00025 
00026 #include <map>
00027 #include <set>
00028 #include <vector>
00029 
00030 #include "net/instaweb/automatic/public/html_detector.h"
00031 #include "net/instaweb/http/public/async_fetch.h"
00032 #include "net/instaweb/http/public/meta_data.h"
00033 #include "net/instaweb/http/public/request_context.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00035 #include "net/instaweb/util/public/queued_worker_pool.h"
00036 #include "net/instaweb/util/public/basictypes.h"
00037 #include "net/instaweb/util/public/fallback_property_page.h"
00038 #include "net/instaweb/util/public/gtest_prod.h"
00039 #include "net/instaweb/util/public/property_cache.h"
00040 #include "net/instaweb/util/public/scoped_ptr.h"
00041 #include "net/instaweb/util/public/string.h"
00042 #include "net/instaweb/util/public/string_util.h"
00043 
00044 namespace net_instaweb {
00045 
00046 class AbstractClientState;
00047 class AbstractLogRecord;
00048 class AbstractMutex;
00049 class CacheUrlAsyncFetcher;
00050 class Function;
00051 class MessageHandler;
00052 class ProxyFetch;
00053 class ProxyFetchPropertyCallbackCollector;
00054 class QueuedAlarm;
00055 class ServerContext;
00056 class ResponseHeaders;
00057 class RewriteDriver;
00058 class RewriteOptions;
00059 class Timer;
00060 
00063 class ProxyFetchFactory {
00064  public:
00065   explicit ProxyFetchFactory(ServerContext* server_context);
00066   ~ProxyFetchFactory();
00067 
00070   void StartNewProxyFetch(
00071       const GoogleString& url,
00072       AsyncFetch* async_fetch,
00073       RewriteDriver* driver,
00074       ProxyFetchPropertyCallbackCollector* property_callback,
00075       AsyncFetch* original_content_fetch);
00076 
00086   ProxyFetch* CreateNewProxyFetch(
00087       const GoogleString& url,
00088       AsyncFetch* async_fetch,
00089       RewriteDriver* driver,
00090       ProxyFetchPropertyCallbackCollector* property_callback,
00091       AsyncFetch* original_content_fetch);
00092 
00093   MessageHandler* message_handler() const { return handler_; }
00094 
00095  private:
00096   friend class ProxyFetch;
00097 
00102   void RegisterNewFetch(ProxyFetch* proxy_fetch);
00103   void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
00104 
00105   ServerContext* server_context_;
00106   Timer* timer_;
00107   MessageHandler* handler_;
00108 
00109   scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
00110   std::set<ProxyFetch*> outstanding_proxy_fetches_;
00111 
00112   DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
00113 };
00114 
00123 class ProxyFetchPropertyCallback : public PropertyPage {
00124  public:
00126   enum PageType {
00127     kPropertyCachePage,
00128     kPropertyCacheFallbackPage,
00129     kClientPropertyCachePage,
00130     kDevicePropertyCachePage,
00131   };
00132 
00133   ProxyFetchPropertyCallback(PageType page_type,
00134                              PropertyCache* property_cache,
00135                              const StringPiece& key,
00136                              UserAgentMatcher::DeviceType device_type,
00137                              ProxyFetchPropertyCallbackCollector* collector,
00138                              AbstractMutex* mutex);
00139 
00140   PageType page_type() const { return page_type_; }
00141 
00142   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00143 
00145   virtual bool IsCacheValid(int64 write_timestamp_ms) const;
00146 
00147   virtual void Done(bool success);
00148 
00150   virtual void LogPageCohortInfo(AbstractLogRecord* log_record,
00151                                  int cohort_index);
00152 
00153  private:
00154   PageType page_type_;
00155   UserAgentMatcher::DeviceType device_type_;
00156   ProxyFetchPropertyCallbackCollector* collector_;
00157   GoogleString url_;
00158   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
00159 };
00160 
00162 class ProxyFetchPropertyCallbackCollector {
00163  public:
00164   ProxyFetchPropertyCallbackCollector(ServerContext* server_context,
00165                                       const StringPiece& url,
00166                                       const RequestContextPtr& req_ctx,
00167                                       const RewriteOptions* options,
00168                                       UserAgentMatcher::DeviceType device_type);
00169   virtual ~ProxyFetchPropertyCallbackCollector();
00170 
00173   void AddCallback(ProxyFetchPropertyCallback* callback);
00174 
00181   void ConnectProxyFetch(ProxyFetch* proxy_fetch);
00182 
00189   void Detach(HttpStatus::Code status_code);
00190 
00192   PropertyPage* property_page() {
00193     return fallback_property_page_ == NULL ?
00194         NULL : fallback_property_page_->actual_property_page();
00195   }
00196 
00198   FallbackPropertyPage* fallback_property_page() {
00199     return fallback_property_page_.get();
00200   }
00201 
00204   PropertyPage* ReleasePropertyPage(
00205       ProxyFetchPropertyCallback::PageType page_type);
00206 
00208   FallbackPropertyPage* ReleaseFallbackPropertyPage() {
00209     return fallback_property_page_.release();
00210   }
00211 
00221   void AddPostLookupTask(Function* func);
00222 
00226   bool IsCacheValid(int64 write_timestamp_ms) const;
00227 
00229   void Done(ProxyFetchPropertyCallback* callback);
00230 
00232   void UpdateStatusCodeInPropertyCache();
00233 
00234   const RequestContextPtr& request_context() { return request_context_; }
00235 
00237   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00238 
00239  private:
00240   std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
00241   std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*>
00242   property_pages_;
00243   scoped_ptr<AbstractMutex> mutex_;
00244   ServerContext* server_context_;
00245   GoogleString url_;
00246   RequestContextPtr request_context_;
00247   UserAgentMatcher::DeviceType device_type_;
00248   bool detached_; 
00249   bool done_; 
00250   ProxyFetch* proxy_fetch_; 
00251 
00252   scoped_ptr<std::vector<Function*> > post_lookup_task_vector_;
00253   const RewriteOptions* options_; 
00254   HttpStatus::Code status_code_; 
00255   scoped_ptr<FallbackPropertyPage> fallback_property_page_;
00256 
00257   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
00258 };
00259 
00280 class ProxyFetch : public SharedAsyncFetch {
00281  public:
00284   static const char kCollectorDone[];
00285   static const char kCollectorPrefix[];
00286   static const char kCollectorReady[];
00287   static const char kCollectorDelete[];
00288   static const char kCollectorDetach[];
00289   static const char kCollectorDoneDelete[];
00290 
00293   static const char kHeadersSetupRaceAlarmQueued[];
00294   static const char kHeadersSetupRaceDone[];
00295   static const char kHeadersSetupRaceFlush[];
00296   static const char kHeadersSetupRacePrefix[];
00297   static const char kHeadersSetupRaceWait[];
00298 
00305   static const int kTestSignalTimeoutMs = 200;
00306 
00307  protected:
00309   virtual void HandleHeadersComplete();
00310   virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
00311   virtual bool HandleFlush(MessageHandler* handler);
00312   virtual void HandleDone(bool success);
00313   virtual bool IsCachedResultValid(const ResponseHeaders& headers);
00314 
00315  private:
00316   friend class ProxyFetchFactory;
00317   friend class ProxyFetchPropertyCallbackCollector;
00318   friend class MockProxyFetch;
00319   FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
00320 
00323   virtual void PropertyCacheComplete(
00324       ProxyFetchPropertyCallbackCollector* collector);
00325 
00330   AbstractClientState* GetClientState(
00331       ProxyFetchPropertyCallbackCollector* collector);
00332 
00335   ProxyFetch(const GoogleString& url,
00336              bool cross_domain,
00337              ProxyFetchPropertyCallbackCollector* property_cache_callback,
00338              AsyncFetch* async_fetch,
00339              AsyncFetch* original_content_fetch,
00340              RewriteDriver* driver,
00341              ServerContext* server_context,
00342              Timer* timer,
00343              ProxyFetchFactory* factory);
00344   virtual ~ProxyFetch();
00345 
00346   const RewriteOptions* Options();
00347 
00349   void SetupForHtml();
00350 
00352   void AddPagespeedHeader();
00353 
00356   bool StartParse();
00357 
00359   void StartFetch();
00360 
00362   void DoFetch();
00363 
00366   void ExecuteQueued();
00367 
00370   void ScheduleQueueExecutionIfNeeded();
00371 
00376   void Finish(bool success);
00377 
00379   void CompleteFinishParse(bool success);
00380 
00383   void FlushDone();
00384 
00387 
00389   void CancelIdleAlarm();
00390 
00392   void QueueIdleAlarm();
00393 
00395   void HandleIdleAlarm();
00396 
00397   GoogleString url_;
00398   ServerContext* server_context_;
00399   Timer* timer_;
00400 
00401   scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
00402 
00405   bool cross_domain_;
00406 
00408   bool claims_html_;
00409 
00412   bool started_parse_;
00413 
00415   bool parse_text_called_;
00416 
00418   bool done_called_;
00419 
00420   HtmlDetector html_detector_;
00421 
00426   ProxyFetchPropertyCallbackCollector* property_cache_callback_;
00427 
00431   AsyncFetch* original_content_fetch_;
00432 
00435   RewriteDriver* driver_;
00436 
00439   bool queue_run_job_created_;
00440 
00452   scoped_ptr<AbstractMutex> mutex_;
00453   StringStarVector text_queue_;
00454   bool network_flush_outstanding_;
00455   QueuedWorkerPool::Sequence* sequence_;
00456 
00459   bool done_outstanding_;
00460 
00463   bool finishing_;
00464 
00467   bool done_result_;
00468 
00472   bool waiting_for_flush_to_finish_;
00473 
00476   QueuedAlarm* idle_alarm_;
00477 
00478   ProxyFetchFactory* factory_;
00479 
00481   bool prepare_success_;
00482 
00483   DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
00484 };
00485 
00486 }  
00487 
00488 #endif  ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines