Page Speed Optimization Libraries  1.3.25.1
net/instaweb/automatic/public/proxy_fetch.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00022 
00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00025 
00026 #include <map>
00027 #include <set>
00028 #include <vector>
00029 
00030 #include "net/instaweb/automatic/public/html_detector.h"
00031 #include "net/instaweb/http/public/async_fetch.h"
00032 #include "net/instaweb/http/public/meta_data.h"
00033 #include "net/instaweb/http/public/request_context.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00035 #include "net/instaweb/util/public/queued_worker_pool.h"
00036 #include "net/instaweb/util/public/basictypes.h"
00037 #include "net/instaweb/util/public/gtest_prod.h"
00038 #include "net/instaweb/util/public/property_cache.h"
00039 #include "net/instaweb/util/public/scoped_ptr.h"
00040 #include "net/instaweb/util/public/string.h"
00041 #include "net/instaweb/util/public/string_util.h"
00042 
00043 namespace net_instaweb {
00044 
00045 class AbstractClientState;
00046 class AbstractMutex;
00047 class CacheUrlAsyncFetcher;
00048 class Function;
00049 class LogRecord;
00050 class MessageHandler;
00051 class ProxyFetch;
00052 class ProxyFetchPropertyCallbackCollector;
00053 class QueuedAlarm;
00054 class ServerContext;
00055 class ResponseHeaders;
00056 class RewriteDriver;
00057 class RewriteOptions;
00058 class Timer;
00059 
00062 class ProxyFetchFactory {
00063  public:
00064   explicit ProxyFetchFactory(ServerContext* manager);
00065   ~ProxyFetchFactory();
00066 
00069   void StartNewProxyFetch(
00070       const GoogleString& url,
00071       AsyncFetch* async_fetch,
00072       RewriteDriver* driver,
00073       ProxyFetchPropertyCallbackCollector* property_callback,
00074       AsyncFetch* original_content_fetch);
00075 
00085   ProxyFetch* CreateNewProxyFetch(
00086       const GoogleString& url,
00087       AsyncFetch* async_fetch,
00088       RewriteDriver* driver,
00089       ProxyFetchPropertyCallbackCollector* property_callback,
00090       AsyncFetch* original_content_fetch);
00091 
00092   MessageHandler* message_handler() const { return handler_; }
00093 
00094  private:
00095   friend class ProxyFetch;
00096 
00101   void RegisterNewFetch(ProxyFetch* proxy_fetch);
00102   void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
00103 
00104   ServerContext* manager_;
00105   Timer* timer_;
00106   MessageHandler* handler_;
00107 
00108   scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
00109   std::set<ProxyFetch*> outstanding_proxy_fetches_;
00110 
00111   DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
00112 };
00113 
00122 class ProxyFetchPropertyCallback : public PropertyPage {
00123  public:
00125   enum CacheType {
00126     kPagePropertyCache,
00127     kClientPropertyCache,
00128     kDevicePropertyCache
00129   };
00130 
00131   ProxyFetchPropertyCallback(CacheType cache_type,
00132                              const PropertyCache& property_cache,
00133                              const StringPiece& key,
00134                              UserAgentMatcher::DeviceType device_type,
00135                              ProxyFetchPropertyCallbackCollector* collector,
00136                              AbstractMutex* mutex);
00137 
00138   CacheType cache_type() const { return cache_type_; }
00139 
00140   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00141 
00143   virtual bool IsCacheValid(int64 write_timestamp_ms) const;
00144 
00145   virtual void Done(bool success);
00146 
00148   virtual void LogPageCohortInfo(LogRecord* log_record, int cohort_index);
00149 
00150  private:
00151   CacheType cache_type_;
00152   UserAgentMatcher::DeviceType device_type_;
00153   ProxyFetchPropertyCallbackCollector* collector_;
00154   GoogleString url_;
00155   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
00156 };
00157 
00159 class ProxyFetchPropertyCallbackCollector {
00160  public:
00161   ProxyFetchPropertyCallbackCollector(ServerContext* manager,
00162                                       const StringPiece& url,
00163                                       const RequestContextPtr& req_ctx,
00164                                       const RewriteOptions* options,
00165                                       const StringPiece& user_agent);
00166   virtual ~ProxyFetchPropertyCallbackCollector();
00167 
00170   void AddCallback(ProxyFetchPropertyCallback* callback);
00171 
00178   void ConnectProxyFetch(ProxyFetch* proxy_fetch);
00179 
00186   void Detach(HttpStatus::Code status_code);
00187 
00190   PropertyPage* GetPropertyPage(
00191       ProxyFetchPropertyCallback::CacheType cache_type);
00192 
00195   PropertyPage* GetPropertyPageWithoutOwnership(
00196       ProxyFetchPropertyCallback::CacheType cache_type);
00197 
00207   void AddPostLookupTask(Function* func);
00208 
00212   bool IsCacheValid(int64 write_timestamp_ms) const;
00213 
00215   void Done(ProxyFetchPropertyCallback* callback, bool success);
00216 
00218   void UpdateStatusCodeInPropertyCache();
00219 
00220   const RequestContextPtr& request_context() { return request_context_; }
00221 
00223   UserAgentMatcher::DeviceType GetDeviceTypeFromDeviceCacheMutexHeld();
00224 
00225  private:
00227   void SetPropertyPageForDeviceTypeMutexHeld(
00228       UserAgentMatcher::DeviceType device_type);
00229 
00230   std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
00231   std::map<ProxyFetchPropertyCallback::CacheType, PropertyPage*>
00232   property_pages_;
00233   std::map<UserAgentMatcher::DeviceType, PropertyPage*>
00234   property_pages_for_device_types_;
00235   scoped_ptr<AbstractMutex> mutex_;
00236   ServerContext* server_context_;
00237   GoogleString url_;
00238   RequestContextPtr request_context_;
00239   GoogleString user_agent_;
00240   bool detached_; 
00241   bool done_; 
00242   bool success_; 
00243   ProxyFetch* proxy_fetch_; 
00244 
00245   scoped_ptr<std::vector<Function*> > post_lookup_task_vector_;
00246   const RewriteOptions* options_; 
00247   HttpStatus::Code status_code_; 
00248 
00249   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
00250 };
00251 
00272 class ProxyFetch : public SharedAsyncFetch {
00273  public:
00276   static const char kCollectorDone[];
00277   static const char kCollectorPrefix[];
00278   static const char kCollectorReady[];
00279   static const char kCollectorDelete[];
00280   static const char kCollectorDetach[];
00281   static const char kCollectorDoneDelete[];
00282 
00285   static const char kHeadersSetupRaceAlarmQueued[];
00286   static const char kHeadersSetupRaceDone[];
00287   static const char kHeadersSetupRaceFlush[];
00288   static const char kHeadersSetupRacePrefix[];
00289   static const char kHeadersSetupRaceWait[];
00290 
00297   static const int kTestSignalTimeoutMs = 200;
00298 
00299  protected:
00301   virtual void HandleHeadersComplete();
00302   virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
00303   virtual bool HandleFlush(MessageHandler* handler);
00304   virtual void HandleDone(bool success);
00305   virtual bool IsCachedResultValid(const ResponseHeaders& headers);
00306 
00307  private:
00308   friend class ProxyFetchFactory;
00309   friend class ProxyFetchPropertyCallbackCollector;
00310   friend class MockProxyFetch;
00311   FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
00312 
00315   virtual void PropertyCacheComplete(
00316       bool success, ProxyFetchPropertyCallbackCollector* collector);
00317 
00322   AbstractClientState* GetClientState(
00323       ProxyFetchPropertyCallbackCollector* collector);
00324 
00327   ProxyFetch(const GoogleString& url,
00328              bool cross_domain,
00329              ProxyFetchPropertyCallbackCollector* property_cache_callback,
00330              AsyncFetch* async_fetch,
00331              AsyncFetch* original_content_fetch,
00332              RewriteDriver* driver,
00333              ServerContext* manager,
00334              Timer* timer,
00335              ProxyFetchFactory* factory);
00336   virtual ~ProxyFetch();
00337 
00338   const RewriteOptions* Options();
00339 
00341   void SetupForHtml();
00342 
00344   void AddPagespeedHeader();
00345 
00348   bool StartParse();
00349 
00351   void StartFetch();
00352 
00354   void DoFetch();
00355 
00358   void ExecuteQueued();
00359 
00362   void ScheduleQueueExecutionIfNeeded();
00363 
00368   void Finish(bool success);
00369 
00371   void CompleteFinishParse(bool success);
00372 
00375   void FlushDone();
00376 
00379 
00381   void CancelIdleAlarm();
00382 
00384   void QueueIdleAlarm();
00385 
00387   void HandleIdleAlarm();
00388 
00389   GoogleString url_;
00390   ServerContext* server_context_;
00391   Timer* timer_;
00392 
00393   scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
00394 
00397   bool cross_domain_;
00398 
00400   bool claims_html_;
00401 
00404   bool started_parse_;
00405 
00407   bool parse_text_called_;
00408 
00410   bool done_called_;
00411 
00412   HtmlDetector html_detector_;
00413 
00418   ProxyFetchPropertyCallbackCollector* property_cache_callback_;
00419 
00423   AsyncFetch* original_content_fetch_;
00424 
00427   RewriteDriver* driver_;
00428 
00431   bool queue_run_job_created_;
00432 
00444   scoped_ptr<AbstractMutex> mutex_;
00445   StringStarVector text_queue_;
00446   bool network_flush_outstanding_;
00447   QueuedWorkerPool::Sequence* sequence_;
00448 
00451   bool done_outstanding_;
00452 
00455   bool finishing_;
00456 
00459   bool done_result_;
00460 
00464   bool waiting_for_flush_to_finish_;
00465 
00468   QueuedAlarm* idle_alarm_;
00469 
00470   ProxyFetchFactory* factory_;
00471 
00473   bool prepare_success_;
00474 
00475   DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
00476 };
00477 
00478 }  
00479 
00480 #endif  ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines