Page Speed Optimization Libraries  1.7.30.3
net/instaweb/automatic/public/proxy_fetch.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00022 
00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00025 
00026 #include <map>
00027 #include <set>
00028 #include <vector>
00029 
00030 #include "net/instaweb/automatic/public/html_detector.h"
00031 #include "net/instaweb/http/public/async_fetch.h"
00032 #include "net/instaweb/http/public/meta_data.h"
00033 #include "net/instaweb/http/public/request_context.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00035 #include "net/instaweb/util/public/queued_worker_pool.h"
00036 #include "net/instaweb/util/public/basictypes.h"
00037 #include "net/instaweb/util/public/fallback_property_page.h"
00038 #include "net/instaweb/util/public/gtest_prod.h"
00039 #include "net/instaweb/util/public/property_cache.h"
00040 #include "net/instaweb/util/public/scoped_ptr.h"
00041 #include "net/instaweb/util/public/string.h"
00042 #include "net/instaweb/util/public/string_util.h"
00043 
00044 namespace net_instaweb {
00045 
00046 class AbstractMutex;
00047 class CacheUrlAsyncFetcher;
00048 class Function;
00049 class GoogleUrl;
00050 class MessageHandler;
00051 class ProxyFetch;
00052 class ProxyFetchPropertyCallbackCollector;
00053 class QueuedAlarm;
00054 class ServerContext;
00055 class ResponseHeaders;
00056 class RewriteDriver;
00057 class RewriteOptions;
00058 class Timer;
00059 
00062 class ProxyFetchFactory {
00063  public:
00064   explicit ProxyFetchFactory(ServerContext* server_context);
00065   ~ProxyFetchFactory();
00066 
00069   void StartNewProxyFetch(
00070       const GoogleString& url,
00071       AsyncFetch* async_fetch,
00072       RewriteDriver* driver,
00073       ProxyFetchPropertyCallbackCollector* property_callback,
00074       AsyncFetch* original_content_fetch);
00075 
00085   ProxyFetch* CreateNewProxyFetch(
00086       const GoogleString& url,
00087       AsyncFetch* async_fetch,
00088       RewriteDriver* driver,
00089       ProxyFetchPropertyCallbackCollector* property_callback,
00090       AsyncFetch* original_content_fetch);
00091 
00094   static ProxyFetchPropertyCallbackCollector* InitiatePropertyCacheLookup(
00095       bool is_resource_fetch,
00096       const GoogleUrl& request_url,
00097       ServerContext* server_context,
00098       RewriteOptions* options,
00099       AsyncFetch* async_fetch,
00100       const bool requires_blink_cohort,
00101       bool* added_page_property_callback);
00102 
00103   MessageHandler* message_handler() const { return handler_; }
00104 
00105  private:
00106   friend class ProxyFetch;
00107 
00112   void RegisterNewFetch(ProxyFetch* proxy_fetch);
00113   void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
00114 
00115   ServerContext* server_context_;
00116   Timer* timer_;
00117   MessageHandler* handler_;
00118 
00119   scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
00120   std::set<ProxyFetch*> outstanding_proxy_fetches_;
00121 
00122   DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
00123 };
00124 
00133 class ProxyFetchPropertyCallback : public PropertyPage {
00134  public:
00135   ProxyFetchPropertyCallback(PageType page_type,
00136                              PropertyCache* property_cache,
00137                              const StringPiece& url,
00138                              const StringPiece& options_signature_hash,
00139                              UserAgentMatcher::DeviceType device_type,
00140                              ProxyFetchPropertyCallbackCollector* collector,
00141                              AbstractMutex* mutex);
00142 
00143   PageType page_type() const { return page_type_; }
00144 
00146   virtual bool IsCacheValid(int64 write_timestamp_ms) const;
00147 
00148   virtual void Done(bool success);
00149 
00150  private:
00151   PageType page_type_;
00152   UserAgentMatcher::DeviceType device_type_;
00153   ProxyFetchPropertyCallbackCollector* collector_;
00154   GoogleString url_;
00155   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
00156 };
00157 
00175 class ProxyFetchPropertyCallbackCollector {
00176  public:
00177   ProxyFetchPropertyCallbackCollector(ServerContext* server_context,
00178                                       const StringPiece& url,
00179                                       const RequestContextPtr& req_ctx,
00180                                       const RewriteOptions* options,
00181                                       UserAgentMatcher::DeviceType device_type);
00182   virtual ~ProxyFetchPropertyCallbackCollector();
00183 
00186   void AddCallback(ProxyFetchPropertyCallback* callback);
00187 
00194   void ConnectProxyFetch(ProxyFetch* proxy_fetch);
00195 
00202   void Detach(HttpStatus::Code status_code);
00203 
00205   PropertyPage* property_page() {
00206     return fallback_property_page_ == NULL ?
00207         NULL : fallback_property_page_->actual_property_page();
00208   }
00209 
00211   FallbackPropertyPage* fallback_property_page() {
00212     return fallback_property_page_.get();
00213   }
00214 
00217   PropertyPage* ReleasePropertyPage(
00218       ProxyFetchPropertyCallback::PageType page_type);
00219 
00221   FallbackPropertyPage* ReleaseFallbackPropertyPage() {
00222     return fallback_property_page_.release();
00223   }
00224 
00234   void AddPostLookupTask(Function* func);
00235 
00239   bool IsCacheValid(int64 write_timestamp_ms) const;
00240 
00242   void Done(ProxyFetchPropertyCallback* callback);
00243 
00244   const RequestContextPtr& request_context() { return request_context_; }
00245 
00247   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00248 
00249  private:
00250   friend class ProxyFetchPropertyCallbackCollectorTest;
00251   void ExecuteDone(ProxyFetchPropertyCallback* callback);
00252   void ExecuteAddPostLookupTask(Function* func);
00253   void ExecuteConnectProxyFetch(ProxyFetch* proxy_fetch);
00254   void ExecuteDetach(HttpStatus::Code status_code);
00255 
00257   void UpdateStatusCodeInPropertyCache();
00258 
00259   std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
00260   std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*>
00261   property_pages_;
00262   scoped_ptr<AbstractMutex> mutex_;
00263   ServerContext* server_context_;
00264   QueuedWorkerPool::Sequence* sequence_;
00265   GoogleString url_;
00266   RequestContextPtr request_context_;
00267   UserAgentMatcher::DeviceType device_type_;
00268   bool is_options_valid_; 
00269   bool detached_; 
00270   bool done_; 
00271   ProxyFetch* proxy_fetch_; 
00272 
00273   std::vector<Function*> post_lookup_task_vector_;
00274   const RewriteOptions* options_; 
00275   HttpStatus::Code status_code_; 
00276   scoped_ptr<FallbackPropertyPage> fallback_property_page_;
00277 
00278   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
00279 };
00280 
00301 class ProxyFetch : public SharedAsyncFetch {
00302  public:
00305   static const char kCollectorConnectProxyFetchFinish[];
00306   static const char kCollectorDetachFinish[];
00307   static const char kCollectorDoneFinish[];
00308   static const char kCollectorFinish[];
00309   static const char kCollectorDetachStart[];
00310 
00313   static const char kHeadersSetupRaceAlarmQueued[];
00314   static const char kHeadersSetupRaceDone[];
00315   static const char kHeadersSetupRaceFlush[];
00316   static const char kHeadersSetupRacePrefix[];
00317   static const char kHeadersSetupRaceWait[];
00318 
00325   static const int kTestSignalTimeoutMs = 200;
00326 
00327  protected:
00329   virtual void HandleHeadersComplete();
00330   virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
00331   virtual bool HandleFlush(MessageHandler* handler);
00332   virtual void HandleDone(bool success);
00333   virtual bool IsCachedResultValid(const ResponseHeaders& headers);
00334 
00335  private:
00336   friend class ProxyFetchFactory;
00337   friend class ProxyFetchPropertyCallbackCollector;
00338   friend class MockProxyFetch;
00339   FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
00340 
00343   virtual void PropertyCacheComplete(
00344       ProxyFetchPropertyCallbackCollector* collector);
00345 
00348   ProxyFetch(const GoogleString& url,
00349              bool cross_domain,
00350              ProxyFetchPropertyCallbackCollector* property_cache_callback,
00351              AsyncFetch* async_fetch,
00352              AsyncFetch* original_content_fetch,
00353              RewriteDriver* driver,
00354              ServerContext* server_context,
00355              Timer* timer,
00356              ProxyFetchFactory* factory);
00357   virtual ~ProxyFetch();
00358 
00359   const RewriteOptions* Options();
00360 
00362   void SetupForHtml();
00363 
00365   void AddPagespeedHeader();
00366 
00369   bool StartParse();
00370 
00372   void StartFetch();
00373 
00377   void DoFetch(bool prepare_success);
00378 
00381   void ExecuteQueued();
00382 
00385   void ScheduleQueueExecutionIfNeeded();
00386 
00391   void Finish(bool success);
00392 
00394   void CompleteFinishParse(bool success);
00395 
00398   void FlushDone();
00399 
00402 
00404   void CancelIdleAlarm();
00405 
00407   void QueueIdleAlarm();
00408 
00410   void HandleIdleAlarm();
00411 
00412   GoogleString url_;
00413   ServerContext* server_context_;
00414   Timer* timer_;
00415 
00416   scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
00417 
00420   bool cross_domain_;
00421 
00423   bool claims_html_;
00424 
00427   bool started_parse_;
00428 
00430   bool parse_text_called_;
00431 
00433   bool done_called_;
00434 
00435   HtmlDetector html_detector_;
00436 
00441   ProxyFetchPropertyCallbackCollector* property_cache_callback_;
00442 
00446   AsyncFetch* original_content_fetch_;
00447 
00450   RewriteDriver* driver_;
00451 
00454   bool queue_run_job_created_;
00455 
00467   scoped_ptr<AbstractMutex> mutex_;
00468   StringStarVector text_queue_;
00469   bool network_flush_outstanding_;
00470   QueuedWorkerPool::Sequence* sequence_;
00471 
00474   bool done_outstanding_;
00475 
00478   bool finishing_;
00479 
00482   bool done_result_;
00483 
00487   bool waiting_for_flush_to_finish_;
00488 
00491   QueuedAlarm* idle_alarm_;
00492 
00493   ProxyFetchFactory* factory_;
00494 
00496   bool distributed_fetch_;
00497 
00498   DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
00499 };
00500 
00501 }  
00502 
00503 #endif  ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines