Page Speed Optimization Libraries  1.6.29.3
net/instaweb/automatic/public/proxy_fetch.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00022 
00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
00025 
00026 #include <map>
00027 #include <set>
00028 #include <vector>
00029 
00030 #include "net/instaweb/automatic/public/html_detector.h"
00031 #include "net/instaweb/http/public/async_fetch.h"
00032 #include "net/instaweb/http/public/meta_data.h"
00033 #include "net/instaweb/http/public/request_context.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00035 #include "net/instaweb/util/public/queued_worker_pool.h"
00036 #include "net/instaweb/util/public/basictypes.h"
00037 #include "net/instaweb/util/public/fallback_property_page.h"
00038 #include "net/instaweb/util/public/gtest_prod.h"
00039 #include "net/instaweb/util/public/property_cache.h"
00040 #include "net/instaweb/util/public/scoped_ptr.h"
00041 #include "net/instaweb/util/public/string.h"
00042 #include "net/instaweb/util/public/string_util.h"
00043 
00044 namespace net_instaweb {
00045 
00046 class AbstractMutex;
00047 class CacheUrlAsyncFetcher;
00048 class Function;
00049 class GoogleUrl;
00050 class MessageHandler;
00051 class ProxyFetch;
00052 class ProxyFetchPropertyCallbackCollector;
00053 class QueuedAlarm;
00054 class ServerContext;
00055 class ResponseHeaders;
00056 class RewriteDriver;
00057 class RewriteOptions;
00058 class Timer;
00059 
00062 class ProxyFetchFactory {
00063  public:
00064   explicit ProxyFetchFactory(ServerContext* server_context);
00065   ~ProxyFetchFactory();
00066 
00069   void StartNewProxyFetch(
00070       const GoogleString& url,
00071       AsyncFetch* async_fetch,
00072       RewriteDriver* driver,
00073       ProxyFetchPropertyCallbackCollector* property_callback,
00074       AsyncFetch* original_content_fetch);
00075 
00085   ProxyFetch* CreateNewProxyFetch(
00086       const GoogleString& url,
00087       AsyncFetch* async_fetch,
00088       RewriteDriver* driver,
00089       ProxyFetchPropertyCallbackCollector* property_callback,
00090       AsyncFetch* original_content_fetch);
00091 
00094   static ProxyFetchPropertyCallbackCollector* InitiatePropertyCacheLookup(
00095       bool is_resource_fetch,
00096       const GoogleUrl& request_url,
00097       ServerContext* server_context,
00098       RewriteOptions* options,
00099       AsyncFetch* async_fetch,
00100       const bool requires_blink_cohort,
00101       bool* added_page_property_callback);
00102 
00103   MessageHandler* message_handler() const { return handler_; }
00104 
00105  private:
00106   friend class ProxyFetch;
00107 
00112   void RegisterNewFetch(ProxyFetch* proxy_fetch);
00113   void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
00114 
00115   ServerContext* server_context_;
00116   Timer* timer_;
00117   MessageHandler* handler_;
00118 
00119   scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
00120   std::set<ProxyFetch*> outstanding_proxy_fetches_;
00121 
00122   DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
00123 };
00124 
00133 class ProxyFetchPropertyCallback : public PropertyPage {
00134  public:
00135   ProxyFetchPropertyCallback(PageType page_type,
00136                              PropertyCache* property_cache,
00137                              const StringPiece& key,
00138                              UserAgentMatcher::DeviceType device_type,
00139                              ProxyFetchPropertyCallbackCollector* collector,
00140                              AbstractMutex* mutex);
00141 
00142   PageType page_type() const { return page_type_; }
00143 
00144   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00145 
00147   virtual bool IsCacheValid(int64 write_timestamp_ms) const;
00148 
00149   virtual void Done(bool success);
00150 
00151  private:
00152   PageType page_type_;
00153   UserAgentMatcher::DeviceType device_type_;
00154   ProxyFetchPropertyCallbackCollector* collector_;
00155   GoogleString url_;
00156   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
00157 };
00158 
00160 class ProxyFetchPropertyCallbackCollector {
00161  public:
00162   ProxyFetchPropertyCallbackCollector(ServerContext* server_context,
00163                                       const StringPiece& url,
00164                                       const RequestContextPtr& req_ctx,
00165                                       const RewriteOptions* options,
00166                                       UserAgentMatcher::DeviceType device_type);
00167   virtual ~ProxyFetchPropertyCallbackCollector();
00168 
00171   void AddCallback(ProxyFetchPropertyCallback* callback);
00172 
00179   void ConnectProxyFetch(ProxyFetch* proxy_fetch);
00180 
00187   void Detach(HttpStatus::Code status_code);
00188 
00190   PropertyPage* property_page() {
00191     return fallback_property_page_ == NULL ?
00192         NULL : fallback_property_page_->actual_property_page();
00193   }
00194 
00196   FallbackPropertyPage* fallback_property_page() {
00197     return fallback_property_page_.get();
00198   }
00199 
00202   PropertyPage* ReleasePropertyPage(
00203       ProxyFetchPropertyCallback::PageType page_type);
00204 
00206   FallbackPropertyPage* ReleaseFallbackPropertyPage() {
00207     return fallback_property_page_.release();
00208   }
00209 
00219   void AddPostLookupTask(Function* func);
00220 
00224   bool IsCacheValid(int64 write_timestamp_ms) const;
00225 
00227   void Done(ProxyFetchPropertyCallback* callback);
00228 
00230   void UpdateStatusCodeInPropertyCache();
00231 
00232   const RequestContextPtr& request_context() { return request_context_; }
00233 
00235   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00236 
00237  private:
00238   std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
00239   std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*>
00240   property_pages_;
00241   scoped_ptr<AbstractMutex> mutex_;
00242   ServerContext* server_context_;
00243   GoogleString url_;
00244   RequestContextPtr request_context_;
00245   UserAgentMatcher::DeviceType device_type_;
00246   bool detached_; 
00247   bool done_; 
00248   ProxyFetch* proxy_fetch_; 
00249 
00250   scoped_ptr<std::vector<Function*> > post_lookup_task_vector_;
00251   const RewriteOptions* options_; 
00252   HttpStatus::Code status_code_; 
00253   scoped_ptr<FallbackPropertyPage> fallback_property_page_;
00254 
00255   DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
00256 };
00257 
00278 class ProxyFetch : public SharedAsyncFetch {
00279  public:
00282   static const char kCollectorDone[];
00283   static const char kCollectorPrefix[];
00284   static const char kCollectorReady[];
00285   static const char kCollectorDelete[];
00286   static const char kCollectorDetach[];
00287   static const char kCollectorDoneDelete[];
00288 
00291   static const char kHeadersSetupRaceAlarmQueued[];
00292   static const char kHeadersSetupRaceDone[];
00293   static const char kHeadersSetupRaceFlush[];
00294   static const char kHeadersSetupRacePrefix[];
00295   static const char kHeadersSetupRaceWait[];
00296 
00303   static const int kTestSignalTimeoutMs = 200;
00304 
00305  protected:
00307   virtual void HandleHeadersComplete();
00308   virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
00309   virtual bool HandleFlush(MessageHandler* handler);
00310   virtual void HandleDone(bool success);
00311   virtual bool IsCachedResultValid(const ResponseHeaders& headers);
00312 
00313  private:
00314   friend class ProxyFetchFactory;
00315   friend class ProxyFetchPropertyCallbackCollector;
00316   friend class MockProxyFetch;
00317   FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
00318 
00321   virtual void PropertyCacheComplete(
00322       ProxyFetchPropertyCallbackCollector* collector);
00323 
00326   ProxyFetch(const GoogleString& url,
00327              bool cross_domain,
00328              ProxyFetchPropertyCallbackCollector* property_cache_callback,
00329              AsyncFetch* async_fetch,
00330              AsyncFetch* original_content_fetch,
00331              RewriteDriver* driver,
00332              ServerContext* server_context,
00333              Timer* timer,
00334              ProxyFetchFactory* factory);
00335   virtual ~ProxyFetch();
00336 
00337   const RewriteOptions* Options();
00338 
00340   void SetupForHtml();
00341 
00343   void AddPagespeedHeader();
00344 
00347   bool StartParse();
00348 
00350   void StartFetch();
00351 
00355   void DoFetch(bool prepare_success);
00356 
00359   void ExecuteQueued();
00360 
00363   void ScheduleQueueExecutionIfNeeded();
00364 
00369   void Finish(bool success);
00370 
00372   void CompleteFinishParse(bool success);
00373 
00376   void FlushDone();
00377 
00380 
00382   void CancelIdleAlarm();
00383 
00385   void QueueIdleAlarm();
00386 
00388   void HandleIdleAlarm();
00389 
00390   GoogleString url_;
00391   ServerContext* server_context_;
00392   Timer* timer_;
00393 
00394   scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
00395 
00398   bool cross_domain_;
00399 
00401   bool claims_html_;
00402 
00405   bool started_parse_;
00406 
00408   bool parse_text_called_;
00409 
00411   bool done_called_;
00412 
00413   HtmlDetector html_detector_;
00414 
00419   ProxyFetchPropertyCallbackCollector* property_cache_callback_;
00420 
00424   AsyncFetch* original_content_fetch_;
00425 
00428   RewriteDriver* driver_;
00429 
00432   bool queue_run_job_created_;
00433 
00445   scoped_ptr<AbstractMutex> mutex_;
00446   StringStarVector text_queue_;
00447   bool network_flush_outstanding_;
00448   QueuedWorkerPool::Sequence* sequence_;
00449 
00452   bool done_outstanding_;
00453 
00456   bool finishing_;
00457 
00460   bool done_result_;
00461 
00465   bool waiting_for_flush_to_finish_;
00466 
00469   QueuedAlarm* idle_alarm_;
00470 
00471   ProxyFetchFactory* factory_;
00472 
00474   bool distributed_fetch_;
00475 
00476   DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
00477 };
00478 
00479 }  
00480 
00481 #endif  ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines