Page Speed Optimization Libraries
1.8.31.3
|
00001 /* 00002 * Copyright 2011 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00022 00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_ 00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_ 00025 00026 #include <map> 00027 #include <set> 00028 #include <vector> 00029 00030 #include "net/instaweb/automatic/public/html_detector.h" 00031 #include "net/instaweb/http/public/async_fetch.h" 00032 #include "net/instaweb/http/public/meta_data.h" 00033 #include "net/instaweb/http/public/request_context.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00035 #include "net/instaweb/util/public/queued_worker_pool.h" 00036 #include "net/instaweb/util/public/basictypes.h" 00037 #include "net/instaweb/util/public/fallback_property_page.h" 00038 #include "net/instaweb/util/public/gtest_prod.h" 00039 #include "net/instaweb/util/public/property_cache.h" 00040 #include "net/instaweb/util/public/scoped_ptr.h" 00041 #include "net/instaweb/util/public/string.h" 00042 #include "net/instaweb/util/public/string_util.h" 00043 00044 namespace net_instaweb { 00045 00046 class AbstractMutex; 00047 class CacheUrlAsyncFetcher; 00048 class Function; 00049 class GoogleUrl; 00050 class MessageHandler; 00051 class ProxyFetch; 00052 class ProxyFetchPropertyCallbackCollector; 00053 class QueuedAlarm; 00054 class ServerContext; 00055 class ResponseHeaders; 00056 class RewriteDriver; 00057 class RewriteOptions; 00058 class Timer; 00059 00062 class ProxyFetchFactory { 00063 public: 00064 explicit ProxyFetchFactory(ServerContext* server_context); 00065 ~ProxyFetchFactory(); 00066 00069 void StartNewProxyFetch( 00070 const GoogleString& url, 00071 AsyncFetch* async_fetch, 00072 RewriteDriver* driver, 00073 ProxyFetchPropertyCallbackCollector* property_callback, 00074 AsyncFetch* original_content_fetch); 00075 00085 ProxyFetch* CreateNewProxyFetch( 00086 const GoogleString& url, 00087 AsyncFetch* async_fetch, 00088 RewriteDriver* driver, 00089 ProxyFetchPropertyCallbackCollector* property_callback, 00090 AsyncFetch* original_content_fetch); 00091 00094 static ProxyFetchPropertyCallbackCollector* InitiatePropertyCacheLookup( 00095 bool is_resource_fetch, 00096 const GoogleUrl& request_url, 00097 ServerContext* server_context, 00098 RewriteOptions* options, 00099 AsyncFetch* async_fetch, 00100 const bool requires_blink_cohort, 00101 bool* added_page_property_callback); 00102 00103 MessageHandler* message_handler() const { return handler_; } 00104 00105 private: 00106 friend class ProxyFetch; 00107 00112 void RegisterNewFetch(ProxyFetch* proxy_fetch); 00113 void RegisterFinishedFetch(ProxyFetch* proxy_fetch); 00114 00115 ServerContext* server_context_; 00116 Timer* timer_; 00117 MessageHandler* handler_; 00118 00119 scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_; 00120 std::set<ProxyFetch*> outstanding_proxy_fetches_; 00121 00122 DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory); 00123 }; 00124 00133 class ProxyFetchPropertyCallback : public PropertyPage { 00134 public: 00135 ProxyFetchPropertyCallback(PageType page_type, 00136 PropertyCache* property_cache, 00137 const StringPiece& url, 00138 const StringPiece& options_signature_hash, 00139 UserAgentMatcher::DeviceType device_type, 00140 ProxyFetchPropertyCallbackCollector* collector, 00141 AbstractMutex* mutex); 00142 00143 PageType page_type() const { return page_type_; } 00144 00146 virtual bool IsCacheValid(int64 write_timestamp_ms) const; 00147 00148 virtual void Done(bool success); 00149 00150 private: 00151 PageType page_type_; 00152 UserAgentMatcher::DeviceType device_type_; 00153 ProxyFetchPropertyCallbackCollector* collector_; 00154 GoogleString url_; 00155 DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback); 00156 }; 00157 00175 class ProxyFetchPropertyCallbackCollector { 00176 public: 00177 ProxyFetchPropertyCallbackCollector(ServerContext* server_context, 00178 const StringPiece& url, 00179 const RequestContextPtr& req_ctx, 00180 const RewriteOptions* options, 00181 UserAgentMatcher::DeviceType device_type); 00182 virtual ~ProxyFetchPropertyCallbackCollector(); 00183 00186 void AddCallback(ProxyFetchPropertyCallback* callback); 00187 00194 void ConnectProxyFetch(ProxyFetch* proxy_fetch); 00195 00202 void Detach(HttpStatus::Code status_code); 00203 00205 PropertyPage* property_page() { 00206 return fallback_property_page_ == NULL ? 00207 NULL : fallback_property_page_->actual_property_page(); 00208 } 00209 00211 FallbackPropertyPage* fallback_property_page() { 00212 return fallback_property_page_.get(); 00213 } 00214 00217 PropertyPage* ReleasePropertyPage( 00218 ProxyFetchPropertyCallback::PageType page_type); 00219 00221 FallbackPropertyPage* ReleaseFallbackPropertyPage() { 00222 return fallback_property_page_.release(); 00223 } 00224 00234 void AddPostLookupTask(Function* func); 00235 00239 bool IsCacheValid(int64 write_timestamp_ms) const; 00240 00242 void Done(ProxyFetchPropertyCallback* callback); 00243 00244 const RequestContextPtr& request_context() { return request_context_; } 00245 00247 UserAgentMatcher::DeviceType device_type() { return device_type_; } 00248 00249 private: 00250 friend class ProxyFetchPropertyCallbackCollectorTest; 00251 void ExecuteDone(ProxyFetchPropertyCallback* callback); 00252 void ExecuteAddPostLookupTask(Function* func); 00253 void ExecuteConnectProxyFetch(ProxyFetch* proxy_fetch); 00254 void ExecuteDetach(HttpStatus::Code status_code); 00255 00257 void UpdateStatusCodeInPropertyCache(); 00258 00259 std::set<ProxyFetchPropertyCallback*> pending_callbacks_; 00260 std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*> 00261 property_pages_; 00262 scoped_ptr<AbstractMutex> mutex_; 00263 ServerContext* server_context_; 00264 QueuedWorkerPool::Sequence* sequence_; 00265 GoogleString url_; 00266 RequestContextPtr request_context_; 00267 UserAgentMatcher::DeviceType device_type_; 00268 bool is_options_valid_; 00269 bool detached_; 00270 bool done_; 00271 ProxyFetch* proxy_fetch_; 00272 00273 std::vector<Function*> post_lookup_task_vector_; 00274 const RewriteOptions* options_; 00275 HttpStatus::Code status_code_; 00276 scoped_ptr<FallbackPropertyPage> fallback_property_page_; 00277 00278 DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector); 00279 }; 00280 00301 class ProxyFetch : public SharedAsyncFetch { 00302 public: 00305 static const char kCollectorConnectProxyFetchFinish[]; 00306 static const char kCollectorDetachFinish[]; 00307 static const char kCollectorDoneFinish[]; 00308 static const char kCollectorFinish[]; 00309 static const char kCollectorDetachStart[]; 00310 00313 static const char kHeadersSetupRaceAlarmQueued[]; 00314 static const char kHeadersSetupRaceDone[]; 00315 static const char kHeadersSetupRaceFlush[]; 00316 static const char kHeadersSetupRacePrefix[]; 00317 static const char kHeadersSetupRaceWait[]; 00318 00325 static const int kTestSignalTimeoutMs = 200; 00326 00327 protected: 00329 virtual void HandleHeadersComplete(); 00330 virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler); 00331 virtual bool HandleFlush(MessageHandler* handler); 00332 virtual void HandleDone(bool success); 00333 virtual bool IsCachedResultValid(const ResponseHeaders& headers); 00334 00335 private: 00336 friend class ProxyFetchFactory; 00337 friend class ProxyFetchPropertyCallbackCollector; 00338 friend class MockProxyFetch; 00339 FRIEND_TEST(ProxyFetchTest, TestInhibitParsing); 00340 00343 virtual void PropertyCacheComplete( 00344 ProxyFetchPropertyCallbackCollector* collector); 00345 00348 ProxyFetch(const GoogleString& url, 00349 bool cross_domain, 00350 ProxyFetchPropertyCallbackCollector* property_cache_callback, 00351 AsyncFetch* async_fetch, 00352 AsyncFetch* original_content_fetch, 00353 RewriteDriver* driver, 00354 ServerContext* server_context, 00355 Timer* timer, 00356 ProxyFetchFactory* factory); 00357 virtual ~ProxyFetch(); 00358 00359 const RewriteOptions* Options(); 00360 00362 void SetupForHtml(); 00363 00365 void AddPagespeedHeader(); 00366 00369 bool StartParse(); 00370 00372 void StartFetch(); 00373 00377 void DoFetch(bool prepare_success); 00378 00381 void ExecuteQueued(); 00382 00385 void ScheduleQueueExecutionIfNeeded(); 00386 00391 void Finish(bool success); 00392 00394 void CompleteFinishParse(bool success); 00395 00398 void FlushDone(); 00399 00402 00404 void CancelIdleAlarm(); 00405 00407 void QueueIdleAlarm(); 00408 00410 void HandleIdleAlarm(); 00411 00412 GoogleString url_; 00413 ServerContext* server_context_; 00414 Timer* timer_; 00415 00416 scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_; 00417 00420 bool cross_domain_; 00421 00423 bool claims_html_; 00424 00427 bool started_parse_; 00428 00430 bool parse_text_called_; 00431 00433 bool done_called_; 00434 00435 HtmlDetector html_detector_; 00436 00441 ProxyFetchPropertyCallbackCollector* property_cache_callback_; 00442 00446 AsyncFetch* original_content_fetch_; 00447 00450 RewriteDriver* driver_; 00451 00454 bool queue_run_job_created_; 00455 00467 scoped_ptr<AbstractMutex> mutex_; 00468 StringStarVector text_queue_; 00469 bool network_flush_outstanding_; 00470 QueuedWorkerPool::Sequence* sequence_; 00471 00474 bool done_outstanding_; 00475 00478 bool finishing_; 00479 00482 bool done_result_; 00483 00487 bool waiting_for_flush_to_finish_; 00488 00491 QueuedAlarm* idle_alarm_; 00492 00493 ProxyFetchFactory* factory_; 00494 00496 bool distributed_fetch_; 00497 00498 DISALLOW_COPY_AND_ASSIGN(ProxyFetch); 00499 }; 00500 00501 } 00502 00503 #endif ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_