Page Speed Optimization Libraries
1.6.29.3
|
00001 /* 00002 * Copyright 2011 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00022 00023 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_ 00024 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_ 00025 00026 #include <map> 00027 #include <set> 00028 #include <vector> 00029 00030 #include "net/instaweb/automatic/public/html_detector.h" 00031 #include "net/instaweb/http/public/async_fetch.h" 00032 #include "net/instaweb/http/public/meta_data.h" 00033 #include "net/instaweb/http/public/request_context.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00035 #include "net/instaweb/util/public/queued_worker_pool.h" 00036 #include "net/instaweb/util/public/basictypes.h" 00037 #include "net/instaweb/util/public/fallback_property_page.h" 00038 #include "net/instaweb/util/public/gtest_prod.h" 00039 #include "net/instaweb/util/public/property_cache.h" 00040 #include "net/instaweb/util/public/scoped_ptr.h" 00041 #include "net/instaweb/util/public/string.h" 00042 #include "net/instaweb/util/public/string_util.h" 00043 00044 namespace net_instaweb { 00045 00046 class AbstractMutex; 00047 class CacheUrlAsyncFetcher; 00048 class Function; 00049 class GoogleUrl; 00050 class MessageHandler; 00051 class ProxyFetch; 00052 class ProxyFetchPropertyCallbackCollector; 00053 class QueuedAlarm; 00054 class ServerContext; 00055 class ResponseHeaders; 00056 class RewriteDriver; 00057 class RewriteOptions; 00058 class Timer; 00059 00062 class ProxyFetchFactory { 00063 public: 00064 explicit ProxyFetchFactory(ServerContext* server_context); 00065 ~ProxyFetchFactory(); 00066 00069 void StartNewProxyFetch( 00070 const GoogleString& url, 00071 AsyncFetch* async_fetch, 00072 RewriteDriver* driver, 00073 ProxyFetchPropertyCallbackCollector* property_callback, 00074 AsyncFetch* original_content_fetch); 00075 00085 ProxyFetch* CreateNewProxyFetch( 00086 const GoogleString& url, 00087 AsyncFetch* async_fetch, 00088 RewriteDriver* driver, 00089 ProxyFetchPropertyCallbackCollector* property_callback, 00090 AsyncFetch* original_content_fetch); 00091 00094 static ProxyFetchPropertyCallbackCollector* InitiatePropertyCacheLookup( 00095 bool is_resource_fetch, 00096 const GoogleUrl& request_url, 00097 ServerContext* server_context, 00098 RewriteOptions* options, 00099 AsyncFetch* async_fetch, 00100 const bool requires_blink_cohort, 00101 bool* added_page_property_callback); 00102 00103 MessageHandler* message_handler() const { return handler_; } 00104 00105 private: 00106 friend class ProxyFetch; 00107 00112 void RegisterNewFetch(ProxyFetch* proxy_fetch); 00113 void RegisterFinishedFetch(ProxyFetch* proxy_fetch); 00114 00115 ServerContext* server_context_; 00116 Timer* timer_; 00117 MessageHandler* handler_; 00118 00119 scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_; 00120 std::set<ProxyFetch*> outstanding_proxy_fetches_; 00121 00122 DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory); 00123 }; 00124 00133 class ProxyFetchPropertyCallback : public PropertyPage { 00134 public: 00135 ProxyFetchPropertyCallback(PageType page_type, 00136 PropertyCache* property_cache, 00137 const StringPiece& key, 00138 UserAgentMatcher::DeviceType device_type, 00139 ProxyFetchPropertyCallbackCollector* collector, 00140 AbstractMutex* mutex); 00141 00142 PageType page_type() const { return page_type_; } 00143 00144 UserAgentMatcher::DeviceType device_type() const { return device_type_; } 00145 00147 virtual bool IsCacheValid(int64 write_timestamp_ms) const; 00148 00149 virtual void Done(bool success); 00150 00151 private: 00152 PageType page_type_; 00153 UserAgentMatcher::DeviceType device_type_; 00154 ProxyFetchPropertyCallbackCollector* collector_; 00155 GoogleString url_; 00156 DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback); 00157 }; 00158 00160 class ProxyFetchPropertyCallbackCollector { 00161 public: 00162 ProxyFetchPropertyCallbackCollector(ServerContext* server_context, 00163 const StringPiece& url, 00164 const RequestContextPtr& req_ctx, 00165 const RewriteOptions* options, 00166 UserAgentMatcher::DeviceType device_type); 00167 virtual ~ProxyFetchPropertyCallbackCollector(); 00168 00171 void AddCallback(ProxyFetchPropertyCallback* callback); 00172 00179 void ConnectProxyFetch(ProxyFetch* proxy_fetch); 00180 00187 void Detach(HttpStatus::Code status_code); 00188 00190 PropertyPage* property_page() { 00191 return fallback_property_page_ == NULL ? 00192 NULL : fallback_property_page_->actual_property_page(); 00193 } 00194 00196 FallbackPropertyPage* fallback_property_page() { 00197 return fallback_property_page_.get(); 00198 } 00199 00202 PropertyPage* ReleasePropertyPage( 00203 ProxyFetchPropertyCallback::PageType page_type); 00204 00206 FallbackPropertyPage* ReleaseFallbackPropertyPage() { 00207 return fallback_property_page_.release(); 00208 } 00209 00219 void AddPostLookupTask(Function* func); 00220 00224 bool IsCacheValid(int64 write_timestamp_ms) const; 00225 00227 void Done(ProxyFetchPropertyCallback* callback); 00228 00230 void UpdateStatusCodeInPropertyCache(); 00231 00232 const RequestContextPtr& request_context() { return request_context_; } 00233 00235 UserAgentMatcher::DeviceType device_type() { return device_type_; } 00236 00237 private: 00238 std::set<ProxyFetchPropertyCallback*> pending_callbacks_; 00239 std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*> 00240 property_pages_; 00241 scoped_ptr<AbstractMutex> mutex_; 00242 ServerContext* server_context_; 00243 GoogleString url_; 00244 RequestContextPtr request_context_; 00245 UserAgentMatcher::DeviceType device_type_; 00246 bool detached_; 00247 bool done_; 00248 ProxyFetch* proxy_fetch_; 00249 00250 scoped_ptr<std::vector<Function*> > post_lookup_task_vector_; 00251 const RewriteOptions* options_; 00252 HttpStatus::Code status_code_; 00253 scoped_ptr<FallbackPropertyPage> fallback_property_page_; 00254 00255 DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector); 00256 }; 00257 00278 class ProxyFetch : public SharedAsyncFetch { 00279 public: 00282 static const char kCollectorDone[]; 00283 static const char kCollectorPrefix[]; 00284 static const char kCollectorReady[]; 00285 static const char kCollectorDelete[]; 00286 static const char kCollectorDetach[]; 00287 static const char kCollectorDoneDelete[]; 00288 00291 static const char kHeadersSetupRaceAlarmQueued[]; 00292 static const char kHeadersSetupRaceDone[]; 00293 static const char kHeadersSetupRaceFlush[]; 00294 static const char kHeadersSetupRacePrefix[]; 00295 static const char kHeadersSetupRaceWait[]; 00296 00303 static const int kTestSignalTimeoutMs = 200; 00304 00305 protected: 00307 virtual void HandleHeadersComplete(); 00308 virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler); 00309 virtual bool HandleFlush(MessageHandler* handler); 00310 virtual void HandleDone(bool success); 00311 virtual bool IsCachedResultValid(const ResponseHeaders& headers); 00312 00313 private: 00314 friend class ProxyFetchFactory; 00315 friend class ProxyFetchPropertyCallbackCollector; 00316 friend class MockProxyFetch; 00317 FRIEND_TEST(ProxyFetchTest, TestInhibitParsing); 00318 00321 virtual void PropertyCacheComplete( 00322 ProxyFetchPropertyCallbackCollector* collector); 00323 00326 ProxyFetch(const GoogleString& url, 00327 bool cross_domain, 00328 ProxyFetchPropertyCallbackCollector* property_cache_callback, 00329 AsyncFetch* async_fetch, 00330 AsyncFetch* original_content_fetch, 00331 RewriteDriver* driver, 00332 ServerContext* server_context, 00333 Timer* timer, 00334 ProxyFetchFactory* factory); 00335 virtual ~ProxyFetch(); 00336 00337 const RewriteOptions* Options(); 00338 00340 void SetupForHtml(); 00341 00343 void AddPagespeedHeader(); 00344 00347 bool StartParse(); 00348 00350 void StartFetch(); 00351 00355 void DoFetch(bool prepare_success); 00356 00359 void ExecuteQueued(); 00360 00363 void ScheduleQueueExecutionIfNeeded(); 00364 00369 void Finish(bool success); 00370 00372 void CompleteFinishParse(bool success); 00373 00376 void FlushDone(); 00377 00380 00382 void CancelIdleAlarm(); 00383 00385 void QueueIdleAlarm(); 00386 00388 void HandleIdleAlarm(); 00389 00390 GoogleString url_; 00391 ServerContext* server_context_; 00392 Timer* timer_; 00393 00394 scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_; 00395 00398 bool cross_domain_; 00399 00401 bool claims_html_; 00402 00405 bool started_parse_; 00406 00408 bool parse_text_called_; 00409 00411 bool done_called_; 00412 00413 HtmlDetector html_detector_; 00414 00419 ProxyFetchPropertyCallbackCollector* property_cache_callback_; 00420 00424 AsyncFetch* original_content_fetch_; 00425 00428 RewriteDriver* driver_; 00429 00432 bool queue_run_job_created_; 00433 00445 scoped_ptr<AbstractMutex> mutex_; 00446 StringStarVector text_queue_; 00447 bool network_flush_outstanding_; 00448 QueuedWorkerPool::Sequence* sequence_; 00449 00452 bool done_outstanding_; 00453 00456 bool finishing_; 00457 00460 bool done_result_; 00461 00465 bool waiting_for_flush_to_finish_; 00466 00469 QueuedAlarm* idle_alarm_; 00470 00471 ProxyFetchFactory* factory_; 00472 00474 bool distributed_fetch_; 00475 00476 DISALLOW_COPY_AND_ASSIGN(ProxyFetch); 00477 }; 00478 00479 } 00480 00481 #endif ///< NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_