Page Speed Optimization Libraries
1.7.30.3
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00021 00022 #include <map> 00023 #include <set> 00024 #include <vector> 00025 00026 #include "base/logging.h" 00027 #include "net/instaweb/htmlparse/public/html_element.h" 00028 #include "net/instaweb/htmlparse/public/html_parse.h" 00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h" 00030 #include "net/instaweb/http/public/http_cache.h" 00031 #include "net/instaweb/http/public/request_context.h" 00032 #include "net/instaweb/http/public/user_agent_matcher.h" 00033 #include "net/instaweb/rewriter/public/critical_images_finder.h" 00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h" 00035 #include "net/instaweb/rewriter/public/output_resource_kind.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/resource_slot.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/scan_filter.h" 00040 #include "net/instaweb/rewriter/public/server_context.h" 00041 #include "net/instaweb/util/public/basictypes.h" 00042 #include "net/instaweb/util/public/google_url.h" 00043 #include "net/instaweb/util/public/printf_format.h" 00044 #include "net/instaweb/util/public/queued_worker_pool.h" 00045 #include "net/instaweb/util/public/scheduler.h" 00046 #include "net/instaweb/util/public/scoped_ptr.h" 00047 #include "net/instaweb/util/public/string.h" 00048 #include "net/instaweb/util/public/string_util.h" 00049 #include "net/instaweb/util/public/thread_system.h" 00050 #include "net/instaweb/util/public/url_segment_encoder.h" 00051 #include "pagespeed/kernel/http/content_type.h" 00052 #include "pagespeed/kernel/util/categorized_refcount.h" 00053 00054 namespace net_instaweb { 00055 00056 class AbstractLogRecord; 00057 class AbstractMutex; 00058 class AbstractPropertyPage; 00059 class AddInstrumentationFilter; 00060 class AsyncFetch; 00061 class CriticalCssResult; 00062 class CriticalLineInfo; 00063 class DebugFilter; 00064 class DomStatsFilter; 00065 class DomainRewriteFilter; 00066 class FallbackPropertyPage; 00067 class FileSystem; 00068 class FlushEarlyInfo; 00069 class FlushEarlyRenderInfo; 00070 class Function; 00071 class HtmlFilter; 00072 class HtmlWriterFilter; 00073 class MessageHandler; 00074 class OutputResource; 00075 class PropertyPage; 00076 class RequestHeaders; 00077 class RequestProperties; 00078 class RequestTrace; 00079 class ResourceContext; 00080 class ResourceNamer; 00081 class ResponseHeaders; 00082 class RewriteContext; 00083 class RewriteDriverPool; 00084 class RewriteFilter; 00085 class SplitHtmlConfig; 00086 class Statistics; 00087 class UrlAsyncFetcher; 00088 class UrlLeftTrimFilter; 00089 class Writer; 00090 00093 class RewriteDriver : public HtmlParse { 00094 public: 00096 enum CssResolutionStatus { 00097 kWriteFailed, 00098 kNoResolutionNeeded, 00099 kSuccess 00100 }; 00101 00103 enum WaitMode { 00104 kNoWait, 00105 kWaitForCompletion, 00106 kWaitForCachedRender, 00107 00108 kWaitForShutDown 00109 00110 }; 00111 00123 enum XhtmlStatus { 00124 kXhtmlUnknown, 00125 kIsXhtml, 00126 kIsNotXhtml 00127 }; 00128 00135 static const char* kPassThroughRequestAttributes[7]; 00136 00140 static const char kDomCohort[]; 00142 static const char kBeaconCohort[]; 00143 00146 static const char kLastRequestTimestamp[]; 00148 static const char kParseSizeLimitExceeded[]; 00150 static const char kSubresourcesPropertyName[]; 00152 static const char kStatusCodePropertyName[]; 00153 00154 RewriteDriver(MessageHandler* message_handler, 00155 FileSystem* file_system, 00156 UrlAsyncFetcher* url_async_fetcher); 00157 00160 virtual ~RewriteDriver(); 00161 00164 RewriteDriver* Clone(); 00165 00171 void Clear(); 00172 00174 static void InitStats(Statistics* statistics); 00175 00177 static void Initialize(); 00178 static void Terminate(); 00179 00182 void SetServerContext(ServerContext* server_context); 00183 00186 bool MayCacheExtendCss() const; 00187 bool MayCacheExtendImages() const; 00188 bool MayCacheExtendPdfs() const; 00189 bool MayCacheExtendScripts() const; 00190 00191 const GoogleString& user_agent() const { return user_agent_; } 00192 00193 void SetUserAgent(const StringPiece& user_agent_string); 00194 00195 const RequestProperties* request_properties() const { 00196 return request_properties_.get(); 00197 } 00198 00200 void ClearRequestProperties(); 00201 00203 bool using_spdy() const { return request_context_->using_spdy(); } 00204 00205 bool write_property_cache_dom_cohort() const { 00206 return write_property_cache_dom_cohort_; 00207 } 00208 void set_write_property_cache_dom_cohort(bool x) { 00209 write_property_cache_dom_cohort_ = x; 00210 } 00211 00212 RequestContextPtr request_context() { return request_context_; } 00213 void set_request_context(const RequestContextPtr& x); 00214 00217 RequestTrace* trace_context(); 00218 00221 void TracePrintf(const char* fmt, ...); 00222 00225 ResponseHeaders* mutable_response_headers() { 00226 return flush_occurred_ ? NULL : response_headers_; 00227 } 00228 00235 const ResponseHeaders* response_headers() { 00236 return response_headers_; 00237 } 00238 00242 void set_response_headers_ptr(ResponseHeaders* headers) { 00243 response_headers_ = headers; 00244 } 00245 00251 void SetRequestHeaders(const RequestHeaders& headers); 00252 00253 const RequestHeaders* request_headers() const { 00254 return request_headers_.get(); 00255 } 00256 00257 UserAgentMatcher* user_agent_matcher() const { 00258 DCHECK(server_context() != NULL); 00259 return server_context()->user_agent_matcher(); 00260 } 00261 00266 void AddFilters(); 00267 00271 void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter); 00272 00274 void PrependOwnedPreRenderFilter(HtmlFilter* filter); 00276 void AppendOwnedPreRenderFilter(HtmlFilter* filter); 00277 00279 void AddOwnedPostRenderFilter(HtmlFilter* filter); 00281 void AddUnownedPostRenderFilter(HtmlFilter* filter); 00282 00294 void AppendRewriteFilter(RewriteFilter* filter); 00295 00298 void PrependRewriteFilter(RewriteFilter* filter); 00299 00305 void SetWriter(Writer* writer); 00306 00307 Writer* writer() const { return writer_; } 00308 00337 bool FetchResource(const StringPiece& url, AsyncFetch* fetch); 00338 00356 void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode, 00357 AsyncFetch* async_fetch); 00358 00365 bool FetchOutputResource(const OutputResourcePtr& output_resource, 00366 RewriteFilter* filter, 00367 AsyncFetch* async_fetch); 00368 00375 OutputResourcePtr DecodeOutputResource(const GoogleUrl& url, 00376 RewriteFilter** filter) const; 00377 00381 bool DecodeOutputResourceName(const GoogleUrl& url, 00382 ResourceNamer* name_out, 00383 OutputResourceKind* kind_out, 00384 RewriteFilter** filter_out) const; 00385 00387 bool DecodeUrl(const GoogleUrl& url, 00388 StringVector* decoded_urls) const; 00389 00390 FileSystem* file_system() { return file_system_; } 00391 UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; } 00392 00397 void SetSessionFetcher(UrlAsyncFetcher* f); 00398 00399 UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; } 00401 void set_distributed_fetcher(UrlAsyncFetcher* fetcher) { 00402 distributed_async_fetcher_ = fetcher; 00403 } 00404 00407 CacheUrlAsyncFetcher* CreateCacheFetcher(); 00409 CacheUrlAsyncFetcher* CreateCacheOnlyFetcher(); 00410 00411 ServerContext* server_context() const { return server_context_; } 00412 Statistics* statistics() const; 00413 00414 AddInstrumentationFilter* add_instrumentation_filter() { 00415 return add_instrumentation_filter_; 00416 } 00417 00419 void set_custom_options(RewriteOptions* options) { 00420 set_options_for_pool(NULL, options); 00421 } 00422 00425 void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) { 00426 controlling_pool_ = pool; 00427 options_.reset(options); 00428 } 00429 00431 RewriteDriverPool* controlling_pool() { return controlling_pool_; } 00432 00434 const RewriteOptions* options() const { return options_.get(); } 00435 00439 virtual bool StartParseId(const StringPiece& url, const StringPiece& id, 00440 const ContentType& content_type); 00441 00448 virtual void FinishParse(); 00449 00453 void FinishParseAsync(Function* callback); 00454 00458 void InfoAt(const RewriteContext* context, 00459 const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4); 00460 00465 00472 OutputResourcePtr CreateOutputResourceFromResource( 00473 const StringPiece& filter_id, 00474 const UrlSegmentEncoder* encoder, 00475 const ResourceContext* data, 00476 const ResourcePtr& input_resource, 00477 OutputResourceKind kind); 00478 00495 OutputResourcePtr CreateOutputResourceWithPath( 00496 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00497 const StringPiece& base_url, const StringPiece& filter_id, 00498 const StringPiece& name, OutputResourceKind kind); 00499 00502 void PopulateResourceNamer( 00503 const StringPiece& filter_id, 00504 const StringPiece& name, 00505 ResourceNamer* full_name); 00506 00510 OutputResourcePtr CreateOutputResourceWithUnmappedUrl( 00511 const GoogleUrl& unmapped_gurl, const StringPiece& filter_id, 00512 const StringPiece& name, OutputResourceKind kind); 00513 00516 OutputResourcePtr CreateOutputResourceWithMappedPath( 00517 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00518 const StringPiece& filter_id, const StringPiece& name, 00519 OutputResourceKind kind) { 00520 return CreateOutputResourceWithPath(mapped_path, unmapped_path, 00521 decoded_base_url_.AllExceptLeaf(), 00522 filter_id, name, kind); 00523 } 00524 00527 OutputResourcePtr CreateOutputResourceWithPath( 00528 const StringPiece& path, const StringPiece& filter_id, 00529 const StringPiece& name, OutputResourceKind kind) { 00530 return CreateOutputResourceWithPath(path, path, path, filter_id, name, 00531 kind); 00532 } 00533 00537 ResourcePtr CreateInputResource(const GoogleUrl& input_url); 00538 00545 ResourcePtr CreateInputResource(const GoogleUrl& input_url, 00546 bool allow_unauthorized_domain); 00547 00551 ResourcePtr CreateInputResourceAbsoluteUnchecked( 00552 const StringPiece& absolute_url); 00553 00558 bool MatchesBaseUrl(const GoogleUrl& input_url) const; 00559 00566 bool MayRewriteUrl(const GoogleUrl& domain_url, 00567 const GoogleUrl& input_url, 00568 bool allow_unauthorized_domain, 00569 bool* is_authorized_domain) const; 00570 00574 const GoogleUrl& base_url() const { return base_url_; } 00575 00577 StringPiece fetch_url() const { return fetch_url_; } 00578 00582 const GoogleUrl& decoded_base_url() const { return decoded_base_url_; } 00583 StringPiece decoded_base() const { return decoded_base_url_.Spec(); } 00584 00586 bool IsHttps() const { return google_url().SchemeIs("https"); } 00587 00588 const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; } 00589 00591 RewriteFilter* FindFilter(const StringPiece& id) const; 00592 00594 bool refs_before_base() { return refs_before_base_; } 00595 00600 void set_refs_before_base() { refs_before_base_ = true; } 00601 00606 StringPiece containing_charset() { return containing_charset_; } 00607 void set_containing_charset(const StringPiece charset) { 00608 charset.CopyToString(&containing_charset_); 00609 } 00610 00612 HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource, 00613 HtmlElement* elt, 00614 HtmlElement::Attribute* attr); 00615 00620 bool InitiateRewrite(RewriteContext* rewrite_context); 00621 void InitiateFetch(RewriteContext* rewrite_context); 00622 00633 void RewriteComplete(RewriteContext* rewrite_context, bool permit_render); 00634 00638 void ReportSlowRewrites(int num); 00639 00644 void Cleanup(); 00645 00650 void AddUserReference(); 00651 00653 GoogleString ToString(bool show_detached_contexts); 00654 void PrintState(bool show_detached_contexts); 00655 void PrintStateToErrorLog(bool show_detached_contexts); 00656 00659 void WaitForCompletion(); 00660 00667 void WaitForShutDown(); 00668 00672 void BoundedWaitFor(WaitMode mode, int64 timeout_ms); 00673 00681 void set_fully_rewrite_on_flush(bool x) { 00682 fully_rewrite_on_flush_ = x; 00683 } 00684 00686 bool fully_rewrite_on_flush() const { 00687 return fully_rewrite_on_flush_; 00688 } 00689 00693 void set_fast_blocking_rewrite(bool x) { 00694 fast_blocking_rewrite_ = x; 00695 } 00696 00697 bool fast_blocking_rewrite() const { 00698 return fast_blocking_rewrite_; 00699 } 00700 00703 void EnableBlockingRewrite(RequestHeaders* request_headers); 00704 00711 void set_externally_managed(bool x) { externally_managed_ = x; } 00712 00716 void DetachFetch(); 00717 00720 void DetachedFetchComplete(); 00721 00725 void FetchComplete(); 00726 00732 void DeleteRewriteContext(RewriteContext* rewrite_context); 00733 00734 int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); } 00735 00740 void set_max_page_processing_delay_ms(int x) { 00741 max_page_processing_delay_ms_ = x; 00742 } 00743 int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; } 00744 00746 void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; } 00747 UserAgentMatcher::DeviceType device_type() { return device_type_; } 00748 00754 RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key, 00755 RewriteContext* candidate); 00756 00761 void DeregisterForPartitionKey( 00762 const GoogleString& partition_key, RewriteContext* candidate); 00763 00766 void RequestFlush() { flush_requested_ = true; } 00767 bool flush_requested() const { return flush_requested_; } 00768 00780 void ExecuteFlushIfRequested(); 00781 00785 void ExecuteFlushIfRequestedAsync(Function* callback); 00786 00795 virtual void Flush(); 00796 00800 void FlushAsync(Function* done); 00801 00803 void AddRewriteTask(Function* task); 00804 00807 void AddLowPriorityRewriteTask(Function* task); 00808 00809 QueuedWorkerPool::Sequence* html_worker() { return html_worker_; } 00810 QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; } 00811 QueuedWorkerPool::Sequence* low_priority_rewrite_worker() { 00812 return low_priority_rewrite_worker_; 00813 } 00814 00815 Scheduler* scheduler() { return scheduler_; } 00816 00819 DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); } 00820 UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); } 00821 00829 CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base, 00830 const StringPiece& output_css_base, 00831 const StringPiece& contents, 00832 Writer* writer, 00833 MessageHandler* handler); 00834 00842 bool ShouldAbsolutifyUrl(const GoogleUrl& input_base, 00843 const GoogleUrl& output_base, 00844 bool* proxy_mode) const; 00845 00853 void UpdatePropertyValueInDomCohort( 00854 AbstractPropertyPage* page, 00855 StringPiece property_name, 00856 StringPiece property_value); 00857 00860 PropertyPage* property_page() const; 00865 FallbackPropertyPage* fallback_property_page() const { 00866 return fallback_property_page_; 00867 } 00869 void set_property_page(PropertyPage* page); 00871 void set_fallback_property_page(FallbackPropertyPage* page); 00873 void set_unowned_fallback_property_page(FallbackPropertyPage* page); 00874 00876 const CriticalLineInfo* critical_line_info() const; 00877 00880 void set_critical_line_info(CriticalLineInfo* critical_line_info); 00881 00882 const SplitHtmlConfig* split_html_config(); 00883 00884 CriticalCssResult* critical_css_result() const; 00887 void set_critical_css_result(CriticalCssResult* critical_css_rules); 00888 00890 CriticalImagesInfo* critical_images_info() const { 00891 return critical_images_info_.get(); 00892 } 00893 00898 CriticalSelectorInfo* critical_selector_info() { 00899 return critical_selector_info_.get(); 00900 } 00901 00905 void set_critical_selector_info(CriticalSelectorInfo* info) { 00906 critical_selector_info_.reset(info); 00907 } 00908 00913 void set_critical_images_info(CriticalImagesInfo* critical_images_info) { 00914 critical_images_info_.reset(critical_images_info); 00915 } 00916 00919 bool CriticalSelectorsEnabled() const; 00920 00923 bool FlattenCssImportsEnabled() const { 00924 return (options()->Enabled(RewriteOptions::kFlattenCssImports) || 00925 (!options()->Forbidden(RewriteOptions::kFlattenCssImports) && 00926 (CriticalSelectorsEnabled() || 00927 options()->Enabled(RewriteOptions::kComputeCriticalCss)))); 00928 } 00929 00933 int num_inline_preview_images() const { return num_inline_preview_images_; } 00934 00936 void increment_num_inline_preview_images(); 00937 00940 int num_flushed_early_pagespeed_resources() const { 00941 return num_flushed_early_pagespeed_resources_; 00942 } 00943 00946 void increment_num_flushed_early_pagespeed_resources() { 00947 ++num_flushed_early_pagespeed_resources_; 00948 } 00949 00952 void increment_async_events_count(); 00953 00955 void decrement_async_events_count(); 00956 00959 XhtmlStatus MimeTypeXhtmlStatus(); 00960 00961 void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; } 00962 bool flushed_cached_html() { return flushed_cached_html_; } 00963 00964 void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; } 00965 bool flushing_cached_html() const { return flushing_cached_html_; } 00966 00967 void set_flushed_early(bool x) { flushed_early_ = x; } 00968 bool flushed_early() const { return flushed_early_; } 00969 00970 void set_flushing_early(bool x) { flushing_early_ = x; } 00971 bool flushing_early() const { return flushing_early_; } 00972 00973 void set_is_lazyload_script_flushed(bool x) { 00974 is_lazyload_script_flushed_ = x; 00975 } 00976 bool is_lazyload_script_flushed() const { 00977 return is_lazyload_script_flushed_; } 00978 00980 FlushEarlyInfo* flush_early_info(); 00981 00982 FlushEarlyRenderInfo* flush_early_render_info() const; 00983 00986 void set_flush_early_render_info( 00987 FlushEarlyRenderInfo* flush_early_render_info); 00988 00991 bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); } 00992 00995 void SaveOriginalHeaders(const ResponseHeaders& response_headers); 00996 00999 AbstractLogRecord* log_record(); 01000 01001 DomStatsFilter* dom_stats_filter() const { 01002 return dom_stats_filter_; 01003 } 01004 01007 bool can_rewrite_resources() const { return can_rewrite_resources_; } 01008 01010 bool is_nested() const { return is_nested_; } 01011 01015 bool MetadataRequested(const RequestHeaders& request_headers) const; 01016 01018 bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; } 01019 01032 bool Write(const ResourceVector& inputs, 01033 const StringPiece& contents, 01034 const ContentType* type, 01035 StringPiece charset, 01036 OutputResource* output); 01037 01038 void set_defer_instrumentation_script(bool x) { 01039 defer_instrumentation_script_ = x; 01040 } 01041 bool defer_instrumentation_script() const { 01042 return defer_instrumentation_script_; 01043 } 01044 01045 protected: 01046 virtual void DetermineEnabledFiltersImpl(); 01047 01048 private: 01049 friend class DistributedRewriteContextTest; 01050 friend class RewriteContext; 01051 friend class RewriteDriverTest; 01052 friend class RewriteTestBase; 01053 friend class ServerContextTest; 01054 01055 typedef std::map<GoogleString, RewriteFilter*> StringFilterMap; 01056 01058 bool ShouldDistributeFetch(const StringPiece& filter_id); 01059 01071 bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id, 01072 AsyncFetch* async_fetch); 01073 01079 void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms, 01080 Function* done); 01081 01086 void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms, 01087 Function* done); 01088 01090 bool IsDone(WaitMode wait_mode, bool deadline_reached); 01091 01094 bool WaitForPendingAsyncEvents(WaitMode wait_mode) { 01095 return wait_mode == kWaitForShutDown || 01096 (fully_rewrite_on_flush_ && !fast_blocking_rewrite_); 01097 } 01098 01102 void FlushAsyncDone(int num_rewrites, Function* callback); 01103 01108 int64 ComputeCurrentFlushWindowRewriteDelayMs(); 01109 01111 void QueueFlushAsyncDone(int num_rewrites, Function* callback); 01112 01115 void QueueFinishParseAfterFlush(Function* user_callback); 01116 void FinishParseAfterFlush(Function* user_callback); 01117 01119 bool RewritesComplete() const; 01120 01123 void SetBaseUrlIfUnset(const StringPiece& new_base); 01124 01127 void SetBaseUrlForFetch(const StringPiece& url); 01128 01131 void SetDecodedUrlFromBase(); 01132 01134 AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); } 01135 01137 virtual void ParseTextInternal(const char* content, int size); 01138 01140 bool ShouldSkipParsing(); 01141 01142 friend class ScanFilter; 01143 01147 void RegisterRewriteFilter(RewriteFilter* filter); 01148 01153 void EnableRewriteFilter(const char* id); 01154 01160 ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl, 01161 bool is_authorized_domain); 01162 01163 void AddPreRenderFilters(); 01164 void AddPostRenderFilters(); 01165 01167 bool DecodeOutputResourceNameHelper(const GoogleUrl& url, 01168 ResourceNamer* name_out, 01169 OutputResourceKind* kind_out, 01170 RewriteFilter** filter_out, 01171 GoogleString* url_base, 01172 StringVector* urls) const; 01173 01183 void WriteDomCohortIntoPropertyCache(); 01184 01186 CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher); 01187 01194 void PossiblyPurgeCachedResponseAndReleaseDriver(); 01195 01200 bool ShouldPurgeRewrittenResponse(); 01201 01204 static bool GetPurgeUrl(const GoogleUrl& google_url, 01205 const RewriteOptions* options, 01206 GoogleString* purge_url, 01207 GoogleString* purge_method); 01208 01210 void PurgeDownstreamCache(const GoogleString& purge_url, 01211 const GoogleString& purge_method); 01212 01214 void LogStats(); 01215 01232 bool PrepareShouldSignal(); 01233 void SignalIfRequired(bool result_of_prepare_should_signal); 01234 01246 bool base_was_set_; 01247 01252 bool refs_before_base_; 01253 01255 GoogleString containing_charset_; 01256 01259 void PopulateRequestContext(); 01260 01261 bool filters_added_; 01262 bool externally_managed_; 01263 01272 enum RefCategory { 01273 kRefUser, 01274 kRefParsing, 01275 01279 kRefPendingRewrites, 01280 01284 kRefDetachedRewrites, 01285 01293 kRefDeletingRewrites, 01294 01296 kRefFetchUserFacing, 01297 01299 kRefFetchBackground, 01300 01305 kRefAsyncEvents, 01306 01307 kNumRefCategories 01308 }; 01309 01310 friend class CategorizedRefcount<RewriteDriver, RefCategory>; 01311 01313 CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_; 01314 01316 void LastRefRemoved(); 01317 StringPiece RefCategoryName(RefCategory cat); 01318 01321 void DropReference(RefCategory cat); 01322 01325 bool release_driver_; 01326 01329 bool parsing_; 01330 01334 WaitMode waiting_; 01335 01337 bool waiting_deadline_reached_; 01338 01343 bool fully_rewrite_on_flush_; 01344 01347 bool fast_blocking_rewrite_; 01348 01349 bool flush_requested_; 01350 bool flush_occurred_; 01351 01353 bool flushed_cached_html_; 01354 01356 bool flushing_cached_html_; 01357 01360 bool flushed_early_; 01364 bool flushing_early_; 01365 01368 bool is_lazyload_script_flushed_; 01369 01372 bool made_downstream_purge_attempt_; 01373 01377 bool write_property_cache_dom_cohort_; 01378 01381 GoogleUrl base_url_; 01382 01386 GoogleUrl decoded_base_url_; 01387 01390 GoogleString fetch_url_; 01391 01392 GoogleString user_agent_; 01393 01394 LazyBool should_skip_parsing_; 01395 01396 StringFilterMap resource_filter_map_; 01397 01398 ResponseHeaders* response_headers_; 01399 01402 scoped_ptr<const RequestHeaders> request_headers_; 01403 01404 int status_code_; 01405 01408 typedef std::vector<RewriteContext*> RewriteContextVector; 01409 RewriteContextVector rewrites_; 01410 01413 int max_page_processing_delay_ms_; 01414 01415 typedef std::set<RewriteContext*> RewriteContextSet; 01416 01421 RewriteContextSet initiated_rewrites_; 01422 01424 int64 num_initiated_rewrites_; 01425 01433 int64 num_detached_rewrites_; 01434 01442 RewriteContextSet detached_rewrites_; 01443 01445 int possibly_quick_rewrites_; 01446 01449 RewriteContextVector fetch_rewrites_; 01450 01453 FileSystem* file_system_; 01454 ServerContext* server_context_; 01455 Scheduler* scheduler_; 01456 UrlAsyncFetcher* default_url_async_fetcher_; 01457 01461 UrlAsyncFetcher* url_async_fetcher_; 01462 01466 UrlAsyncFetcher* distributed_async_fetcher_; 01467 01470 std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_; 01471 01472 AddInstrumentationFilter* add_instrumentation_filter_; 01473 DomStatsFilter* dom_stats_filter_; 01474 scoped_ptr<HtmlWriterFilter> html_writer_filter_; 01475 01476 ScanFilter scan_filter_; 01477 scoped_ptr<DomainRewriteFilter> domain_rewriter_; 01478 scoped_ptr<UrlLeftTrimFilter> url_trim_filter_; 01479 01482 typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap; 01483 PrimaryRewriteContextMap primary_rewrite_context_map_; 01484 01485 HtmlResourceSlotSet slots_; 01486 01487 scoped_ptr<RewriteOptions> options_; 01488 01489 RewriteDriverPool* controlling_pool_; 01490 01492 scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks> 01493 cache_url_async_fetcher_async_op_hooks_; 01494 01496 UrlSegmentEncoder default_encoder_; 01497 01499 FilterList early_pre_render_filters_; 01501 FilterList pre_render_filters_; 01502 01506 FilterVector filters_to_delete_; 01507 01508 QueuedWorkerPool::Sequence* html_worker_; 01509 QueuedWorkerPool::Sequence* rewrite_worker_; 01510 QueuedWorkerPool::Sequence* low_priority_rewrite_worker_; 01511 01512 Writer* writer_; 01513 01516 FallbackPropertyPage* fallback_property_page_; 01517 01519 bool owns_property_page_; 01520 01522 UserAgentMatcher::DeviceType device_type_; 01523 01524 scoped_ptr<CriticalLineInfo> critical_line_info_; 01525 01526 scoped_ptr<SplitHtmlConfig> split_html_config_; 01527 01530 scoped_ptr<CriticalImagesInfo> critical_images_info_; 01531 scoped_ptr<CriticalSelectorInfo> critical_selector_info_; 01532 01533 scoped_ptr<CriticalCssResult> critical_css_result_; 01534 01536 bool xhtml_mimetype_computed_; 01537 XhtmlStatus xhtml_status_ : 8; 01538 01541 int num_inline_preview_images_; 01542 01544 int num_flushed_early_pagespeed_resources_; 01545 01547 int num_bytes_in_; 01548 01549 DebugFilter* debug_filter_; 01550 01551 scoped_ptr<FlushEarlyInfo> flush_early_info_; 01552 scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_; 01553 01554 bool can_rewrite_resources_; 01555 bool is_nested_; 01556 01559 RequestContextPtr request_context_; 01560 01562 int64 start_time_ms_; 01563 01564 scoped_ptr<RequestProperties> request_properties_; 01565 01569 static int initialized_count_; 01570 01573 bool tried_to_distribute_fetch_; 01574 01577 bool defer_instrumentation_script_; 01578 01579 DISALLOW_COPY_AND_ASSIGN(RewriteDriver); 01580 }; 01581 01584 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback { 01585 public: 01586 virtual ~OptionsAwareHTTPCacheCallback(); 01587 virtual bool IsCacheValid(const GoogleString& key, 01588 const ResponseHeaders& headers); 01589 virtual int64 OverrideCacheTtlMs(const GoogleString& key); 01590 01594 static bool IsCacheValid(const GoogleString& key, 01595 const RewriteOptions& rewrite_options, 01596 const RequestContextPtr& request_ctx, 01597 const ResponseHeaders& headers); 01598 01599 protected: 01602 OptionsAwareHTTPCacheCallback( 01603 const RewriteOptions* rewrite_options, 01604 const RequestContextPtr& request_ctx); 01605 01606 private: 01607 const RewriteOptions* rewrite_options_; 01608 01609 DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback); 01610 }; 01611 01612 } 01613 01614 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_