Page Speed Optimization Libraries
1.6.29.3
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00021 00022 #include <map> 00023 #include <set> 00024 #include <vector> 00025 00026 #include "base/logging.h" 00027 #include "net/instaweb/htmlparse/public/html_element.h" 00028 #include "net/instaweb/htmlparse/public/html_parse.h" 00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h" 00030 #include "net/instaweb/http/public/http_cache.h" 00031 #include "net/instaweb/http/public/request_context.h" 00032 #include "net/instaweb/http/public/user_agent_matcher.h" 00033 #include "net/instaweb/rewriter/public/critical_images_finder.h" 00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h" 00035 #include "net/instaweb/rewriter/public/output_resource_kind.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/resource_slot.h" 00038 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h" 00039 #include "net/instaweb/rewriter/public/rewrite_options.h" 00040 #include "net/instaweb/rewriter/public/scan_filter.h" 00041 #include "net/instaweb/rewriter/public/server_context.h" 00042 #include "net/instaweb/util/public/basictypes.h" 00043 #include "net/instaweb/util/public/google_url.h" 00044 #include "net/instaweb/util/public/printf_format.h" 00045 #include "net/instaweb/util/public/queued_worker_pool.h" 00046 #include "net/instaweb/util/public/scheduler.h" 00047 #include "net/instaweb/util/public/scoped_ptr.h" 00048 #include "net/instaweb/util/public/string.h" 00049 #include "net/instaweb/util/public/string_util.h" 00050 #include "net/instaweb/util/public/thread_system.h" 00051 #include "net/instaweb/util/public/url_segment_encoder.h" 00052 #include "pagespeed/kernel/http/content_type.h" 00053 00054 namespace net_instaweb { 00055 00056 class AbstractLogRecord; 00057 class AbstractMutex; 00058 class AbstractPropertyPage; 00059 class AddInstrumentationFilter; 00060 class AsyncFetch; 00061 class CommonFilter; 00062 class CriticalCssResult; 00063 class CriticalLineInfo; 00064 class DebugFilter; 00065 class DomStatsFilter; 00066 class DomainRewriteFilter; 00067 class FallbackPropertyPage; 00068 class FileSystem; 00069 class FlushEarlyInfo; 00070 class FlushEarlyRenderInfo; 00071 class Function; 00072 class HtmlFilter; 00073 class HtmlWriterFilter; 00074 class MessageHandler; 00075 class OutputResource; 00076 class PropertyPage; 00077 class RequestHeaders; 00078 class RequestProperties; 00079 class RequestTrace; 00080 class ResourceContext; 00081 class ResourceNamer; 00082 class ResponseHeaders; 00083 class RewriteContext; 00084 class RewriteDriverPool; 00085 class RewriteFilter; 00086 class ScopedMutex; 00087 class Statistics; 00088 class UrlAsyncFetcher; 00089 class UrlLeftTrimFilter; 00090 class Writer; 00091 00094 class RewriteDriver : public HtmlParse { 00095 public: 00097 enum CssResolutionStatus { 00098 kWriteFailed, 00099 kNoResolutionNeeded, 00100 kSuccess 00101 }; 00102 00104 enum WaitMode { 00105 kNoWait, 00106 kWaitForCompletion, 00107 kWaitForCachedRender, 00108 00109 kWaitForShutDown 00110 00111 }; 00112 00124 enum XhtmlStatus { 00125 kXhtmlUnknown, 00126 kIsXhtml, 00127 kIsNotXhtml 00128 }; 00129 00136 static const char* kPassThroughRequestAttributes[7]; 00137 00141 static const char kDomCohort[]; 00143 static const char kBeaconCohort[]; 00144 00147 static const char kLastRequestTimestamp[]; 00149 static const char kParseSizeLimitExceeded[]; 00151 static const char kSubresourcesPropertyName[]; 00153 static const char kStatusCodePropertyName[]; 00156 static const char kXPsaBlockingRewriteModeSlow[]; 00157 00158 RewriteDriver(MessageHandler* message_handler, 00159 FileSystem* file_system, 00160 UrlAsyncFetcher* url_async_fetcher); 00161 00164 virtual ~RewriteDriver(); 00165 00168 RewriteDriver* Clone(); 00169 00175 void Clear(); 00176 00178 static void InitStats(Statistics* statistics); 00179 00181 static void Initialize(); 00182 static void Terminate(); 00183 00186 void SetServerContext(ServerContext* server_context); 00187 00190 bool MayCacheExtendCss() const; 00191 bool MayCacheExtendImages() const; 00192 bool MayCacheExtendPdfs() const; 00193 bool MayCacheExtendScripts() const; 00194 00195 const GoogleString& user_agent() const { return user_agent_; } 00196 00197 void SetUserAgent(const StringPiece& user_agent_string); 00198 00199 const RequestProperties* request_properties() const { 00200 return request_properties_.get(); 00201 } 00202 00204 void ClearRequestProperties(); 00205 00207 bool using_spdy() const { return request_context_->using_spdy(); } 00208 00209 bool write_property_cache_dom_cohort() const { 00210 return write_property_cache_dom_cohort_; 00211 } 00212 void set_write_property_cache_dom_cohort(bool x) { 00213 write_property_cache_dom_cohort_ = x; 00214 } 00215 00216 RequestContextPtr request_context() { return request_context_; } 00217 void set_request_context(const RequestContextPtr& x); 00218 00221 RequestTrace* trace_context(); 00222 00225 void TracePrintf(const char* fmt, ...); 00226 00229 ResponseHeaders* mutable_response_headers() { 00230 return flush_occurred_ ? NULL : response_headers_; 00231 } 00232 00239 const ResponseHeaders* response_headers() { 00240 return response_headers_; 00241 } 00242 00246 void set_response_headers_ptr(ResponseHeaders* headers) { 00247 response_headers_ = headers; 00248 } 00249 00255 void SetRequestHeaders(const RequestHeaders& headers); 00256 00257 const RequestHeaders* request_headers() const { 00258 return request_headers_.get(); 00259 } 00260 00261 UserAgentMatcher* user_agent_matcher() const { 00262 DCHECK(server_context() != NULL); 00263 return server_context()->user_agent_matcher(); 00264 } 00265 00270 void AddFilters(); 00271 00275 void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter); 00276 00278 void PrependOwnedPreRenderFilter(HtmlFilter* filter); 00280 void AppendOwnedPreRenderFilter(HtmlFilter* filter); 00281 00283 void AddOwnedPostRenderFilter(HtmlFilter* filter); 00285 void AddUnownedPostRenderFilter(HtmlFilter* filter); 00286 00298 void AppendRewriteFilter(RewriteFilter* filter); 00299 00302 void PrependRewriteFilter(RewriteFilter* filter); 00303 00309 void SetWriter(Writer* writer); 00310 00311 Writer* writer() const { return writer_; } 00312 00341 bool FetchResource(const StringPiece& url, AsyncFetch* fetch); 00342 00360 void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode, 00361 AsyncFetch* async_fetch); 00362 00369 bool FetchOutputResource(const OutputResourcePtr& output_resource, 00370 RewriteFilter* filter, 00371 AsyncFetch* async_fetch); 00372 00379 OutputResourcePtr DecodeOutputResource(const GoogleUrl& url, 00380 RewriteFilter** filter) const; 00381 00385 bool DecodeOutputResourceName(const GoogleUrl& url, 00386 ResourceNamer* name_out, 00387 OutputResourceKind* kind_out, 00388 RewriteFilter** filter_out) const; 00389 00391 bool DecodeUrl(const GoogleUrl& url, 00392 StringVector* decoded_urls) const; 00393 00394 FileSystem* file_system() { return file_system_; } 00395 UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; } 00396 00401 void SetSessionFetcher(UrlAsyncFetcher* f); 00402 00403 UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; } 00405 void set_distributed_fetcher(UrlAsyncFetcher* fetcher) { 00406 distributed_async_fetcher_ = fetcher; 00407 } 00408 00411 CacheUrlAsyncFetcher* CreateCacheFetcher(); 00413 CacheUrlAsyncFetcher* CreateCacheOnlyFetcher(); 00414 00415 ServerContext* server_context() const { return server_context_; } 00416 Statistics* statistics() const; 00417 00418 AddInstrumentationFilter* add_instrumentation_filter() { 00419 return add_instrumentation_filter_; 00420 } 00421 00423 void set_custom_options(RewriteOptions* options) { 00424 set_options_for_pool(NULL, options); 00425 } 00426 00429 void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) { 00430 controlling_pool_ = pool; 00431 options_.reset(options); 00432 } 00433 00435 RewriteDriverPool* controlling_pool() { return controlling_pool_; } 00436 00438 const RewriteOptions* options() const { return options_.get(); } 00439 00441 virtual bool StartParseId(const StringPiece& url, const StringPiece& id, 00442 const ContentType& content_type); 00443 00450 virtual void FinishParse(); 00451 00454 void FinishParseAsync(Function* callback); 00455 00459 void InfoAt(const RewriteContext* context, 00460 const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4); 00461 00466 00473 OutputResourcePtr CreateOutputResourceFromResource( 00474 const StringPiece& filter_id, 00475 const UrlSegmentEncoder* encoder, 00476 const ResourceContext* data, 00477 const ResourcePtr& input_resource, 00478 OutputResourceKind kind); 00479 00496 OutputResourcePtr CreateOutputResourceWithPath( 00497 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00498 const StringPiece& base_url, const StringPiece& filter_id, 00499 const StringPiece& name, OutputResourceKind kind); 00500 00503 void PopulateResourceNamer( 00504 const StringPiece& filter_id, 00505 const StringPiece& name, 00506 ResourceNamer* full_name); 00507 00511 OutputResourcePtr CreateOutputResourceWithUnmappedUrl( 00512 const GoogleUrl& unmapped_gurl, const StringPiece& filter_id, 00513 const StringPiece& name, OutputResourceKind kind); 00514 00517 OutputResourcePtr CreateOutputResourceWithMappedPath( 00518 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00519 const StringPiece& filter_id, const StringPiece& name, 00520 OutputResourceKind kind) { 00521 return CreateOutputResourceWithPath(mapped_path, unmapped_path, 00522 decoded_base_url_.AllExceptLeaf(), 00523 filter_id, name, kind); 00524 } 00525 00528 OutputResourcePtr CreateOutputResourceWithPath( 00529 const StringPiece& path, const StringPiece& filter_id, 00530 const StringPiece& name, OutputResourceKind kind) { 00531 return CreateOutputResourceWithPath(path, path, path, filter_id, name, 00532 kind); 00533 } 00534 00538 ResourcePtr CreateInputResource(const GoogleUrl& input_url); 00539 00543 ResourcePtr CreateInputResourceAbsoluteUnchecked( 00544 const StringPiece& absolute_url); 00545 00550 bool MatchesBaseUrl(const GoogleUrl& input_url) const; 00551 00555 bool MayRewriteUrl(const GoogleUrl& domain_url, 00556 const GoogleUrl& input_url) const; 00557 00561 const GoogleUrl& base_url() const { return base_url_; } 00562 00564 StringPiece fetch_url() const { return fetch_url_; } 00565 00569 const GoogleUrl& decoded_base_url() const { return decoded_base_url_; } 00570 StringPiece decoded_base() const { return decoded_base_url_.Spec(); } 00571 00573 bool IsHttps() const { return google_url().SchemeIs("https"); } 00574 00575 const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; } 00576 00578 RewriteFilter* FindFilter(const StringPiece& id) const; 00579 00581 bool refs_before_base() { return refs_before_base_; } 00582 00587 void set_refs_before_base() { refs_before_base_ = true; } 00588 00593 StringPiece containing_charset() { return containing_charset_; } 00594 void set_containing_charset(const StringPiece charset) { 00595 charset.CopyToString(&containing_charset_); 00596 } 00597 00599 HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource, 00600 HtmlElement* elt, 00601 HtmlElement::Attribute* attr); 00602 00607 bool InitiateRewrite(RewriteContext* rewrite_context); 00608 void InitiateFetch(RewriteContext* rewrite_context); 00609 00620 void RewriteComplete(RewriteContext* rewrite_context, bool permit_render); 00621 00625 void ReportSlowRewrites(int num); 00626 00631 void Cleanup(); 00632 00634 GoogleString ToString(bool show_detached_contexts); 00635 void PrintState(bool show_detached_contexts); 00636 void PrintStateToErrorLog(bool show_detached_contexts); 00637 00640 void WaitForCompletion(); 00641 00648 void WaitForShutDown(); 00649 00653 void BoundedWaitFor(WaitMode mode, int64 timeout_ms); 00654 00662 void set_fully_rewrite_on_flush(bool x) { 00663 fully_rewrite_on_flush_ = x; 00664 } 00665 00667 bool fully_rewrite_on_flush() const { 00668 return fully_rewrite_on_flush_; 00669 } 00670 00674 void set_fast_blocking_rewrite(bool x) { 00675 fast_blocking_rewrite_ = x; 00676 } 00677 00678 bool fast_blocking_rewrite() const { 00679 return fast_blocking_rewrite_; 00680 } 00681 00684 void EnableBlockingRewrite(RequestHeaders* request_headers); 00685 00692 void set_externally_managed(bool x) { externally_managed_ = x; } 00693 00697 void DetachFetch(); 00698 00701 void DetachedFetchComplete(); 00702 00706 void FetchComplete(); 00707 00713 void DeleteRewriteContext(RewriteContext* rewrite_context); 00714 00715 int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); } 00716 00721 void set_max_page_processing_delay_ms(int x) { 00722 max_page_processing_delay_ms_ = x; 00723 } 00724 int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; } 00725 00727 void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; } 00728 UserAgentMatcher::DeviceType device_type() { return device_type_; } 00729 00735 RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key, 00736 RewriteContext* candidate); 00737 00742 void DeregisterForPartitionKey( 00743 const GoogleString& partition_key, RewriteContext* candidate); 00744 00747 void RequestFlush() { flush_requested_ = true; } 00748 bool flush_requested() const { return flush_requested_; } 00749 00761 void ExecuteFlushIfRequested(); 00762 00766 void ExecuteFlushIfRequestedAsync(Function* callback); 00767 00776 virtual void Flush(); 00777 00781 void FlushAsync(Function* done); 00782 00784 void AddRewriteTask(Function* task); 00785 00788 void AddLowPriorityRewriteTask(Function* task); 00789 00790 QueuedWorkerPool::Sequence* html_worker() { return html_worker_; } 00791 QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; } 00792 QueuedWorkerPool::Sequence* low_priority_rewrite_worker() { 00793 return low_priority_rewrite_worker_; 00794 } 00795 00796 Scheduler* scheduler() { return scheduler_; } 00797 00800 DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); } 00801 UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); } 00802 00810 CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base, 00811 const StringPiece& output_css_base, 00812 const StringPiece& contents, 00813 Writer* writer, 00814 MessageHandler* handler); 00815 00823 bool ShouldAbsolutifyUrl(const GoogleUrl& input_base, 00824 const GoogleUrl& output_base, 00825 bool* proxy_mode) const; 00826 00834 void UpdatePropertyValueInDomCohort( 00835 AbstractPropertyPage* page, 00836 StringPiece property_name, 00837 StringPiece property_value); 00838 00839 void set_client_id(const StringPiece& id) { client_id_ = id.as_string(); } 00840 const GoogleString& client_id() const { return client_id_; } 00841 00844 PropertyPage* property_page() const; 00849 FallbackPropertyPage* fallback_property_page() const { 00850 return fallback_property_page_; 00851 } 00853 void set_property_page(PropertyPage* page); 00855 void set_fallback_property_page(FallbackPropertyPage* page); 00857 void set_unowned_fallback_property_page(FallbackPropertyPage* page); 00858 00860 const CriticalLineInfo* critical_line_info() const; 00861 00864 void set_critical_line_info(CriticalLineInfo* critical_line_info); 00865 00866 CriticalCssResult* critical_css_result() const; 00869 void set_critical_css_result(CriticalCssResult* critical_css_rules); 00870 00872 CriticalImagesInfo* critical_images_info() const { 00873 return critical_images_info_.get(); 00874 } 00875 00880 CriticalSelectorInfo* critical_selector_info() { 00881 return critical_selector_info_.get(); 00882 } 00883 00887 void set_critical_selector_info(CriticalSelectorInfo* info) { 00888 critical_selector_info_.reset(info); 00889 } 00890 00895 void set_critical_images_info(CriticalImagesInfo* critical_images_info) { 00896 critical_images_info_.reset(critical_images_info); 00897 } 00898 00901 bool CriticalSelectorsEnabled() const { 00902 return (options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) && 00903 server_context()->factory()->UseSelectorFilterForCriticalCss()); 00904 } 00905 00908 bool FlattenCssImportsEnabled() const { 00909 return (options()->Enabled(RewriteOptions::kFlattenCssImports) || 00910 (!options()->Forbidden(RewriteOptions::kFlattenCssImports) && 00911 (CriticalSelectorsEnabled() || 00912 options()->Enabled(RewriteOptions::kComputeCriticalCss)))); 00913 } 00914 00918 int num_inline_preview_images() const { return num_inline_preview_images_; } 00919 00921 void increment_num_inline_preview_images(); 00922 00925 int num_flushed_early_pagespeed_resources() const { 00926 return num_flushed_early_pagespeed_resources_; 00927 } 00928 00931 void increment_num_flushed_early_pagespeed_resources() { 00932 ++num_flushed_early_pagespeed_resources_; 00933 } 00934 00938 void increment_async_events_count(); 00939 00941 void decrement_async_events_count(); 00942 00945 XhtmlStatus MimeTypeXhtmlStatus(); 00946 00947 void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; } 00948 bool flushed_cached_html() { return flushed_cached_html_; } 00949 00950 void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; } 00951 bool flushing_cached_html() const { return flushing_cached_html_; } 00952 00953 void set_flushed_early(bool x) { flushed_early_ = x; } 00954 bool flushed_early() const { return flushed_early_; } 00955 00956 void set_flushing_early(bool x) { flushing_early_ = x; } 00957 bool flushing_early() const { return flushing_early_; } 00958 00959 void set_is_lazyload_script_flushed(bool x) { 00960 is_lazyload_script_flushed_ = x; 00961 } 00962 bool is_lazyload_script_flushed() const { 00963 return is_lazyload_script_flushed_; } 00964 00966 FlushEarlyInfo* flush_early_info(); 00967 00968 FlushEarlyRenderInfo* flush_early_render_info() const; 00969 00972 void set_flush_early_render_info( 00973 FlushEarlyRenderInfo* flush_early_render_info); 00974 00975 void set_serve_blink_non_critical(bool x) { serve_blink_non_critical_ = x; } 00976 bool serve_blink_non_critical() const { return serve_blink_non_critical_; } 00977 00978 void set_is_blink_request(bool x) { is_blink_request_ = x; } 00979 bool is_blink_request() const { return is_blink_request_; } 00980 00983 bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); } 00984 00987 void SaveOriginalHeaders(const ResponseHeaders& response_headers); 00988 00991 AbstractLogRecord* log_record(); 00992 00993 DomStatsFilter* dom_stats_filter() const { 00994 return dom_stats_filter_; 00995 } 00996 00999 bool can_rewrite_resources() const { return can_rewrite_resources_; } 01000 01002 bool is_nested() const { return is_nested_; } 01003 01007 bool MetadataRequested(const RequestHeaders& request_headers) const; 01008 01021 bool Write(const ResourceVector& inputs, 01022 const StringPiece& contents, 01023 const ContentType* type, 01024 StringPiece charset, 01025 OutputResource* output); 01026 01027 protected: 01028 virtual void DetermineEnabledFilters(); 01029 01030 private: 01031 friend class RewriteContext; 01032 friend class RewriteDriverTest; 01033 friend class RewriteTestBase; 01034 friend class ServerContextTest; 01035 01036 typedef std::map<GoogleString, RewriteFilter*> StringFilterMap; 01037 01042 void FetchCompleteImpl(bool signal, ScopedMutex* lock); 01043 01048 void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms, 01049 Function* done); 01050 01054 void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms, 01055 Function* done); 01056 01058 bool IsDone(WaitMode wait_mode, bool deadline_reached); 01059 01062 bool WaitForPendingAsyncEvents(WaitMode wait_mode) { 01063 return wait_mode == kWaitForShutDown || 01064 (fully_rewrite_on_flush_ && !fast_blocking_rewrite_); 01065 } 01066 01070 void FlushAsyncDone(int num_rewrites, Function* callback); 01071 01076 int64 ComputeCurrentFlushWindowRewriteDelayMs(); 01077 01079 void QueueFlushAsyncDone(int num_rewrites, Function* callback); 01080 01083 void QueueFinishParseAfterFlush(Function* user_callback); 01084 void FinishParseAfterFlush(Function* user_callback); 01085 01087 bool RewritesComplete() const; 01088 01093 bool HaveBackgroundFetchRewrite() const; 01094 01097 void SetBaseUrlIfUnset(const StringPiece& new_base); 01098 01101 void SetBaseUrlForFetch(const StringPiece& url); 01102 01105 void SetDecodedUrlFromBase(); 01106 01108 AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); } 01109 01111 virtual void ParseTextInternal(const char* content, int size); 01112 01114 bool ShouldSkipParsing(); 01115 01116 friend class ScanFilter; 01117 01121 void AddCommonFilter(CommonFilter* filter); 01122 01126 void RegisterRewriteFilter(RewriteFilter* filter); 01127 01132 void EnableRewriteFilter(const char* id); 01133 01137 ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl); 01138 01139 void AddPreRenderFilters(); 01140 void AddPostRenderFilters(); 01141 01143 bool DecodeOutputResourceNameHelper(const GoogleUrl& url, 01144 ResourceNamer* name_out, 01145 OutputResourceKind* kind_out, 01146 RewriteFilter** filter_out, 01147 GoogleString* url_base, 01148 StringVector* urls) const; 01149 01159 void WriteDomCohortIntoPropertyCache(); 01160 01161 void FinalizeFilterLogging(); 01162 01164 CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher); 01165 01172 void PossiblyPurgeCachedResponseAndReleaseDriver(); 01173 01178 bool ShouldPurgeRewrittenResponse(); 01179 01182 static bool GetPurgeUrl(const GoogleUrl& google_url, 01183 const RewriteOptions* options, 01184 GoogleString* purge_url, 01185 GoogleString* purge_method); 01186 01188 void PurgeDownstreamCache(const GoogleString& purge_url, 01189 const GoogleString& purge_method); 01190 01192 void LogStats(); 01193 01205 bool base_was_set_; 01206 01211 bool refs_before_base_; 01212 01214 GoogleString containing_charset_; 01215 01216 bool filters_added_; 01217 bool externally_managed_; 01218 01222 bool fetch_queued_; 01223 01228 bool fetch_detached_; 01229 01236 bool detached_fetch_main_path_complete_; 01237 bool detached_fetch_detached_path_complete_; 01238 01241 bool parsing_; 01242 01246 WaitMode waiting_; 01247 01252 bool fully_rewrite_on_flush_; 01253 01256 bool fast_blocking_rewrite_; 01257 01260 bool cleanup_on_fetch_complete_; 01261 01262 bool flush_requested_; 01263 bool flush_occurred_; 01264 01266 bool flushed_cached_html_; 01267 01269 bool flushing_cached_html_; 01270 01273 bool flushed_early_; 01277 bool flushing_early_; 01278 01281 bool is_lazyload_script_flushed_; 01282 01284 bool release_driver_; 01285 01288 bool made_downstream_purge_attempt_; 01289 01293 bool write_property_cache_dom_cohort_; 01294 01302 int rewrites_to_delete_; 01303 01306 GoogleUrl base_url_; 01307 01311 GoogleUrl decoded_base_url_; 01312 01315 GoogleString fetch_url_; 01316 01317 GoogleString user_agent_; 01318 01319 LazyBool should_skip_parsing_; 01320 01321 StringFilterMap resource_filter_map_; 01322 01323 ResponseHeaders* response_headers_; 01324 01327 scoped_ptr<const RequestHeaders> request_headers_; 01328 01329 int status_code_; 01330 01333 typedef std::vector<RewriteContext*> RewriteContextVector; 01334 RewriteContextVector rewrites_; 01335 01338 int max_page_processing_delay_ms_; 01339 01340 typedef std::set<RewriteContext*> RewriteContextSet; 01341 01346 RewriteContextSet initiated_rewrites_; 01347 01349 int64 num_initiated_rewrites_; 01350 01358 RewriteContextSet detached_rewrites_; 01359 01362 int64 num_detached_rewrites_; 01363 01368 int pending_rewrites_; 01369 01371 int possibly_quick_rewrites_; 01372 01377 int pending_async_events_; 01378 01381 FileSystem* file_system_; 01382 ServerContext* server_context_; 01383 Scheduler* scheduler_; 01384 UrlAsyncFetcher* default_url_async_fetcher_; 01385 01389 UrlAsyncFetcher* url_async_fetcher_; 01390 01394 UrlAsyncFetcher* distributed_async_fetcher_; 01395 01398 std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_; 01399 01400 AddInstrumentationFilter* add_instrumentation_filter_; 01401 DomStatsFilter* dom_stats_filter_; 01402 scoped_ptr<HtmlWriterFilter> html_writer_filter_; 01403 01404 ScanFilter scan_filter_; 01405 scoped_ptr<DomainRewriteFilter> domain_rewriter_; 01406 scoped_ptr<UrlLeftTrimFilter> url_trim_filter_; 01407 01410 typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap; 01411 PrimaryRewriteContextMap primary_rewrite_context_map_; 01412 01413 HtmlResourceSlotSet slots_; 01414 01415 scoped_ptr<RewriteOptions> options_; 01416 01417 RewriteDriverPool* controlling_pool_; 01418 01420 scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks> 01421 cache_url_async_fetcher_async_op_hooks_; 01422 01424 UrlSegmentEncoder default_encoder_; 01425 01427 FilterList early_pre_render_filters_; 01429 FilterList pre_render_filters_; 01430 01434 FilterVector filters_to_delete_; 01435 01436 QueuedWorkerPool::Sequence* html_worker_; 01437 QueuedWorkerPool::Sequence* rewrite_worker_; 01438 QueuedWorkerPool::Sequence* low_priority_rewrite_worker_; 01439 01440 Writer* writer_; 01441 01443 GoogleString client_id_; 01444 01447 FallbackPropertyPage* fallback_property_page_; 01448 01450 bool owns_property_page_; 01451 01453 UserAgentMatcher::DeviceType device_type_; 01454 01455 scoped_ptr<CriticalLineInfo> critical_line_info_; 01456 01459 scoped_ptr<CriticalImagesInfo> critical_images_info_; 01460 scoped_ptr<CriticalSelectorInfo> critical_selector_info_; 01461 01462 scoped_ptr<CriticalCssResult> critical_css_result_; 01463 01465 bool xhtml_mimetype_computed_; 01466 XhtmlStatus xhtml_status_ : 8; 01467 01470 int num_inline_preview_images_; 01471 01473 int num_flushed_early_pagespeed_resources_; 01474 01476 int num_bytes_in_; 01477 01478 DebugFilter* debug_filter_; 01479 01480 scoped_ptr<FlushEarlyInfo> flush_early_info_; 01481 scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_; 01482 01486 bool serve_blink_non_critical_; 01488 bool is_blink_request_; 01489 bool can_rewrite_resources_; 01490 bool is_nested_; 01491 01494 RequestContextPtr request_context_; 01495 01497 int64 start_time_ms_; 01498 01499 scoped_ptr<RequestProperties> request_properties_; 01500 01504 static int initialized_count_; 01505 01506 DISALLOW_COPY_AND_ASSIGN(RewriteDriver); 01507 }; 01508 01511 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback { 01512 public: 01513 virtual ~OptionsAwareHTTPCacheCallback(); 01514 virtual bool IsCacheValid(const GoogleString& key, 01515 const ResponseHeaders& headers); 01516 virtual int64 OverrideCacheTtlMs(const GoogleString& key); 01517 protected: 01520 OptionsAwareHTTPCacheCallback( 01521 const RewriteOptions* rewrite_options, 01522 const RequestContextPtr& request_ctx); 01523 01524 private: 01525 const RewriteOptions* rewrite_options_; 01526 01527 DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback); 01528 }; 01529 01530 } 01531 01532 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_