Page Speed Optimization Libraries
1.7.30.1
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00021 00022 #include <map> 00023 #include <set> 00024 #include <vector> 00025 00026 #include "base/logging.h" 00027 #include "net/instaweb/htmlparse/public/html_element.h" 00028 #include "net/instaweb/htmlparse/public/html_parse.h" 00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h" 00030 #include "net/instaweb/http/public/http_cache.h" 00031 #include "net/instaweb/http/public/request_context.h" 00032 #include "net/instaweb/http/public/user_agent_matcher.h" 00033 #include "net/instaweb/rewriter/public/critical_images_finder.h" 00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h" 00035 #include "net/instaweb/rewriter/public/output_resource_kind.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/resource_slot.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/scan_filter.h" 00040 #include "net/instaweb/rewriter/public/server_context.h" 00041 #include "net/instaweb/util/public/basictypes.h" 00042 #include "net/instaweb/util/public/google_url.h" 00043 #include "net/instaweb/util/public/printf_format.h" 00044 #include "net/instaweb/util/public/queued_worker_pool.h" 00045 #include "net/instaweb/util/public/scheduler.h" 00046 #include "net/instaweb/util/public/scoped_ptr.h" 00047 #include "net/instaweb/util/public/string.h" 00048 #include "net/instaweb/util/public/string_util.h" 00049 #include "net/instaweb/util/public/thread_system.h" 00050 #include "net/instaweb/util/public/url_segment_encoder.h" 00051 #include "pagespeed/kernel/http/content_type.h" 00052 #include "pagespeed/kernel/util/categorized_refcount.h" 00053 00054 namespace net_instaweb { 00055 00056 class AbstractLogRecord; 00057 class AbstractMutex; 00058 class AbstractPropertyPage; 00059 class AddInstrumentationFilter; 00060 class AsyncFetch; 00061 class CommonFilter; 00062 class CriticalCssResult; 00063 class CriticalLineInfo; 00064 class DebugFilter; 00065 class DomStatsFilter; 00066 class DomainRewriteFilter; 00067 class FallbackPropertyPage; 00068 class FileSystem; 00069 class FlushEarlyInfo; 00070 class FlushEarlyRenderInfo; 00071 class Function; 00072 class HtmlFilter; 00073 class HtmlWriterFilter; 00074 class MessageHandler; 00075 class OutputResource; 00076 class PropertyPage; 00077 class RequestHeaders; 00078 class RequestProperties; 00079 class RequestTrace; 00080 class ResourceContext; 00081 class ResourceNamer; 00082 class ResponseHeaders; 00083 class RewriteContext; 00084 class RewriteDriverPool; 00085 class RewriteFilter; 00086 class SplitHtmlConfig; 00087 class Statistics; 00088 class UrlAsyncFetcher; 00089 class UrlLeftTrimFilter; 00090 class Writer; 00091 00094 class RewriteDriver : public HtmlParse { 00095 public: 00097 enum CssResolutionStatus { 00098 kWriteFailed, 00099 kNoResolutionNeeded, 00100 kSuccess 00101 }; 00102 00104 enum WaitMode { 00105 kNoWait, 00106 kWaitForCompletion, 00107 kWaitForCachedRender, 00108 00109 kWaitForShutDown 00110 00111 }; 00112 00124 enum XhtmlStatus { 00125 kXhtmlUnknown, 00126 kIsXhtml, 00127 kIsNotXhtml 00128 }; 00129 00136 static const char* kPassThroughRequestAttributes[7]; 00137 00141 static const char kDomCohort[]; 00143 static const char kBeaconCohort[]; 00144 00147 static const char kLastRequestTimestamp[]; 00149 static const char kParseSizeLimitExceeded[]; 00151 static const char kSubresourcesPropertyName[]; 00153 static const char kStatusCodePropertyName[]; 00154 00155 RewriteDriver(MessageHandler* message_handler, 00156 FileSystem* file_system, 00157 UrlAsyncFetcher* url_async_fetcher); 00158 00161 virtual ~RewriteDriver(); 00162 00165 RewriteDriver* Clone(); 00166 00172 void Clear(); 00173 00175 static void InitStats(Statistics* statistics); 00176 00178 static void Initialize(); 00179 static void Terminate(); 00180 00183 void SetServerContext(ServerContext* server_context); 00184 00187 bool MayCacheExtendCss() const; 00188 bool MayCacheExtendImages() const; 00189 bool MayCacheExtendPdfs() const; 00190 bool MayCacheExtendScripts() const; 00191 00192 const GoogleString& user_agent() const { return user_agent_; } 00193 00194 void SetUserAgent(const StringPiece& user_agent_string); 00195 00196 const RequestProperties* request_properties() const { 00197 return request_properties_.get(); 00198 } 00199 00201 void ClearRequestProperties(); 00202 00204 bool using_spdy() const { return request_context_->using_spdy(); } 00205 00206 bool write_property_cache_dom_cohort() const { 00207 return write_property_cache_dom_cohort_; 00208 } 00209 void set_write_property_cache_dom_cohort(bool x) { 00210 write_property_cache_dom_cohort_ = x; 00211 } 00212 00213 RequestContextPtr request_context() { return request_context_; } 00214 void set_request_context(const RequestContextPtr& x); 00215 00218 RequestTrace* trace_context(); 00219 00222 void TracePrintf(const char* fmt, ...); 00223 00226 ResponseHeaders* mutable_response_headers() { 00227 return flush_occurred_ ? NULL : response_headers_; 00228 } 00229 00236 const ResponseHeaders* response_headers() { 00237 return response_headers_; 00238 } 00239 00243 void set_response_headers_ptr(ResponseHeaders* headers) { 00244 response_headers_ = headers; 00245 } 00246 00252 void SetRequestHeaders(const RequestHeaders& headers); 00253 00254 const RequestHeaders* request_headers() const { 00255 return request_headers_.get(); 00256 } 00257 00258 UserAgentMatcher* user_agent_matcher() const { 00259 DCHECK(server_context() != NULL); 00260 return server_context()->user_agent_matcher(); 00261 } 00262 00267 void AddFilters(); 00268 00272 void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter); 00273 00275 void PrependOwnedPreRenderFilter(HtmlFilter* filter); 00277 void AppendOwnedPreRenderFilter(HtmlFilter* filter); 00278 00280 void AddOwnedPostRenderFilter(HtmlFilter* filter); 00282 void AddUnownedPostRenderFilter(HtmlFilter* filter); 00283 00295 void AppendRewriteFilter(RewriteFilter* filter); 00296 00299 void PrependRewriteFilter(RewriteFilter* filter); 00300 00306 void SetWriter(Writer* writer); 00307 00308 Writer* writer() const { return writer_; } 00309 00338 bool FetchResource(const StringPiece& url, AsyncFetch* fetch); 00339 00357 void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode, 00358 AsyncFetch* async_fetch); 00359 00366 bool FetchOutputResource(const OutputResourcePtr& output_resource, 00367 RewriteFilter* filter, 00368 AsyncFetch* async_fetch); 00369 00376 OutputResourcePtr DecodeOutputResource(const GoogleUrl& url, 00377 RewriteFilter** filter) const; 00378 00382 bool DecodeOutputResourceName(const GoogleUrl& url, 00383 ResourceNamer* name_out, 00384 OutputResourceKind* kind_out, 00385 RewriteFilter** filter_out) const; 00386 00388 bool DecodeUrl(const GoogleUrl& url, 00389 StringVector* decoded_urls) const; 00390 00391 FileSystem* file_system() { return file_system_; } 00392 UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; } 00393 00398 void SetSessionFetcher(UrlAsyncFetcher* f); 00399 00400 UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; } 00402 void set_distributed_fetcher(UrlAsyncFetcher* fetcher) { 00403 distributed_async_fetcher_ = fetcher; 00404 } 00405 00408 CacheUrlAsyncFetcher* CreateCacheFetcher(); 00410 CacheUrlAsyncFetcher* CreateCacheOnlyFetcher(); 00411 00412 ServerContext* server_context() const { return server_context_; } 00413 Statistics* statistics() const; 00414 00415 AddInstrumentationFilter* add_instrumentation_filter() { 00416 return add_instrumentation_filter_; 00417 } 00418 00420 void set_custom_options(RewriteOptions* options) { 00421 set_options_for_pool(NULL, options); 00422 } 00423 00426 void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) { 00427 controlling_pool_ = pool; 00428 options_.reset(options); 00429 } 00430 00432 RewriteDriverPool* controlling_pool() { return controlling_pool_; } 00433 00435 const RewriteOptions* options() const { return options_.get(); } 00436 00440 virtual bool StartParseId(const StringPiece& url, const StringPiece& id, 00441 const ContentType& content_type); 00442 00449 virtual void FinishParse(); 00450 00453 void FinishParseAsync(Function* callback); 00454 00458 void InfoAt(const RewriteContext* context, 00459 const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4); 00460 00465 00472 OutputResourcePtr CreateOutputResourceFromResource( 00473 const StringPiece& filter_id, 00474 const UrlSegmentEncoder* encoder, 00475 const ResourceContext* data, 00476 const ResourcePtr& input_resource, 00477 OutputResourceKind kind); 00478 00495 OutputResourcePtr CreateOutputResourceWithPath( 00496 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00497 const StringPiece& base_url, const StringPiece& filter_id, 00498 const StringPiece& name, OutputResourceKind kind); 00499 00502 void PopulateResourceNamer( 00503 const StringPiece& filter_id, 00504 const StringPiece& name, 00505 ResourceNamer* full_name); 00506 00510 OutputResourcePtr CreateOutputResourceWithUnmappedUrl( 00511 const GoogleUrl& unmapped_gurl, const StringPiece& filter_id, 00512 const StringPiece& name, OutputResourceKind kind); 00513 00516 OutputResourcePtr CreateOutputResourceWithMappedPath( 00517 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00518 const StringPiece& filter_id, const StringPiece& name, 00519 OutputResourceKind kind) { 00520 return CreateOutputResourceWithPath(mapped_path, unmapped_path, 00521 decoded_base_url_.AllExceptLeaf(), 00522 filter_id, name, kind); 00523 } 00524 00527 OutputResourcePtr CreateOutputResourceWithPath( 00528 const StringPiece& path, const StringPiece& filter_id, 00529 const StringPiece& name, OutputResourceKind kind) { 00530 return CreateOutputResourceWithPath(path, path, path, filter_id, name, 00531 kind); 00532 } 00533 00537 ResourcePtr CreateInputResource(const GoogleUrl& input_url); 00538 00542 ResourcePtr CreateInputResourceAbsoluteUnchecked( 00543 const StringPiece& absolute_url); 00544 00549 bool MatchesBaseUrl(const GoogleUrl& input_url) const; 00550 00554 bool MayRewriteUrl(const GoogleUrl& domain_url, 00555 const GoogleUrl& input_url) const; 00556 00560 const GoogleUrl& base_url() const { return base_url_; } 00561 00563 StringPiece fetch_url() const { return fetch_url_; } 00564 00568 const GoogleUrl& decoded_base_url() const { return decoded_base_url_; } 00569 StringPiece decoded_base() const { return decoded_base_url_.Spec(); } 00570 00572 bool IsHttps() const { return google_url().SchemeIs("https"); } 00573 00574 const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; } 00575 00577 RewriteFilter* FindFilter(const StringPiece& id) const; 00578 00580 bool refs_before_base() { return refs_before_base_; } 00581 00586 void set_refs_before_base() { refs_before_base_ = true; } 00587 00592 StringPiece containing_charset() { return containing_charset_; } 00593 void set_containing_charset(const StringPiece charset) { 00594 charset.CopyToString(&containing_charset_); 00595 } 00596 00598 HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource, 00599 HtmlElement* elt, 00600 HtmlElement::Attribute* attr); 00601 00606 bool InitiateRewrite(RewriteContext* rewrite_context); 00607 void InitiateFetch(RewriteContext* rewrite_context); 00608 00619 void RewriteComplete(RewriteContext* rewrite_context, bool permit_render); 00620 00624 void ReportSlowRewrites(int num); 00625 00630 void Cleanup(); 00631 00636 void AddUserReference(); 00637 00639 GoogleString ToString(bool show_detached_contexts); 00640 void PrintState(bool show_detached_contexts); 00641 void PrintStateToErrorLog(bool show_detached_contexts); 00642 00645 void WaitForCompletion(); 00646 00653 void WaitForShutDown(); 00654 00658 void BoundedWaitFor(WaitMode mode, int64 timeout_ms); 00659 00667 void set_fully_rewrite_on_flush(bool x) { 00668 fully_rewrite_on_flush_ = x; 00669 } 00670 00672 bool fully_rewrite_on_flush() const { 00673 return fully_rewrite_on_flush_; 00674 } 00675 00679 void set_fast_blocking_rewrite(bool x) { 00680 fast_blocking_rewrite_ = x; 00681 } 00682 00683 bool fast_blocking_rewrite() const { 00684 return fast_blocking_rewrite_; 00685 } 00686 00689 void EnableBlockingRewrite(RequestHeaders* request_headers); 00690 00697 void set_externally_managed(bool x) { externally_managed_ = x; } 00698 00702 void DetachFetch(); 00703 00706 void DetachedFetchComplete(); 00707 00711 void FetchComplete(); 00712 00718 void DeleteRewriteContext(RewriteContext* rewrite_context); 00719 00720 int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); } 00721 00726 void set_max_page_processing_delay_ms(int x) { 00727 max_page_processing_delay_ms_ = x; 00728 } 00729 int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; } 00730 00732 void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; } 00733 UserAgentMatcher::DeviceType device_type() { return device_type_; } 00734 00740 RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key, 00741 RewriteContext* candidate); 00742 00747 void DeregisterForPartitionKey( 00748 const GoogleString& partition_key, RewriteContext* candidate); 00749 00752 void RequestFlush() { flush_requested_ = true; } 00753 bool flush_requested() const { return flush_requested_; } 00754 00766 void ExecuteFlushIfRequested(); 00767 00771 void ExecuteFlushIfRequestedAsync(Function* callback); 00772 00781 virtual void Flush(); 00782 00786 void FlushAsync(Function* done); 00787 00789 void AddRewriteTask(Function* task); 00790 00793 void AddLowPriorityRewriteTask(Function* task); 00794 00795 QueuedWorkerPool::Sequence* html_worker() { return html_worker_; } 00796 QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; } 00797 QueuedWorkerPool::Sequence* low_priority_rewrite_worker() { 00798 return low_priority_rewrite_worker_; 00799 } 00800 00801 Scheduler* scheduler() { return scheduler_; } 00802 00805 DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); } 00806 UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); } 00807 00815 CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base, 00816 const StringPiece& output_css_base, 00817 const StringPiece& contents, 00818 Writer* writer, 00819 MessageHandler* handler); 00820 00828 bool ShouldAbsolutifyUrl(const GoogleUrl& input_base, 00829 const GoogleUrl& output_base, 00830 bool* proxy_mode) const; 00831 00839 void UpdatePropertyValueInDomCohort( 00840 AbstractPropertyPage* page, 00841 StringPiece property_name, 00842 StringPiece property_value); 00843 00844 void set_client_id(const StringPiece& id) { client_id_ = id.as_string(); } 00845 const GoogleString& client_id() const { return client_id_; } 00846 00849 PropertyPage* property_page() const; 00854 FallbackPropertyPage* fallback_property_page() const { 00855 return fallback_property_page_; 00856 } 00858 void set_property_page(PropertyPage* page); 00860 void set_fallback_property_page(FallbackPropertyPage* page); 00862 void set_unowned_fallback_property_page(FallbackPropertyPage* page); 00863 00865 const CriticalLineInfo* critical_line_info() const; 00866 00869 void set_critical_line_info(CriticalLineInfo* critical_line_info); 00870 00871 const SplitHtmlConfig* split_html_config(); 00872 00873 CriticalCssResult* critical_css_result() const; 00876 void set_critical_css_result(CriticalCssResult* critical_css_rules); 00877 00879 CriticalImagesInfo* critical_images_info() const { 00880 return critical_images_info_.get(); 00881 } 00882 00887 CriticalSelectorInfo* critical_selector_info() { 00888 return critical_selector_info_.get(); 00889 } 00890 00894 void set_critical_selector_info(CriticalSelectorInfo* info) { 00895 critical_selector_info_.reset(info); 00896 } 00897 00902 void set_critical_images_info(CriticalImagesInfo* critical_images_info) { 00903 critical_images_info_.reset(critical_images_info); 00904 } 00905 00908 bool CriticalSelectorsEnabled() const; 00909 00912 bool FlattenCssImportsEnabled() const { 00913 return (options()->Enabled(RewriteOptions::kFlattenCssImports) || 00914 (!options()->Forbidden(RewriteOptions::kFlattenCssImports) && 00915 (CriticalSelectorsEnabled() || 00916 options()->Enabled(RewriteOptions::kComputeCriticalCss)))); 00917 } 00918 00922 int num_inline_preview_images() const { return num_inline_preview_images_; } 00923 00925 void increment_num_inline_preview_images(); 00926 00929 int num_flushed_early_pagespeed_resources() const { 00930 return num_flushed_early_pagespeed_resources_; 00931 } 00932 00935 void increment_num_flushed_early_pagespeed_resources() { 00936 ++num_flushed_early_pagespeed_resources_; 00937 } 00938 00941 void increment_async_events_count(); 00942 00944 void decrement_async_events_count(); 00945 00948 XhtmlStatus MimeTypeXhtmlStatus(); 00949 00950 void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; } 00951 bool flushed_cached_html() { return flushed_cached_html_; } 00952 00953 void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; } 00954 bool flushing_cached_html() const { return flushing_cached_html_; } 00955 00956 void set_flushed_early(bool x) { flushed_early_ = x; } 00957 bool flushed_early() const { return flushed_early_; } 00958 00959 void set_flushing_early(bool x) { flushing_early_ = x; } 00960 bool flushing_early() const { return flushing_early_; } 00961 00962 void set_is_lazyload_script_flushed(bool x) { 00963 is_lazyload_script_flushed_ = x; 00964 } 00965 bool is_lazyload_script_flushed() const { 00966 return is_lazyload_script_flushed_; } 00967 00969 FlushEarlyInfo* flush_early_info(); 00970 00971 FlushEarlyRenderInfo* flush_early_render_info() const; 00972 00975 void set_flush_early_render_info( 00976 FlushEarlyRenderInfo* flush_early_render_info); 00977 00978 void set_is_blink_request(bool x) { is_blink_request_ = x; } 00979 bool is_blink_request() const { return is_blink_request_; } 00980 00983 bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); } 00984 00987 void SaveOriginalHeaders(const ResponseHeaders& response_headers); 00988 00991 AbstractLogRecord* log_record(); 00992 00993 DomStatsFilter* dom_stats_filter() const { 00994 return dom_stats_filter_; 00995 } 00996 00999 bool can_rewrite_resources() const { return can_rewrite_resources_; } 01000 01002 bool is_nested() const { return is_nested_; } 01003 01007 bool MetadataRequested(const RequestHeaders& request_headers) const; 01008 01010 bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; } 01011 01024 bool Write(const ResourceVector& inputs, 01025 const StringPiece& contents, 01026 const ContentType* type, 01027 StringPiece charset, 01028 OutputResource* output); 01029 01030 void set_defer_instrumentation_script(bool x) { 01031 defer_instrumentation_script_ = x; 01032 } 01033 bool defer_instrumentation_script() const { 01034 return defer_instrumentation_script_; 01035 } 01036 01037 protected: 01038 virtual void DetermineEnabledFiltersImpl(); 01039 01040 private: 01041 friend class DistributedRewriteContextTest; 01042 friend class RewriteContext; 01043 friend class RewriteDriverTest; 01044 friend class RewriteTestBase; 01045 friend class ServerContextTest; 01046 01047 typedef std::map<GoogleString, RewriteFilter*> StringFilterMap; 01048 01050 bool ShouldDistributeFetch(const StringPiece& filter_id); 01051 01063 bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id, 01064 AsyncFetch* async_fetch); 01065 01070 void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms, 01071 Function* done); 01072 01076 void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms, 01077 Function* done); 01078 01080 bool IsDone(WaitMode wait_mode, bool deadline_reached); 01081 01084 bool WaitForPendingAsyncEvents(WaitMode wait_mode) { 01085 return wait_mode == kWaitForShutDown || 01086 (fully_rewrite_on_flush_ && !fast_blocking_rewrite_); 01087 } 01088 01092 void FlushAsyncDone(int num_rewrites, Function* callback); 01093 01098 int64 ComputeCurrentFlushWindowRewriteDelayMs(); 01099 01101 void QueueFlushAsyncDone(int num_rewrites, Function* callback); 01102 01105 void QueueFinishParseAfterFlush(Function* user_callback); 01106 void FinishParseAfterFlush(Function* user_callback); 01107 01109 bool RewritesComplete() const; 01110 01113 void SetBaseUrlIfUnset(const StringPiece& new_base); 01114 01117 void SetBaseUrlForFetch(const StringPiece& url); 01118 01121 void SetDecodedUrlFromBase(); 01122 01124 AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); } 01125 01127 virtual void ParseTextInternal(const char* content, int size); 01128 01130 bool ShouldSkipParsing(); 01131 01132 friend class ScanFilter; 01133 01137 void AddCommonFilter(CommonFilter* filter); 01138 01142 void RegisterRewriteFilter(RewriteFilter* filter); 01143 01148 void EnableRewriteFilter(const char* id); 01149 01153 ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl); 01154 01155 void AddPreRenderFilters(); 01156 void AddPostRenderFilters(); 01157 01159 bool DecodeOutputResourceNameHelper(const GoogleUrl& url, 01160 ResourceNamer* name_out, 01161 OutputResourceKind* kind_out, 01162 RewriteFilter** filter_out, 01163 GoogleString* url_base, 01164 StringVector* urls) const; 01165 01175 void WriteDomCohortIntoPropertyCache(); 01176 01178 CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher); 01179 01186 void PossiblyPurgeCachedResponseAndReleaseDriver(); 01187 01192 bool ShouldPurgeRewrittenResponse(); 01193 01196 static bool GetPurgeUrl(const GoogleUrl& google_url, 01197 const RewriteOptions* options, 01198 GoogleString* purge_url, 01199 GoogleString* purge_method); 01200 01202 void PurgeDownstreamCache(const GoogleString& purge_url, 01203 const GoogleString& purge_method); 01204 01206 void LogStats(); 01207 01224 bool PrepareShouldSignal(); 01225 void SignalIfRequired(bool result_of_prepare_should_signal); 01226 01238 bool base_was_set_; 01239 01244 bool refs_before_base_; 01245 01247 GoogleString containing_charset_; 01248 01249 bool filters_added_; 01250 bool externally_managed_; 01251 01260 enum RefCategory { 01261 kRefUser, 01262 kRefParsing, 01263 01267 kRefPendingRewrites, 01268 01272 kRefDetachedRewrites, 01273 01281 kRefDeletingRewrites, 01282 01284 kRefFetchUserFacing, 01285 01287 kRefFetchBackground, 01288 01293 kRefAsyncEvents, 01294 01295 kNumRefCategories 01296 }; 01297 01298 friend class CategorizedRefcount<RewriteDriver, RefCategory>; 01299 01301 CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_; 01302 01304 void LastRefRemoved(); 01305 StringPiece RefCategoryName(RefCategory cat); 01306 01309 void DropReference(RefCategory cat); 01310 01313 bool release_driver_; 01314 01317 bool parsing_; 01318 01322 WaitMode waiting_; 01323 01325 bool waiting_deadline_reached_; 01326 01331 bool fully_rewrite_on_flush_; 01332 01335 bool fast_blocking_rewrite_; 01336 01337 bool flush_requested_; 01338 bool flush_occurred_; 01339 01341 bool flushed_cached_html_; 01342 01344 bool flushing_cached_html_; 01345 01348 bool flushed_early_; 01352 bool flushing_early_; 01353 01356 bool is_lazyload_script_flushed_; 01357 01360 bool made_downstream_purge_attempt_; 01361 01365 bool write_property_cache_dom_cohort_; 01366 01369 GoogleUrl base_url_; 01370 01374 GoogleUrl decoded_base_url_; 01375 01378 GoogleString fetch_url_; 01379 01380 GoogleString user_agent_; 01381 01382 LazyBool should_skip_parsing_; 01383 01384 StringFilterMap resource_filter_map_; 01385 01386 ResponseHeaders* response_headers_; 01387 01390 scoped_ptr<const RequestHeaders> request_headers_; 01391 01392 int status_code_; 01393 01396 typedef std::vector<RewriteContext*> RewriteContextVector; 01397 RewriteContextVector rewrites_; 01398 01401 int max_page_processing_delay_ms_; 01402 01403 typedef std::set<RewriteContext*> RewriteContextSet; 01404 01409 RewriteContextSet initiated_rewrites_; 01410 01412 int64 num_initiated_rewrites_; 01413 01421 int64 num_detached_rewrites_; 01422 01430 RewriteContextSet detached_rewrites_; 01431 01433 int possibly_quick_rewrites_; 01434 01437 RewriteContextVector fetch_rewrites_; 01438 01441 FileSystem* file_system_; 01442 ServerContext* server_context_; 01443 Scheduler* scheduler_; 01444 UrlAsyncFetcher* default_url_async_fetcher_; 01445 01449 UrlAsyncFetcher* url_async_fetcher_; 01450 01454 UrlAsyncFetcher* distributed_async_fetcher_; 01455 01458 std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_; 01459 01460 AddInstrumentationFilter* add_instrumentation_filter_; 01461 DomStatsFilter* dom_stats_filter_; 01462 scoped_ptr<HtmlWriterFilter> html_writer_filter_; 01463 01464 ScanFilter scan_filter_; 01465 scoped_ptr<DomainRewriteFilter> domain_rewriter_; 01466 scoped_ptr<UrlLeftTrimFilter> url_trim_filter_; 01467 01470 typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap; 01471 PrimaryRewriteContextMap primary_rewrite_context_map_; 01472 01473 HtmlResourceSlotSet slots_; 01474 01475 scoped_ptr<RewriteOptions> options_; 01476 01477 RewriteDriverPool* controlling_pool_; 01478 01480 scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks> 01481 cache_url_async_fetcher_async_op_hooks_; 01482 01484 UrlSegmentEncoder default_encoder_; 01485 01487 FilterList early_pre_render_filters_; 01489 FilterList pre_render_filters_; 01490 01494 FilterVector filters_to_delete_; 01495 01496 QueuedWorkerPool::Sequence* html_worker_; 01497 QueuedWorkerPool::Sequence* rewrite_worker_; 01498 QueuedWorkerPool::Sequence* low_priority_rewrite_worker_; 01499 01500 Writer* writer_; 01501 01503 GoogleString client_id_; 01504 01507 FallbackPropertyPage* fallback_property_page_; 01508 01510 bool owns_property_page_; 01511 01513 UserAgentMatcher::DeviceType device_type_; 01514 01515 scoped_ptr<CriticalLineInfo> critical_line_info_; 01516 01517 scoped_ptr<SplitHtmlConfig> split_html_config_; 01518 01521 scoped_ptr<CriticalImagesInfo> critical_images_info_; 01522 scoped_ptr<CriticalSelectorInfo> critical_selector_info_; 01523 01524 scoped_ptr<CriticalCssResult> critical_css_result_; 01525 01527 bool xhtml_mimetype_computed_; 01528 XhtmlStatus xhtml_status_ : 8; 01529 01532 int num_inline_preview_images_; 01533 01535 int num_flushed_early_pagespeed_resources_; 01536 01538 int num_bytes_in_; 01539 01540 DebugFilter* debug_filter_; 01541 01542 scoped_ptr<FlushEarlyInfo> flush_early_info_; 01543 scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_; 01544 01546 bool is_blink_request_; 01547 bool can_rewrite_resources_; 01548 bool is_nested_; 01549 01552 RequestContextPtr request_context_; 01553 01555 int64 start_time_ms_; 01556 01557 scoped_ptr<RequestProperties> request_properties_; 01558 01562 static int initialized_count_; 01563 01566 bool tried_to_distribute_fetch_; 01567 01570 bool defer_instrumentation_script_; 01571 01572 DISALLOW_COPY_AND_ASSIGN(RewriteDriver); 01573 }; 01574 01577 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback { 01578 public: 01579 virtual ~OptionsAwareHTTPCacheCallback(); 01580 virtual bool IsCacheValid(const GoogleString& key, 01581 const ResponseHeaders& headers); 01582 virtual int64 OverrideCacheTtlMs(const GoogleString& key); 01583 protected: 01586 OptionsAwareHTTPCacheCallback( 01587 const RewriteOptions* rewrite_options, 01588 const RequestContextPtr& request_ctx); 01589 01590 private: 01591 const RewriteOptions* rewrite_options_; 01592 01593 DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback); 01594 }; 01595 01596 } 01597 01598 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_