Page Speed Optimization Libraries
1.7.30.4
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_ 00021 00022 #include <map> 00023 #include <set> 00024 #include <vector> 00025 00026 #include "base/logging.h" 00027 #include "net/instaweb/htmlparse/public/html_element.h" 00028 #include "net/instaweb/htmlparse/public/html_parse.h" 00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h" 00030 #include "net/instaweb/http/public/http_cache.h" 00031 #include "net/instaweb/http/public/request_context.h" 00032 #include "net/instaweb/http/public/user_agent_matcher.h" 00033 #include "net/instaweb/rewriter/public/critical_images_finder.h" 00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h" 00035 #include "net/instaweb/rewriter/public/output_resource_kind.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/resource_slot.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/scan_filter.h" 00040 #include "net/instaweb/rewriter/public/server_context.h" 00041 #include "net/instaweb/util/public/basictypes.h" 00042 #include "net/instaweb/util/public/google_url.h" 00043 #include "net/instaweb/util/public/printf_format.h" 00044 #include "net/instaweb/util/public/queued_worker_pool.h" 00045 #include "net/instaweb/util/public/scheduler.h" 00046 #include "net/instaweb/util/public/scoped_ptr.h" 00047 #include "net/instaweb/util/public/string.h" 00048 #include "net/instaweb/util/public/string_util.h" 00049 #include "net/instaweb/util/public/thread_system.h" 00050 #include "net/instaweb/util/public/url_segment_encoder.h" 00051 #include "pagespeed/kernel/base/thread_annotations.h" 00052 #include "pagespeed/kernel/http/content_type.h" 00053 #include "pagespeed/kernel/http/response_headers.h" 00054 #include "pagespeed/kernel/util/categorized_refcount.h" 00055 00056 namespace net_instaweb { 00057 00058 class AbstractLogRecord; 00059 class AbstractMutex; 00060 class AbstractPropertyPage; 00061 class AsyncFetch; 00062 class CriticalCssResult; 00063 class CriticalKeys; 00064 class CriticalLineInfo; 00065 class DebugFilter; 00066 class DomStatsFilter; 00067 class DomainRewriteFilter; 00068 class FallbackPropertyPage; 00069 class FileSystem; 00070 class FlushEarlyInfo; 00071 class FlushEarlyRenderInfo; 00072 class Function; 00073 class HtmlFilter; 00074 class HtmlWriterFilter; 00075 class MessageHandler; 00076 class OutputResource; 00077 class PropertyPage; 00078 class RequestHeaders; 00079 class RequestProperties; 00080 class RequestTrace; 00081 class ResourceContext; 00082 class ResourceNamer; 00083 class RewriteContext; 00084 class RewriteDriverPool; 00085 class RewriteFilter; 00086 class SplitHtmlConfig; 00087 class Statistics; 00088 class UrlAsyncFetcher; 00089 class UrlLeftTrimFilter; 00090 class UrlNamer; 00091 class Writer; 00092 00095 class RewriteDriver : public HtmlParse { 00096 public: 00098 enum CssResolutionStatus { 00099 kWriteFailed, 00100 kNoResolutionNeeded, 00101 kSuccess 00102 }; 00103 00105 enum WaitMode { 00106 kNoWait, 00107 kWaitForCompletion, 00108 kWaitForCachedRender, 00109 00110 kWaitForShutDown 00111 00112 }; 00113 00125 enum XhtmlStatus { 00126 kXhtmlUnknown, 00127 kIsXhtml, 00128 kIsNotXhtml 00129 }; 00130 00132 enum InlineAuthorizationPolicy { 00133 kInlineUnauthorizedResources, 00134 kInlineOnlyAuthorizedResources 00135 }; 00136 00138 enum IntendedFor { 00139 kIntendedForInlining, 00140 kIntendedForGeneral 00141 }; 00142 00146 static const char kDomCohort[]; 00148 static const char kBeaconCohort[]; 00149 00152 static const char kLastRequestTimestamp[]; 00154 static const char kParseSizeLimitExceeded[]; 00156 static const char kSubresourcesPropertyName[]; 00158 static const char kStatusCodePropertyName[]; 00159 00160 RewriteDriver(MessageHandler* message_handler, 00161 FileSystem* file_system, 00162 UrlAsyncFetcher* url_async_fetcher); 00163 00166 virtual ~RewriteDriver(); 00167 00170 RewriteDriver* Clone(); 00171 00177 void Clear(); 00178 00180 static void InitStats(Statistics* statistics); 00181 00183 static void Initialize(); 00184 static void Terminate(); 00185 00188 void SetServerContext(ServerContext* server_context); 00189 00192 bool MayCacheExtendCss() const; 00193 bool MayCacheExtendImages() const; 00194 bool MayCacheExtendPdfs() const; 00195 bool MayCacheExtendScripts() const; 00196 00197 const GoogleString& user_agent() const { return user_agent_; } 00198 00199 void SetUserAgent(const StringPiece& user_agent_string); 00200 00201 const RequestProperties* request_properties() const { 00202 return request_properties_.get(); 00203 } 00204 00206 void ClearRequestProperties(); 00207 00209 bool using_spdy() const { return request_context_->using_spdy(); } 00210 00211 bool write_property_cache_dom_cohort() const { 00212 return write_property_cache_dom_cohort_; 00213 } 00214 void set_write_property_cache_dom_cohort(bool x) { 00215 write_property_cache_dom_cohort_ = x; 00216 } 00217 00218 RequestContextPtr request_context() { return request_context_; } 00219 void set_request_context(const RequestContextPtr& x); 00220 00223 RequestTrace* trace_context(); 00224 00227 void TracePrintf(const char* fmt, ...); 00228 00231 ResponseHeaders* mutable_response_headers() { 00232 return flush_occurred_ ? NULL : response_headers_; 00233 } 00234 00241 const ResponseHeaders* response_headers() { 00242 return response_headers_; 00243 } 00244 00248 void set_response_headers_ptr(ResponseHeaders* headers) { 00249 response_headers_ = headers; 00250 } 00251 00257 void SetRequestHeaders(const RequestHeaders& headers); 00258 00259 const RequestHeaders* request_headers() const { 00260 return request_headers_.get(); 00261 } 00262 00263 UserAgentMatcher* user_agent_matcher() const { 00264 DCHECK(server_context() != NULL); 00265 return server_context()->user_agent_matcher(); 00266 } 00267 00272 void AddFilters(); 00273 00277 void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter); 00278 00280 void PrependOwnedPreRenderFilter(HtmlFilter* filter); 00282 void AppendOwnedPreRenderFilter(HtmlFilter* filter); 00283 00285 void AddOwnedPostRenderFilter(HtmlFilter* filter); 00287 void AddUnownedPostRenderFilter(HtmlFilter* filter); 00288 00300 void AppendRewriteFilter(RewriteFilter* filter); 00301 00304 void PrependRewriteFilter(RewriteFilter* filter); 00305 00313 void AddResourceUrlClaimant(ResourceUrlClaimant* claimant); 00314 00320 void SetWriter(Writer* writer); 00321 00322 Writer* writer() const { return writer_; } 00323 00352 bool FetchResource(const StringPiece& url, AsyncFetch* fetch); 00353 00371 void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode, 00372 AsyncFetch* async_fetch); 00373 00380 bool FetchOutputResource(const OutputResourcePtr& output_resource, 00381 RewriteFilter* filter, 00382 AsyncFetch* async_fetch); 00383 00390 OutputResourcePtr DecodeOutputResource(const GoogleUrl& url, 00391 RewriteFilter** filter) const; 00392 00398 bool DecodeOutputResourceName(const GoogleUrl& url, 00399 const RewriteOptions* options_to_use, 00400 const UrlNamer* url_namer, 00401 ResourceNamer* name_out, 00402 OutputResourceKind* kind_out, 00403 RewriteFilter** filter_out) const; 00404 00406 bool DecodeUrl(const GoogleUrl& url, 00407 StringVector* decoded_urls) const; 00408 00411 bool DecodeUrlGivenOptions(const GoogleUrl& url, 00412 const RewriteOptions* options, 00413 const UrlNamer* url_namer, 00414 StringVector* decoded_urls) const; 00415 00416 FileSystem* file_system() { return file_system_; } 00417 UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; } 00418 00423 void SetSessionFetcher(UrlAsyncFetcher* f); 00424 00425 UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; } 00427 void set_distributed_fetcher(UrlAsyncFetcher* fetcher) { 00428 distributed_async_fetcher_ = fetcher; 00429 } 00430 00433 CacheUrlAsyncFetcher* CreateCacheFetcher(); 00435 CacheUrlAsyncFetcher* CreateCacheOnlyFetcher(); 00436 00437 ServerContext* server_context() const { return server_context_; } 00438 Statistics* statistics() const; 00439 00441 void set_custom_options(RewriteOptions* options) { 00442 set_options_for_pool(NULL, options); 00443 } 00444 00447 void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) { 00448 controlling_pool_ = pool; 00449 options_.reset(options); 00450 } 00451 00453 RewriteDriverPool* controlling_pool() { return controlling_pool_; } 00454 00456 const RewriteOptions* options() const { return options_.get(); } 00457 00461 virtual bool StartParseId(const StringPiece& url, const StringPiece& id, 00462 const ContentType& content_type); 00463 00470 virtual void FinishParse(); 00471 00475 void FinishParseAsync(Function* callback); 00476 00480 void InfoAt(const RewriteContext* context, 00481 const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4); 00482 00487 00494 OutputResourcePtr CreateOutputResourceFromResource( 00495 const StringPiece& filter_id, 00496 const UrlSegmentEncoder* encoder, 00497 const ResourceContext* data, 00498 const ResourcePtr& input_resource, 00499 OutputResourceKind kind); 00500 00517 OutputResourcePtr CreateOutputResourceWithPath( 00518 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00519 const StringPiece& base_url, const StringPiece& filter_id, 00520 const StringPiece& name, OutputResourceKind kind); 00521 00524 void PopulateResourceNamer( 00525 const StringPiece& filter_id, 00526 const StringPiece& name, 00527 ResourceNamer* full_name); 00528 00532 OutputResourcePtr CreateOutputResourceWithUnmappedUrl( 00533 const GoogleUrl& unmapped_gurl, const StringPiece& filter_id, 00534 const StringPiece& name, OutputResourceKind kind); 00535 00538 OutputResourcePtr CreateOutputResourceWithMappedPath( 00539 const StringPiece& mapped_path, const StringPiece& unmapped_path, 00540 const StringPiece& filter_id, const StringPiece& name, 00541 OutputResourceKind kind) { 00542 return CreateOutputResourceWithPath(mapped_path, unmapped_path, 00543 decoded_base_url_.AllExceptLeaf(), 00544 filter_id, name, kind); 00545 } 00546 00549 OutputResourcePtr CreateOutputResourceWithPath( 00550 const StringPiece& path, const StringPiece& filter_id, 00551 const StringPiece& name, OutputResourceKind kind) { 00552 return CreateOutputResourceWithPath(path, path, path, filter_id, name, 00553 kind); 00554 } 00555 00560 ResourcePtr CreateInputResource(const GoogleUrl& input_url); 00561 00575 ResourcePtr CreateInputResource( 00576 const GoogleUrl& input_url, 00577 InlineAuthorizationPolicy inline_authorization_policy, 00578 IntendedFor intended_for); 00579 00583 ResourcePtr CreateInputResourceAbsoluteUnchecked( 00584 const StringPiece& absolute_url); 00585 00589 bool IsResourceUrlClaimed(const GoogleUrl& url) const; 00590 00595 bool MatchesBaseUrl(const GoogleUrl& input_url) const; 00596 00602 bool MayRewriteUrl(const GoogleUrl& domain_url, 00603 const GoogleUrl& input_url, 00604 InlineAuthorizationPolicy inline_authorization_policy, 00605 IntendedFor intended_for, 00606 bool* is_authorized_domain) const; 00607 00611 const GoogleUrl& base_url() const { return base_url_; } 00612 00614 StringPiece fetch_url() const { return fetch_url_; } 00615 00619 const GoogleUrl& decoded_base_url() const { return decoded_base_url_; } 00620 StringPiece decoded_base() const { return decoded_base_url_.Spec(); } 00621 00623 bool IsHttps() const { return google_url().SchemeIs("https"); } 00624 00625 const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; } 00626 00628 RewriteFilter* FindFilter(const StringPiece& id) const; 00629 00631 bool refs_before_base() { return refs_before_base_; } 00632 00637 void set_refs_before_base() { refs_before_base_ = true; } 00638 00643 StringPiece containing_charset() { return containing_charset_; } 00644 void set_containing_charset(const StringPiece charset) { 00645 charset.CopyToString(&containing_charset_); 00646 } 00647 00649 HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource, 00650 HtmlElement* elt, 00651 HtmlElement::Attribute* attr); 00652 00657 bool InitiateRewrite(RewriteContext* rewrite_context) 00658 LOCKS_EXCLUDED(rewrite_mutex()); 00659 void InitiateFetch(RewriteContext* rewrite_context); 00660 00671 void RewriteComplete(RewriteContext* rewrite_context, bool permit_render); 00672 00676 void ReportSlowRewrites(int num); 00677 00682 void Cleanup(); 00683 00688 void AddUserReference(); 00689 00691 GoogleString ToString(bool show_detached_contexts); 00692 void PrintState(bool show_detached_contexts); 00693 void PrintStateToErrorLog(bool show_detached_contexts); 00694 00697 void WaitForCompletion(); 00698 00705 void WaitForShutDown(); 00706 00710 void BoundedWaitFor(WaitMode mode, int64 timeout_ms) 00711 LOCKS_EXCLUDED(rewrite_mutex()); 00712 00720 void set_fully_rewrite_on_flush(bool x) { 00721 fully_rewrite_on_flush_ = x; 00722 } 00723 00725 bool fully_rewrite_on_flush() const { 00726 return fully_rewrite_on_flush_; 00727 } 00728 00732 void set_fast_blocking_rewrite(bool x) { 00733 fast_blocking_rewrite_ = x; 00734 } 00735 00736 bool fast_blocking_rewrite() const { 00737 return fast_blocking_rewrite_; 00738 } 00739 00742 void EnableBlockingRewrite(RequestHeaders* request_headers); 00743 00750 void set_externally_managed(bool x) { externally_managed_ = x; } 00751 00755 void DetachFetch(); 00756 00759 void DetachedFetchComplete(); 00760 00764 void FetchComplete(); 00765 00771 void DeleteRewriteContext(RewriteContext* rewrite_context); 00772 00773 int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); } 00774 00779 void set_max_page_processing_delay_ms(int x) { 00780 max_page_processing_delay_ms_ = x; 00781 } 00782 int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; } 00783 00785 void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; } 00786 UserAgentMatcher::DeviceType device_type() const { return device_type_; } 00787 00793 RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key, 00794 RewriteContext* candidate); 00795 00800 void DeregisterForPartitionKey( 00801 const GoogleString& partition_key, RewriteContext* candidate); 00802 00805 void RequestFlush() { flush_requested_ = true; } 00806 bool flush_requested() const { return flush_requested_; } 00807 00819 void ExecuteFlushIfRequested(); 00820 00824 void ExecuteFlushIfRequestedAsync(Function* callback); 00825 00834 virtual void Flush(); 00835 00839 void FlushAsync(Function* done); 00840 00842 void AddRewriteTask(Function* task); 00843 00846 void AddLowPriorityRewriteTask(Function* task); 00847 00848 QueuedWorkerPool::Sequence* html_worker() { return html_worker_; } 00849 QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; } 00850 QueuedWorkerPool::Sequence* low_priority_rewrite_worker() { 00851 return low_priority_rewrite_worker_; 00852 } 00853 00854 Scheduler* scheduler() { return scheduler_; } 00855 00858 DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); } 00859 UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); } 00860 00868 CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base, 00869 const StringPiece& output_css_base, 00870 const StringPiece& contents, 00871 Writer* writer, 00872 MessageHandler* handler); 00873 00881 bool ShouldAbsolutifyUrl(const GoogleUrl& input_base, 00882 const GoogleUrl& output_base, 00883 bool* proxy_mode) const; 00884 00892 void UpdatePropertyValueInDomCohort( 00893 AbstractPropertyPage* page, 00894 StringPiece property_name, 00895 StringPiece property_value); 00896 00899 PropertyPage* property_page() const; 00904 FallbackPropertyPage* fallback_property_page() const { 00905 return fallback_property_page_; 00906 } 00908 void set_property_page(PropertyPage* page); 00910 void set_fallback_property_page(FallbackPropertyPage* page); 00912 void set_unowned_fallback_property_page(FallbackPropertyPage* page); 00913 00915 const CriticalLineInfo* critical_line_info() const; 00916 00919 void set_critical_line_info(CriticalLineInfo* critical_line_info); 00920 00921 CriticalKeys* beacon_critical_line_info() const; 00922 void set_beacon_critical_line_info(CriticalKeys* beacon_critical_line_info); 00923 00924 const SplitHtmlConfig* split_html_config(); 00925 00926 CriticalCssResult* critical_css_result() const; 00929 void set_critical_css_result(CriticalCssResult* critical_css_rules); 00930 00932 CriticalImagesInfo* critical_images_info() const { 00933 return critical_images_info_.get(); 00934 } 00935 00940 CriticalSelectorInfo* critical_selector_info() { 00941 return critical_selector_info_.get(); 00942 } 00943 00947 void set_critical_selector_info(CriticalSelectorInfo* info) { 00948 critical_selector_info_.reset(info); 00949 } 00950 00955 void set_critical_images_info(CriticalImagesInfo* critical_images_info) { 00956 critical_images_info_.reset(critical_images_info); 00957 } 00958 00961 bool CriticalSelectorsEnabled() const; 00962 00965 bool FlattenCssImportsEnabled() const { 00966 return (options()->Enabled(RewriteOptions::kFlattenCssImports) || 00967 (!options()->Forbidden(RewriteOptions::kFlattenCssImports) && 00968 (CriticalSelectorsEnabled() || 00969 options()->Enabled(RewriteOptions::kComputeCriticalCss)))); 00970 } 00971 00975 int num_inline_preview_images() const { return num_inline_preview_images_; } 00976 00978 void increment_num_inline_preview_images(); 00979 00982 int num_flushed_early_pagespeed_resources() const { 00983 return num_flushed_early_pagespeed_resources_; 00984 } 00985 00988 void increment_num_flushed_early_pagespeed_resources() { 00989 ++num_flushed_early_pagespeed_resources_; 00990 } 00991 00994 void increment_async_events_count(); 00995 00997 void decrement_async_events_count(); 00998 01001 XhtmlStatus MimeTypeXhtmlStatus(); 01002 01003 void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; } 01004 bool flushed_cached_html() { return flushed_cached_html_; } 01005 01006 void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; } 01007 bool flushing_cached_html() const { return flushing_cached_html_; } 01008 01009 void set_flushed_early(bool x) { flushed_early_ = x; } 01010 bool flushed_early() const { return flushed_early_; } 01011 01012 void set_flushing_early(bool x) { flushing_early_ = x; } 01013 bool flushing_early() const { return flushing_early_; } 01014 01015 void set_is_lazyload_script_flushed(bool x) { 01016 is_lazyload_script_flushed_ = x; 01017 } 01018 bool is_lazyload_script_flushed() const { 01019 return is_lazyload_script_flushed_; } 01020 01022 FlushEarlyInfo* flush_early_info(); 01023 01024 FlushEarlyRenderInfo* flush_early_render_info() const; 01025 01028 void set_flush_early_render_info( 01029 FlushEarlyRenderInfo* flush_early_render_info); 01030 01033 bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); } 01034 01037 void SaveOriginalHeaders(const ResponseHeaders& response_headers); 01038 01041 AbstractLogRecord* log_record(); 01042 01043 DomStatsFilter* dom_stats_filter() const { 01044 return dom_stats_filter_; 01045 } 01046 01049 bool can_rewrite_resources() const { return can_rewrite_resources_; } 01050 01052 bool is_nested() const { return is_nested_; } 01053 01057 bool MetadataRequested(const RequestHeaders& request_headers) const; 01058 01060 bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; } 01061 01074 bool Write(const ResourceVector& inputs, 01075 const StringPiece& contents, 01076 const ContentType* type, 01077 StringPiece charset, 01078 OutputResource* output); 01079 01080 void set_defer_instrumentation_script(bool x) { 01081 defer_instrumentation_script_ = x; 01082 } 01083 bool defer_instrumentation_script() const { 01084 return defer_instrumentation_script_; 01085 } 01086 01089 const GoogleString& CacheFragment() const; 01090 01091 protected: 01092 virtual void DetermineEnabledFiltersImpl(); 01093 01094 private: 01095 friend class DistributedRewriteContextTest; 01096 friend class RewriteContext; 01097 friend class RewriteDriverTest; 01098 friend class RewriteTestBase; 01099 friend class ServerContextTest; 01100 01101 typedef std::map<GoogleString, RewriteFilter*> StringFilterMap; 01102 01104 bool ShouldDistributeFetch(const StringPiece& filter_id); 01105 01117 bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id, 01118 AsyncFetch* async_fetch); 01119 01124 void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms, 01125 Function* done) 01126 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01127 01131 void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms, 01132 Function* done) 01133 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01134 01136 bool IsDone(WaitMode wait_mode, bool deadline_reached) 01137 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01138 01141 bool WaitForPendingAsyncEvents(WaitMode wait_mode) { 01142 return wait_mode == kWaitForShutDown || 01143 (fully_rewrite_on_flush_ && !fast_blocking_rewrite_); 01144 } 01145 01149 void FlushAsyncDone(int num_rewrites, Function* callback); 01150 01155 int64 ComputeCurrentFlushWindowRewriteDelayMs(); 01156 01158 void QueueFlushAsyncDone(int num_rewrites, Function* callback); 01159 01162 void QueueFinishParseAfterFlush(Function* user_callback); 01163 void FinishParseAfterFlush(Function* user_callback); 01164 01165 bool RewritesComplete() const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01166 01169 void SetBaseUrlIfUnset(const StringPiece& new_base); 01170 01173 void SetBaseUrlForFetch(const StringPiece& url); 01174 01177 void SetDecodedUrlFromBase(); 01178 01180 AbstractMutex* rewrite_mutex() const LOCK_RETURNED(scheduler_->mutex()) { 01181 return scheduler_->mutex(); 01182 } 01183 01185 virtual void ParseTextInternal(const char* content, int size); 01186 01188 bool ShouldSkipParsing(); 01189 01190 friend class ScanFilter; 01191 01195 void RegisterRewriteFilter(RewriteFilter* filter); 01196 01201 void EnableRewriteFilter(const char* id); 01202 01208 ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl, 01209 bool is_authorized_domain); 01210 01211 void AddPreRenderFilters(); 01212 void AddPostRenderFilters(); 01213 01215 bool DecodeOutputResourceNameHelper(const GoogleUrl& url, 01216 const RewriteOptions* options_to_use, 01217 const UrlNamer* url_namer, 01218 ResourceNamer* name_out, 01219 OutputResourceKind* kind_out, 01220 RewriteFilter** filter_out, 01221 GoogleString* url_base, 01222 StringVector* urls) const; 01223 01233 void WriteDomCohortIntoPropertyCache(); 01234 01236 CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher); 01237 01244 void PossiblyPurgeCachedResponseAndReleaseDriver(); 01245 01250 bool ShouldPurgeRewrittenResponse(); 01251 01254 static bool GetPurgeUrl(const GoogleUrl& google_url, 01255 const RewriteOptions* options, 01256 GoogleString* purge_url, 01257 GoogleString* purge_method); 01258 01260 void PurgeDownstreamCache(const GoogleString& purge_url, 01261 const GoogleString& purge_method); 01262 01264 void LogStats(); 01265 01281 bool PrepareShouldSignal() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01282 void SignalIfRequired(bool result_of_prepare_should_signal) 01283 EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex()); 01284 01296 bool base_was_set_; 01297 01302 bool refs_before_base_; 01303 01305 GoogleString containing_charset_; 01306 01309 void PopulateRequestContext(); 01310 01311 bool filters_added_; 01312 bool externally_managed_; 01313 01322 enum RefCategory { 01323 kRefUser, 01324 kRefParsing, 01325 01329 kRefPendingRewrites, 01330 01334 kRefDetachedRewrites, 01335 01343 kRefDeletingRewrites, 01344 01346 kRefFetchUserFacing, 01347 01349 kRefFetchBackground, 01350 01355 kRefAsyncEvents, 01356 01357 kNumRefCategories 01358 }; 01359 01360 friend class CategorizedRefcount<RewriteDriver, RefCategory>; 01361 01363 CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_; 01364 01366 void LastRefRemoved(); 01367 StringPiece RefCategoryName(RefCategory cat); 01368 01371 void DropReference(RefCategory cat); 01372 01375 bool release_driver_; 01376 01379 bool parsing_ GUARDED_BY(rewrite_mutex()); 01380 01384 WaitMode waiting_ GUARDED_BY(rewrite_mutex()); 01385 01387 bool waiting_deadline_reached_ GUARDED_BY(rewrite_mutex()); 01388 01393 bool fully_rewrite_on_flush_; 01394 01397 bool fast_blocking_rewrite_; 01398 01399 bool flush_requested_; 01400 bool flush_occurred_; 01401 01403 bool flushed_cached_html_; 01404 01406 bool flushing_cached_html_; 01407 01410 bool flushed_early_; 01414 bool flushing_early_; 01415 01418 bool is_lazyload_script_flushed_; 01419 01422 bool made_downstream_purge_attempt_; 01423 01427 bool write_property_cache_dom_cohort_; 01428 01431 GoogleUrl base_url_; 01432 01436 GoogleUrl decoded_base_url_; 01437 01440 GoogleString fetch_url_; 01441 01442 GoogleString user_agent_; 01443 01444 LazyBool should_skip_parsing_; 01445 01446 StringFilterMap resource_filter_map_; 01447 01448 ResponseHeaders* response_headers_; 01449 01452 scoped_ptr<const RequestHeaders> request_headers_; 01453 01454 int status_code_; 01455 01458 typedef std::vector<RewriteContext*> RewriteContextVector; 01459 RewriteContextVector rewrites_; 01460 01463 int max_page_processing_delay_ms_; 01464 01465 typedef std::set<RewriteContext*> RewriteContextSet; 01466 01471 RewriteContextSet initiated_rewrites_ GUARDED_BY(rewrite_mutex()); 01472 01474 int64 num_initiated_rewrites_ GUARDED_BY(rewrite_mutex()); 01475 01483 int64 num_detached_rewrites_ GUARDED_BY(rewrite_mutex()); 01484 01492 RewriteContextSet detached_rewrites_ GUARDED_BY(rewrite_mutex()); 01493 01495 int possibly_quick_rewrites_ GUARDED_BY(rewrite_mutex()); 01496 01499 RewriteContextVector fetch_rewrites_; 01500 01503 FileSystem* file_system_; 01504 ServerContext* server_context_; 01505 Scheduler* scheduler_; 01506 UrlAsyncFetcher* default_url_async_fetcher_; 01507 01511 UrlAsyncFetcher* url_async_fetcher_; 01512 01516 UrlAsyncFetcher* distributed_async_fetcher_; 01517 01520 std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_; 01521 01522 DomStatsFilter* dom_stats_filter_; 01523 scoped_ptr<HtmlWriterFilter> html_writer_filter_; 01524 01525 ScanFilter scan_filter_; 01526 scoped_ptr<DomainRewriteFilter> domain_rewriter_; 01527 scoped_ptr<UrlLeftTrimFilter> url_trim_filter_; 01528 01531 typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap; 01532 PrimaryRewriteContextMap primary_rewrite_context_map_; 01533 01534 HtmlResourceSlotSet slots_; 01535 01536 scoped_ptr<RewriteOptions> options_; 01537 01538 RewriteDriverPool* controlling_pool_; 01539 01541 scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks> 01542 cache_url_async_fetcher_async_op_hooks_; 01543 01545 UrlSegmentEncoder default_encoder_; 01546 01548 FilterList early_pre_render_filters_; 01550 FilterList pre_render_filters_; 01551 01553 std::vector<ResourceUrlClaimant*> resource_claimants_; 01554 01558 FilterVector filters_to_delete_; 01559 01560 QueuedWorkerPool::Sequence* html_worker_; 01561 QueuedWorkerPool::Sequence* rewrite_worker_; 01562 QueuedWorkerPool::Sequence* low_priority_rewrite_worker_; 01563 01564 Writer* writer_; 01565 01568 FallbackPropertyPage* fallback_property_page_; 01569 01571 bool owns_property_page_; 01572 01574 UserAgentMatcher::DeviceType device_type_; 01575 01576 scoped_ptr<CriticalLineInfo> critical_line_info_; 01577 scoped_ptr<CriticalKeys> beacon_critical_line_info_; 01578 01579 scoped_ptr<SplitHtmlConfig> split_html_config_; 01580 01583 scoped_ptr<CriticalImagesInfo> critical_images_info_; 01584 scoped_ptr<CriticalSelectorInfo> critical_selector_info_; 01585 01586 scoped_ptr<CriticalCssResult> critical_css_result_; 01587 01589 bool xhtml_mimetype_computed_; 01590 XhtmlStatus xhtml_status_ : 8; 01591 01594 int num_inline_preview_images_; 01595 01597 int num_flushed_early_pagespeed_resources_; 01598 01600 int num_bytes_in_; 01601 01602 DebugFilter* debug_filter_; 01603 01604 scoped_ptr<FlushEarlyInfo> flush_early_info_; 01605 scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_; 01606 01607 bool can_rewrite_resources_; 01608 bool is_nested_; 01609 01612 RequestContextPtr request_context_; 01613 01615 int64 start_time_ms_; 01616 01617 scoped_ptr<RequestProperties> request_properties_; 01618 01622 static int initialized_count_; 01623 01626 bool tried_to_distribute_fetch_; 01627 01630 bool defer_instrumentation_script_; 01631 01632 DISALLOW_COPY_AND_ASSIGN(RewriteDriver); 01633 }; 01634 01637 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback { 01638 public: 01639 virtual ~OptionsAwareHTTPCacheCallback(); 01640 virtual bool IsCacheValid(const GoogleString& key, 01641 const ResponseHeaders& headers); 01642 virtual int64 OverrideCacheTtlMs(const GoogleString& key); 01643 virtual ResponseHeaders::VaryOption RespectVaryOnResources() const; 01644 01648 static bool IsCacheValid(const GoogleString& key, 01649 const RewriteOptions& rewrite_options, 01650 const RequestContextPtr& request_ctx, 01651 const ResponseHeaders& headers); 01652 01653 protected: 01656 OptionsAwareHTTPCacheCallback( 01657 const RewriteOptions* rewrite_options, 01658 const RequestContextPtr& request_ctx); 01659 01660 private: 01661 const RewriteOptions* rewrite_options_; 01662 01663 DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback); 01664 }; 01665 01666 } 01667 01668 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_