Page Speed Optimization Libraries  1.8.31.3
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h"
00035 #include "net/instaweb/rewriter/public/downstream_cache_purger.h"
00036 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00037 #include "net/instaweb/rewriter/public/resource.h"
00038 #include "net/instaweb/rewriter/public/resource_slot.h"
00039 #include "net/instaweb/rewriter/public/rewrite_context.h"
00040 #include "net/instaweb/rewriter/public/rewrite_options.h"
00041 #include "net/instaweb/rewriter/public/scan_filter.h"
00042 #include "net/instaweb/rewriter/public/server_context.h"
00043 #include "net/instaweb/util/public/basictypes.h"
00044 #include "net/instaweb/util/public/google_url.h"
00045 #include "net/instaweb/util/public/printf_format.h"
00046 #include "net/instaweb/util/public/queued_worker_pool.h"
00047 #include "net/instaweb/util/public/scheduler.h"
00048 #include "net/instaweb/util/public/scoped_ptr.h"
00049 #include "net/instaweb/util/public/string.h"
00050 #include "net/instaweb/util/public/string_util.h"
00051 #include "net/instaweb/util/public/thread_system.h"
00052 #include "net/instaweb/util/public/url_segment_encoder.h"
00053 #include "pagespeed/kernel/base/abstract_mutex.h"
00054 #include "pagespeed/kernel/base/thread_annotations.h"
00055 #include "pagespeed/kernel/http/content_type.h"
00056 #include "pagespeed/kernel/http/response_headers.h"
00057 #include "pagespeed/kernel/util/categorized_refcount.h"
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class AbstractMutex;
00063 class AbstractPropertyPage;
00064 class AsyncFetch;
00065 class CriticalCssResult;
00066 class CriticalKeys;
00067 class CriticalLineInfo;
00068 class DebugFilter;
00069 class DomStatsFilter;
00070 class DomainRewriteFilter;
00071 class FallbackPropertyPage;
00072 class FileSystem;
00073 class FlushEarlyInfo;
00074 class FlushEarlyRenderInfo;
00075 class Function;
00076 class HtmlFilter;
00077 class HtmlWriterFilter;
00078 class MessageHandler;
00079 class OutputResource;
00080 class PropertyPage;
00081 class RequestHeaders;
00082 class RequestProperties;
00083 class RequestTrace;
00084 class ResourceContext;
00085 class ResourceNamer;
00086 class RewriteDriverPool;
00087 class RewriteFilter;
00088 class SplitHtmlConfig;
00089 class Statistics;
00090 class UrlAsyncFetcher;
00091 class UrlLeftTrimFilter;
00092 class UrlNamer;
00093 class Writer;
00094 
00097 class RewriteDriver : public HtmlParse {
00098  public:
00100   enum CssResolutionStatus {
00101     kWriteFailed,
00102     kNoResolutionNeeded,
00103     kSuccess
00104   };
00105 
00107   enum WaitMode {
00108     kNoWait,  
00109     kWaitForCompletion,    
00110     kWaitForCachedRender,  
00111 
00112     kWaitForShutDown       
00113 
00114   };
00115 
00127   enum XhtmlStatus {
00128     kXhtmlUnknown,
00129     kIsXhtml,
00130     kIsNotXhtml
00131   };
00132 
00134   enum InlineAuthorizationPolicy {
00135     kInlineUnauthorizedResources,
00136     kInlineOnlyAuthorizedResources
00137   };
00138 
00140   enum IntendedFor {
00141     kIntendedForInlining,
00142     kIntendedForGeneral
00143   };
00144 
00148   static const char kDomCohort[];
00150   static const char kBeaconCohort[];
00151 
00154   static const char kLastRequestTimestamp[];
00156   static const char kParseSizeLimitExceeded[];
00158   static const char kSubresourcesPropertyName[];
00160   static const char kStatusCodePropertyName[];
00161 
00162   RewriteDriver(MessageHandler* message_handler,
00163                 FileSystem* file_system,
00164                 UrlAsyncFetcher* url_async_fetcher);
00165 
00168   virtual ~RewriteDriver();
00169 
00172   RewriteDriver* Clone();
00173 
00179   void Clear();
00180 
00182   static void InitStats(Statistics* statistics);
00183 
00185   static void Initialize();
00186   static void Terminate();
00187 
00190   void SetServerContext(ServerContext* server_context);
00191 
00194   bool MayCacheExtendCss() const;
00195   bool MayCacheExtendImages() const;
00196   bool MayCacheExtendPdfs() const;
00197   bool MayCacheExtendScripts() const;
00198 
00199   const GoogleString& user_agent() const { return user_agent_; }
00200 
00201   void SetUserAgent(const StringPiece& user_agent_string);
00202 
00203   const RequestProperties* request_properties() const {
00204     return request_properties_.get();
00205   }
00206 
00208   void ClearRequestProperties();
00209 
00211   bool using_spdy() const { return request_context_->using_spdy(); }
00212 
00213   bool write_property_cache_dom_cohort() const {
00214     return write_property_cache_dom_cohort_;
00215   }
00216   void set_write_property_cache_dom_cohort(bool x) {
00217     write_property_cache_dom_cohort_ = x;
00218   }
00219 
00220   RequestContextPtr request_context() { return request_context_; }
00221   void set_request_context(const RequestContextPtr& x);
00222 
00225   RequestTrace* trace_context();
00226 
00229   void TracePrintf(const char* fmt, ...);
00230 
00233   ResponseHeaders* mutable_response_headers() {
00234     return flush_occurred_ ? NULL : response_headers_;
00235   }
00236 
00243   const ResponseHeaders* response_headers() {
00244     return response_headers_;
00245   }
00246 
00250   void set_response_headers_ptr(ResponseHeaders* headers) {
00251     response_headers_ = headers;
00252   }
00253 
00259   void SetRequestHeaders(const RequestHeaders& headers);
00260 
00261   const RequestHeaders* request_headers() const {
00262     return request_headers_.get();
00263   }
00264 
00265   UserAgentMatcher* user_agent_matcher() const {
00266     DCHECK(server_context() != NULL);
00267     return server_context()->user_agent_matcher();
00268   }
00269 
00274   void AddFilters();
00275 
00279   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00280 
00282   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00284   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00285 
00287   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00289   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00290 
00302   void AppendRewriteFilter(RewriteFilter* filter);
00303 
00306   void PrependRewriteFilter(RewriteFilter* filter);
00307 
00315   void AddResourceUrlClaimant(ResourceUrlClaimant* claimant);
00316 
00322   void SetWriter(Writer* writer);
00323 
00324   Writer* writer() const { return writer_; }
00325 
00354   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00355 
00373   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00374                             AsyncFetch* async_fetch);
00375 
00382   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00383                            RewriteFilter* filter,
00384                            AsyncFetch* async_fetch);
00385 
00392   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00393                                          RewriteFilter** filter) const;
00394 
00400   bool DecodeOutputResourceName(const GoogleUrl& url,
00401                                 const RewriteOptions* options_to_use,
00402                                 const UrlNamer* url_namer,
00403                                 ResourceNamer* name_out,
00404                                 OutputResourceKind* kind_out,
00405                                 RewriteFilter** filter_out) const;
00406 
00418   bool LookupMetadataForOutputResource(
00419       StringPiece url,
00420       GoogleString* error_out,
00421       RewriteContext::CacheLookupResultCallback* callback);
00422 
00424   bool DecodeUrl(const GoogleUrl& url,
00425                  StringVector* decoded_urls) const;
00426 
00429   bool DecodeUrlGivenOptions(const GoogleUrl& url,
00430                              const RewriteOptions* options,
00431                              const UrlNamer* url_namer,
00432                              StringVector* decoded_urls) const;
00433 
00434   FileSystem* file_system() { return file_system_; }
00435   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00436 
00441   void SetSessionFetcher(UrlAsyncFetcher* f);
00442 
00443   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00445   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00446     distributed_async_fetcher_ = fetcher;
00447   }
00448 
00451   CacheUrlAsyncFetcher* CreateCacheFetcher();
00453   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00454 
00455   ServerContext* server_context() const { return server_context_; }
00456   Statistics* statistics() const;
00457 
00459   void set_custom_options(RewriteOptions* options) {
00460     set_options_for_pool(NULL, options);
00461   }
00462 
00465   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00466     controlling_pool_ = pool;
00467     options_.reset(options);
00468   }
00469 
00471   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00472 
00474   const RewriteOptions* options() const { return options_.get(); }
00475 
00479   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00480                             const ContentType& content_type);
00481 
00488   virtual void FinishParse();
00489 
00493   void FinishParseAsync(Function* callback);
00494 
00498   void InfoAt(const RewriteContext* context,
00499               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00500 
00505 
00512   OutputResourcePtr CreateOutputResourceFromResource(
00513       const StringPiece& filter_id,
00514       const UrlSegmentEncoder* encoder,
00515       const ResourceContext* data,
00516       const ResourcePtr& input_resource,
00517       OutputResourceKind kind);
00518 
00535   OutputResourcePtr CreateOutputResourceWithPath(
00536       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00537       const StringPiece& base_url, const StringPiece& filter_id,
00538       const StringPiece& name, OutputResourceKind kind);
00539 
00542   void PopulateResourceNamer(
00543     const StringPiece& filter_id,
00544     const StringPiece& name,
00545     ResourceNamer* full_name);
00546 
00550   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00551       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00552       const StringPiece& name, OutputResourceKind kind);
00553 
00556   OutputResourcePtr CreateOutputResourceWithMappedPath(
00557       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00558       const StringPiece& filter_id, const StringPiece& name,
00559       OutputResourceKind kind) {
00560     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00561                                         decoded_base_url_.AllExceptLeaf(),
00562                                         filter_id, name, kind);
00563   }
00564 
00567   OutputResourcePtr CreateOutputResourceWithPath(
00568       const StringPiece& path, const StringPiece& filter_id,
00569       const StringPiece& name, OutputResourceKind kind) {
00570     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00571                                         kind);
00572   }
00573 
00578   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00579 
00593   ResourcePtr CreateInputResource(
00594       const GoogleUrl& input_url,
00595       InlineAuthorizationPolicy inline_authorization_policy,
00596       IntendedFor intended_for);
00597 
00601   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00602       const StringPiece& absolute_url);
00603 
00607   bool IsResourceUrlClaimed(const GoogleUrl& url) const;
00608 
00613   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00614 
00620   bool MayRewriteUrl(const GoogleUrl& domain_url,
00621                      const GoogleUrl& input_url,
00622                      InlineAuthorizationPolicy inline_authorization_policy,
00623                      IntendedFor intended_for,
00624                      bool* is_authorized_domain) const;
00625 
00629   const GoogleUrl& base_url() const { return base_url_; }
00630 
00632   StringPiece fetch_url() const { return fetch_url_; }
00633 
00637   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00638   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00639 
00641   bool IsHttps() const { return google_url().SchemeIs("https"); }
00642 
00643   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00644 
00646   RewriteFilter* FindFilter(const StringPiece& id) const;
00647 
00649   bool refs_before_base() { return refs_before_base_; }
00650 
00655   void set_refs_before_base() { refs_before_base_ = true; }
00656 
00661   StringPiece containing_charset() { return containing_charset_; }
00662   void set_containing_charset(const StringPiece charset) {
00663     charset.CopyToString(&containing_charset_);
00664   }
00665 
00667   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00668                               HtmlElement* elt,
00669                               HtmlElement::Attribute* attr);
00670 
00675   bool InitiateRewrite(RewriteContext* rewrite_context)
00676       LOCKS_EXCLUDED(rewrite_mutex());
00677   void InitiateFetch(RewriteContext* rewrite_context);
00678 
00689   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00690 
00694   void ReportSlowRewrites(int num);
00695 
00700   void Cleanup();
00701 
00706   void AddUserReference();
00707 
00709   GoogleString ToString(bool show_detached_contexts);
00710   void PrintState(bool show_detached_contexts); 
00711   void PrintStateToErrorLog(bool show_detached_contexts); 
00712 
00715   void WaitForCompletion();
00716 
00723   void WaitForShutDown();
00724 
00728   void BoundedWaitFor(WaitMode mode, int64 timeout_ms)
00729       LOCKS_EXCLUDED(rewrite_mutex());
00730 
00738   void set_fully_rewrite_on_flush(bool x) {
00739     fully_rewrite_on_flush_ = x;
00740   }
00741 
00743   bool fully_rewrite_on_flush() const {
00744     return fully_rewrite_on_flush_;
00745   }
00746 
00750   void set_fast_blocking_rewrite(bool x) {
00751     fast_blocking_rewrite_ = x;
00752   }
00753 
00754   bool fast_blocking_rewrite() const {
00755     return fast_blocking_rewrite_;
00756   }
00757 
00760   void EnableBlockingRewrite(RequestHeaders* request_headers);
00761 
00768   void set_externally_managed(bool x) { externally_managed_ = x; }
00769 
00773   void DetachFetch();
00774 
00777   void DetachedFetchComplete();
00778 
00782   void FetchComplete();
00783 
00789   void DeleteRewriteContext(RewriteContext* rewrite_context);
00790 
00791   int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }
00792 
00797   void set_max_page_processing_delay_ms(int x) {
00798     max_page_processing_delay_ms_ = x;
00799   }
00800   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00801 
00803   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00804   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00805 
00811   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00812                                           RewriteContext* candidate);
00813 
00818   void DeregisterForPartitionKey(
00819       const GoogleString& partition_key, RewriteContext* candidate);
00820 
00823   void RequestFlush() { flush_requested_ = true; }
00824   bool flush_requested() const { return flush_requested_; }
00825 
00837   void ExecuteFlushIfRequested();
00838 
00842   void ExecuteFlushIfRequestedAsync(Function* callback);
00843 
00852   virtual void Flush();
00853 
00857   void FlushAsync(Function* done);
00858 
00860   void AddRewriteTask(Function* task);
00861 
00864   void AddLowPriorityRewriteTask(Function* task);
00865 
00866   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00867   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00868   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00869     return low_priority_rewrite_worker_;
00870   }
00871 
00872   Scheduler* scheduler() { return scheduler_; }
00873 
00876   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00877   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00878 
00886   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00887                                      const StringPiece& output_css_base,
00888                                      const StringPiece& contents,
00889                                      Writer* writer,
00890                                      MessageHandler* handler);
00891 
00899   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00900                            const GoogleUrl& output_base,
00901                            bool* proxy_mode) const;
00902 
00910   void UpdatePropertyValueInDomCohort(
00911       AbstractPropertyPage* page,
00912       StringPiece property_name,
00913       StringPiece property_value);
00914 
00917   PropertyPage* property_page() const;
00922   FallbackPropertyPage* fallback_property_page() const {
00923     return fallback_property_page_;
00924   }
00926   void set_property_page(PropertyPage* page);
00928   void set_fallback_property_page(FallbackPropertyPage* page);
00930   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00931 
00933   const CriticalLineInfo* critical_line_info() const;
00934 
00937   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00938 
00939   CriticalKeys* beacon_critical_line_info() const;
00940   void set_beacon_critical_line_info(CriticalKeys* beacon_critical_line_info);
00941 
00942   const SplitHtmlConfig* split_html_config();
00943 
00944   CriticalCssResult* critical_css_result() const;
00947   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00948 
00950   CriticalImagesInfo* critical_images_info() const {
00951     return critical_images_info_.get();
00952   }
00953 
00958   CriticalSelectorInfo* critical_selector_info() {
00959     return critical_selector_info_.get();
00960   }
00961 
00965   void set_critical_selector_info(CriticalSelectorInfo* info) {
00966     critical_selector_info_.reset(info);
00967   }
00968 
00973   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00974     critical_images_info_.reset(critical_images_info);
00975   }
00976 
00979   bool CriticalSelectorsEnabled() const;
00980 
00983   bool FlattenCssImportsEnabled() const {
00984     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00985             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00986              (CriticalSelectorsEnabled() ||
00987               options()->Enabled(RewriteOptions::kComputeCriticalCss))));
00988   }
00989 
00993   int num_inline_preview_images() const { return num_inline_preview_images_; }
00994 
00996   void increment_num_inline_preview_images();
00997 
01000   int num_flushed_early_pagespeed_resources() const {
01001     return num_flushed_early_pagespeed_resources_;
01002   }
01003 
01006   void increment_num_flushed_early_pagespeed_resources() {
01007     ++num_flushed_early_pagespeed_resources_;
01008   }
01009 
01012   void increment_async_events_count();
01013 
01015   void decrement_async_events_count();
01016 
01019   XhtmlStatus MimeTypeXhtmlStatus();
01020 
01021   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
01022   bool flushed_cached_html() { return flushed_cached_html_; }
01023 
01024   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
01025   bool flushing_cached_html() const { return flushing_cached_html_; }
01026 
01027   void set_flushed_early(bool x) { flushed_early_ = x; }
01028   bool flushed_early() const { return flushed_early_; }
01029 
01030   void set_flushing_early(bool x) { flushing_early_ = x; }
01031   bool flushing_early() const { return flushing_early_; }
01032 
01033   void set_is_lazyload_script_flushed(bool x) {
01034     is_lazyload_script_flushed_ = x;
01035   }
01036   bool is_lazyload_script_flushed() const {
01037     return is_lazyload_script_flushed_; }
01038 
01040   FlushEarlyInfo* flush_early_info();
01041 
01042   FlushEarlyRenderInfo* flush_early_render_info() const;
01043 
01046   void set_flush_early_render_info(
01047       FlushEarlyRenderInfo* flush_early_render_info);
01048 
01051   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
01052 
01055   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
01056 
01059   AbstractLogRecord* log_record();
01060 
01061   DomStatsFilter* dom_stats_filter() const {
01062     return dom_stats_filter_;
01063   }
01064 
01067   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01068 
01070   bool is_nested() const { return is_nested_; }
01071 
01075   bool MetadataRequested(const RequestHeaders& request_headers) const;
01076 
01078   bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; }
01079 
01092   bool Write(const ResourceVector& inputs,
01093              const StringPiece& contents,
01094              const ContentType* type,
01095              StringPiece charset,
01096              OutputResource* output);
01097 
01098   void set_defer_instrumentation_script(bool x) {
01099     defer_instrumentation_script_ = x;
01100   }
01101   bool defer_instrumentation_script() const {
01102     return defer_instrumentation_script_;
01103   }
01104 
01107   void set_num_initiated_rewrites(int64 x) {
01108     ScopedMutex lock(rewrite_mutex());
01109     num_initiated_rewrites_ = x;
01110   }
01111   int64 num_initiated_rewrites() const {
01112     ScopedMutex lock(rewrite_mutex());
01113     return num_initiated_rewrites_;
01114   }
01116   void set_num_detached_rewrites(int64 x) {
01117     ScopedMutex lock(rewrite_mutex());
01118     num_detached_rewrites_ = x;
01119   }
01120   int64 num_detached_rewrites() const {
01121     ScopedMutex lock(rewrite_mutex());
01122     return num_detached_rewrites_;
01123   }
01124 
01125   void set_pagespeed_query_params(StringPiece x) {
01126     x.CopyToString(&pagespeed_query_params_);
01127   }
01128   StringPiece pagespeed_query_params() const {
01129     return pagespeed_query_params_;
01130   }
01131 
01132   void set_pagespeed_option_cookies(StringPiece x) {
01133     x.CopyToString(&pagespeed_option_cookies_);
01134   }
01135   StringPiece pagespeed_option_cookies() const {
01136     return pagespeed_option_cookies_;
01137   }
01138 
01141   const GoogleString& CacheFragment() const;
01142 
01149   bool SetOrClearPageSpeedOptionCookies(const GoogleUrl& gurl,
01150                                         ResponseHeaders* response_headers);
01151 
01152  protected:
01153   virtual void DetermineEnabledFiltersImpl();
01154 
01155  private:
01156   friend class DistributedRewriteContextTest;
01157   friend class RewriteContext;
01158   friend class RewriteDriverTest;
01159   friend class RewriteTestBase;
01160   friend class ServerContextTest;
01161 
01162   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01163 
01165   bool ShouldDistributeFetch(const StringPiece& filter_id);
01166 
01178   bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id,
01179                        AsyncFetch* async_fetch);
01180 
01185   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01186                                Function* done)
01187       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01188 
01192   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01193                              Function* done)
01194       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01195 
01197   bool IsDone(WaitMode wait_mode, bool deadline_reached)
01198       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01199 
01202   bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
01203     return wait_mode == kWaitForShutDown ||
01204         (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
01205   }
01206 
01210   void FlushAsyncDone(int num_rewrites, Function* callback);
01211 
01216   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01217 
01219   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01220 
01223   void QueueFinishParseAfterFlush(Function* user_callback);
01224   void FinishParseAfterFlush(Function* user_callback);
01225 
01226   bool RewritesComplete() const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01227 
01230   void SetBaseUrlIfUnset(const StringPiece& new_base);
01231 
01234   void SetBaseUrlForFetch(const StringPiece& url);
01235 
01238   void SetDecodedUrlFromBase();
01239 
01241   AbstractMutex* rewrite_mutex() const LOCK_RETURNED(scheduler_->mutex()) {
01242     return scheduler_->mutex();
01243   }
01244 
01246   virtual void ParseTextInternal(const char* content, int size);
01247 
01249   bool ShouldSkipParsing();
01250 
01251   friend class ScanFilter;
01252 
01256   void RegisterRewriteFilter(RewriteFilter* filter);
01257 
01262   void EnableRewriteFilter(const char* id);
01263 
01269   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl,
01270                                            bool is_authorized_domain);
01271 
01272   void AddPreRenderFilters();
01273   void AddPostRenderFilters();
01274 
01276   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01277                                       const RewriteOptions* options_to_use,
01278                                       const UrlNamer* url_namer,
01279                                       ResourceNamer* name_out,
01280                                       OutputResourceKind* kind_out,
01281                                       RewriteFilter** filter_out,
01282                                       GoogleString* url_base,
01283                                       StringVector* urls) const;
01284 
01294   void WriteDomCohortIntoPropertyCache();
01295 
01297   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01298 
01305   void PossiblyPurgeCachedResponseAndReleaseDriver();
01306 
01308   void LogStats();
01309 
01325   bool PrepareShouldSignal() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01326   void SignalIfRequired(bool result_of_prepare_should_signal)
01327       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01328 
01340   bool base_was_set_;
01341 
01346   bool refs_before_base_;
01347 
01349   GoogleString containing_charset_;
01350 
01353   void PopulateRequestContext();
01354 
01355   bool filters_added_;
01356   bool externally_managed_;
01357 
01366   enum RefCategory {
01367     kRefUser,  
01368     kRefParsing,  
01369 
01373     kRefPendingRewrites,
01374 
01378     kRefDetachedRewrites,
01379 
01387     kRefDeletingRewrites,
01388 
01390     kRefFetchUserFacing,
01391 
01393     kRefFetchBackground,
01394 
01399     kRefAsyncEvents,
01400 
01401     kNumRefCategories
01402   };
01403 
01404   friend class CategorizedRefcount<RewriteDriver, RefCategory>;
01405 
01407   CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;
01408 
01410   void LastRefRemoved();
01411   StringPiece RefCategoryName(RefCategory cat);
01412 
01415   void DropReference(RefCategory cat);
01416 
01419   bool release_driver_;
01420 
01423   bool parsing_ GUARDED_BY(rewrite_mutex());
01424 
01428   WaitMode waiting_ GUARDED_BY(rewrite_mutex());
01429 
01431   bool waiting_deadline_reached_ GUARDED_BY(rewrite_mutex());
01432 
01437   bool fully_rewrite_on_flush_;
01438 
01441   bool fast_blocking_rewrite_;
01442 
01443   bool flush_requested_;
01444   bool flush_occurred_;
01445 
01447   bool flushed_cached_html_;
01448 
01450   bool flushing_cached_html_;
01451 
01454   bool flushed_early_;
01458   bool flushing_early_;
01459 
01462   bool is_lazyload_script_flushed_;
01463 
01467   bool write_property_cache_dom_cohort_;
01468 
01471   GoogleUrl base_url_;
01472 
01476   GoogleUrl decoded_base_url_;
01477 
01480   GoogleString fetch_url_;
01481 
01482   GoogleString user_agent_;
01483 
01484   LazyBool should_skip_parsing_;
01485 
01486   StringFilterMap resource_filter_map_;
01487 
01488   ResponseHeaders* response_headers_;
01489 
01492   scoped_ptr<const RequestHeaders> request_headers_;
01493 
01494   int status_code_; 
01495 
01498   typedef std::vector<RewriteContext*> RewriteContextVector;
01499   RewriteContextVector rewrites_; 
01500 
01503   int max_page_processing_delay_ms_;
01504 
01505   typedef std::set<RewriteContext*> RewriteContextSet;
01506 
01511   RewriteContextSet initiated_rewrites_ GUARDED_BY(rewrite_mutex());
01512 
01514   int64 num_initiated_rewrites_ GUARDED_BY(rewrite_mutex());
01515 
01523   int64 num_detached_rewrites_ GUARDED_BY(rewrite_mutex());
01524 
01532   RewriteContextSet detached_rewrites_ GUARDED_BY(rewrite_mutex());
01533 
01535   int possibly_quick_rewrites_ GUARDED_BY(rewrite_mutex());
01536 
01539   RewriteContextVector fetch_rewrites_;
01540 
01543   FileSystem* file_system_;
01544   ServerContext* server_context_;
01545   Scheduler* scheduler_;
01546   UrlAsyncFetcher* default_url_async_fetcher_; 
01547 
01551   UrlAsyncFetcher* url_async_fetcher_;
01552 
01556   UrlAsyncFetcher* distributed_async_fetcher_;
01557 
01560   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01561 
01562   DomStatsFilter* dom_stats_filter_;
01563   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01564 
01565   ScanFilter scan_filter_;
01566   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01567   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01568 
01571   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01572   PrimaryRewriteContextMap primary_rewrite_context_map_;
01573 
01574   HtmlResourceSlotSet slots_;
01575 
01576   scoped_ptr<RewriteOptions> options_;
01577 
01578   RewriteDriverPool* controlling_pool_; 
01579 
01581   scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
01582       cache_url_async_fetcher_async_op_hooks_;
01583 
01585   UrlSegmentEncoder default_encoder_;
01586 
01588   FilterList early_pre_render_filters_;
01590   FilterList pre_render_filters_;
01591 
01593   std::vector<ResourceUrlClaimant*> resource_claimants_;
01594 
01598   FilterVector filters_to_delete_;
01599 
01600   QueuedWorkerPool::Sequence* html_worker_;
01601   QueuedWorkerPool::Sequence* rewrite_worker_;
01602   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01603 
01604   Writer* writer_;
01605 
01608   FallbackPropertyPage* fallback_property_page_;
01609 
01611   bool owns_property_page_;
01612 
01614   UserAgentMatcher::DeviceType device_type_;
01615 
01616   scoped_ptr<CriticalLineInfo> critical_line_info_;
01617   scoped_ptr<CriticalKeys> beacon_critical_line_info_;
01618 
01619   scoped_ptr<SplitHtmlConfig> split_html_config_;
01620 
01623   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01624   scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
01625 
01626   scoped_ptr<CriticalCssResult> critical_css_result_;
01627 
01629   bool xhtml_mimetype_computed_;
01630   XhtmlStatus xhtml_status_ : 8;
01631 
01634   int num_inline_preview_images_;
01635 
01637   int num_flushed_early_pagespeed_resources_;
01638 
01640   int num_bytes_in_;
01641 
01642   DebugFilter* debug_filter_;
01643 
01644   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01645   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01646 
01647   bool can_rewrite_resources_;
01648   bool is_nested_;
01649 
01652   RequestContextPtr request_context_;
01653 
01655   int64 start_time_ms_;
01656 
01657   scoped_ptr<RequestProperties> request_properties_;
01658 
01662   static int initialized_count_;
01663 
01666   bool tried_to_distribute_fetch_;
01667 
01670   bool defer_instrumentation_script_;
01671 
01673   DownstreamCachePurger downstream_cache_purger_;
01674 
01676   GoogleString pagespeed_query_params_;
01677 
01679   GoogleString pagespeed_option_cookies_;
01680 
01681   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01682 };
01683 
01686 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01687  public:
01688   virtual ~OptionsAwareHTTPCacheCallback();
01689   virtual bool IsCacheValid(const GoogleString& key,
01690                             const ResponseHeaders& headers);
01691   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01692   virtual ResponseHeaders::VaryOption RespectVaryOnResources() const;
01693 
01697   static bool IsCacheValid(const GoogleString& key,
01698                            const RewriteOptions& rewrite_options,
01699                            const RequestContextPtr& request_ctx,
01700                            const ResponseHeaders& headers);
01701 
01702  protected:
01705   OptionsAwareHTTPCacheCallback(
01706       const RewriteOptions* rewrite_options,
01707       const RequestContextPtr& request_ctx);
01708 
01709  private:
01710   const RewriteOptions* rewrite_options_;
01711 
01712   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01713 };
01714 
01715 }  
01716 
01717 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines