Page Speed Optimization Libraries  1.7.30.2
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h"
00035 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/resource_slot.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/scan_filter.h"
00040 #include "net/instaweb/rewriter/public/server_context.h"
00041 #include "net/instaweb/util/public/basictypes.h"
00042 #include "net/instaweb/util/public/google_url.h"
00043 #include "net/instaweb/util/public/printf_format.h"
00044 #include "net/instaweb/util/public/queued_worker_pool.h"
00045 #include "net/instaweb/util/public/scheduler.h"
00046 #include "net/instaweb/util/public/scoped_ptr.h"
00047 #include "net/instaweb/util/public/string.h"
00048 #include "net/instaweb/util/public/string_util.h"
00049 #include "net/instaweb/util/public/thread_system.h"
00050 #include "net/instaweb/util/public/url_segment_encoder.h"
00051 #include "pagespeed/kernel/http/content_type.h"
00052 #include "pagespeed/kernel/util/categorized_refcount.h"
00053 
00054 namespace net_instaweb {
00055 
00056 class AbstractLogRecord;
00057 class AbstractMutex;
00058 class AbstractPropertyPage;
00059 class AddInstrumentationFilter;
00060 class AsyncFetch;
00061 class CriticalCssResult;
00062 class CriticalLineInfo;
00063 class DebugFilter;
00064 class DomStatsFilter;
00065 class DomainRewriteFilter;
00066 class FallbackPropertyPage;
00067 class FileSystem;
00068 class FlushEarlyInfo;
00069 class FlushEarlyRenderInfo;
00070 class Function;
00071 class HtmlFilter;
00072 class HtmlWriterFilter;
00073 class MessageHandler;
00074 class OutputResource;
00075 class PropertyPage;
00076 class RequestHeaders;
00077 class RequestProperties;
00078 class RequestTrace;
00079 class ResourceContext;
00080 class ResourceNamer;
00081 class ResponseHeaders;
00082 class RewriteContext;
00083 class RewriteDriverPool;
00084 class RewriteFilter;
00085 class SplitHtmlConfig;
00086 class Statistics;
00087 class UrlAsyncFetcher;
00088 class UrlLeftTrimFilter;
00089 class Writer;
00090 
00093 class RewriteDriver : public HtmlParse {
00094  public:
00096   enum CssResolutionStatus {
00097     kWriteFailed,
00098     kNoResolutionNeeded,
00099     kSuccess
00100   };
00101 
00103   enum WaitMode {
00104     kNoWait,  
00105     kWaitForCompletion,    
00106     kWaitForCachedRender,  
00107 
00108     kWaitForShutDown       
00109 
00110   };
00111 
00123   enum XhtmlStatus {
00124     kXhtmlUnknown,
00125     kIsXhtml,
00126     kIsNotXhtml
00127   };
00128 
00135   static const char* kPassThroughRequestAttributes[7];
00136 
00140   static const char kDomCohort[];
00142   static const char kBeaconCohort[];
00143 
00146   static const char kLastRequestTimestamp[];
00148   static const char kParseSizeLimitExceeded[];
00150   static const char kSubresourcesPropertyName[];
00152   static const char kStatusCodePropertyName[];
00153 
00154   RewriteDriver(MessageHandler* message_handler,
00155                 FileSystem* file_system,
00156                 UrlAsyncFetcher* url_async_fetcher);
00157 
00160   virtual ~RewriteDriver();
00161 
00164   RewriteDriver* Clone();
00165 
00171   void Clear();
00172 
00174   static void InitStats(Statistics* statistics);
00175 
00177   static void Initialize();
00178   static void Terminate();
00179 
00182   void SetServerContext(ServerContext* server_context);
00183 
00186   bool MayCacheExtendCss() const;
00187   bool MayCacheExtendImages() const;
00188   bool MayCacheExtendPdfs() const;
00189   bool MayCacheExtendScripts() const;
00190 
00191   const GoogleString& user_agent() const { return user_agent_; }
00192 
00193   void SetUserAgent(const StringPiece& user_agent_string);
00194 
00195   const RequestProperties* request_properties() const {
00196     return request_properties_.get();
00197   }
00198 
00200   void ClearRequestProperties();
00201 
00203   bool using_spdy() const { return request_context_->using_spdy(); }
00204 
00205   bool write_property_cache_dom_cohort() const {
00206     return write_property_cache_dom_cohort_;
00207   }
00208   void set_write_property_cache_dom_cohort(bool x) {
00209     write_property_cache_dom_cohort_ = x;
00210   }
00211 
00212   RequestContextPtr request_context() { return request_context_; }
00213   void set_request_context(const RequestContextPtr& x);
00214 
00217   RequestTrace* trace_context();
00218 
00221   void TracePrintf(const char* fmt, ...);
00222 
00225   ResponseHeaders* mutable_response_headers() {
00226     return flush_occurred_ ? NULL : response_headers_;
00227   }
00228 
00235   const ResponseHeaders* response_headers() {
00236     return response_headers_;
00237   }
00238 
00242   void set_response_headers_ptr(ResponseHeaders* headers) {
00243     response_headers_ = headers;
00244   }
00245 
00251   void SetRequestHeaders(const RequestHeaders& headers);
00252 
00253   const RequestHeaders* request_headers() const {
00254     return request_headers_.get();
00255   }
00256 
00257   UserAgentMatcher* user_agent_matcher() const {
00258     DCHECK(server_context() != NULL);
00259     return server_context()->user_agent_matcher();
00260   }
00261 
00266   void AddFilters();
00267 
00271   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00272 
00274   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00276   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00277 
00279   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00281   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00282 
00294   void AppendRewriteFilter(RewriteFilter* filter);
00295 
00298   void PrependRewriteFilter(RewriteFilter* filter);
00299 
00305   void SetWriter(Writer* writer);
00306 
00307   Writer* writer() const { return writer_; }
00308 
00337   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00338 
00356   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00357                             AsyncFetch* async_fetch);
00358 
00365   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00366                            RewriteFilter* filter,
00367                            AsyncFetch* async_fetch);
00368 
00375   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00376                                          RewriteFilter** filter) const;
00377 
00381   bool DecodeOutputResourceName(const GoogleUrl& url,
00382                                 ResourceNamer* name_out,
00383                                 OutputResourceKind* kind_out,
00384                                 RewriteFilter** filter_out) const;
00385 
00387   bool DecodeUrl(const GoogleUrl& url,
00388                  StringVector* decoded_urls) const;
00389 
00390   FileSystem* file_system() { return file_system_; }
00391   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00392 
00397   void SetSessionFetcher(UrlAsyncFetcher* f);
00398 
00399   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00401   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00402     distributed_async_fetcher_ = fetcher;
00403   }
00404 
00407   CacheUrlAsyncFetcher* CreateCacheFetcher();
00409   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00410 
00411   ServerContext* server_context() const { return server_context_; }
00412   Statistics* statistics() const;
00413 
00414   AddInstrumentationFilter* add_instrumentation_filter() {
00415     return add_instrumentation_filter_;
00416   }
00417 
00419   void set_custom_options(RewriteOptions* options) {
00420     set_options_for_pool(NULL, options);
00421   }
00422 
00425   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00426     controlling_pool_ = pool;
00427     options_.reset(options);
00428   }
00429 
00431   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00432 
00434   const RewriteOptions* options() const { return options_.get(); }
00435 
00439   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00440                             const ContentType& content_type);
00441 
00448   virtual void FinishParse();
00449 
00453   void FinishParseAsync(Function* callback);
00454 
00458   void InfoAt(const RewriteContext* context,
00459               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00460 
00465 
00472   OutputResourcePtr CreateOutputResourceFromResource(
00473       const StringPiece& filter_id,
00474       const UrlSegmentEncoder* encoder,
00475       const ResourceContext* data,
00476       const ResourcePtr& input_resource,
00477       OutputResourceKind kind);
00478 
00495   OutputResourcePtr CreateOutputResourceWithPath(
00496       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00497       const StringPiece& base_url, const StringPiece& filter_id,
00498       const StringPiece& name, OutputResourceKind kind);
00499 
00502   void PopulateResourceNamer(
00503     const StringPiece& filter_id,
00504     const StringPiece& name,
00505     ResourceNamer* full_name);
00506 
00510   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00511       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00512       const StringPiece& name, OutputResourceKind kind);
00513 
00516   OutputResourcePtr CreateOutputResourceWithMappedPath(
00517       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00518       const StringPiece& filter_id, const StringPiece& name,
00519       OutputResourceKind kind) {
00520     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00521                                         decoded_base_url_.AllExceptLeaf(),
00522                                         filter_id, name, kind);
00523   }
00524 
00527   OutputResourcePtr CreateOutputResourceWithPath(
00528       const StringPiece& path, const StringPiece& filter_id,
00529       const StringPiece& name, OutputResourceKind kind) {
00530     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00531                                         kind);
00532   }
00533 
00537   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00538 
00545   ResourcePtr CreateInputResource(const GoogleUrl& input_url,
00546                                   bool allow_unauthorized_domain);
00547 
00551   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00552       const StringPiece& absolute_url);
00553 
00558   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00559 
00566   bool MayRewriteUrl(const GoogleUrl& domain_url,
00567                      const GoogleUrl& input_url,
00568                      bool allow_unauthorized_domain,
00569                      bool* is_authorized_domain) const;
00570 
00574   const GoogleUrl& base_url() const { return base_url_; }
00575 
00577   StringPiece fetch_url() const { return fetch_url_; }
00578 
00582   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00583   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00584 
00586   bool IsHttps() const { return google_url().SchemeIs("https"); }
00587 
00588   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00589 
00591   RewriteFilter* FindFilter(const StringPiece& id) const;
00592 
00594   bool refs_before_base() { return refs_before_base_; }
00595 
00600   void set_refs_before_base() { refs_before_base_ = true; }
00601 
00606   StringPiece containing_charset() { return containing_charset_; }
00607   void set_containing_charset(const StringPiece charset) {
00608     charset.CopyToString(&containing_charset_);
00609   }
00610 
00612   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00613                               HtmlElement* elt,
00614                               HtmlElement::Attribute* attr);
00615 
00620   bool InitiateRewrite(RewriteContext* rewrite_context);
00621   void InitiateFetch(RewriteContext* rewrite_context);
00622 
00633   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00634 
00638   void ReportSlowRewrites(int num);
00639 
00644   void Cleanup();
00645 
00650   void AddUserReference();
00651 
00653   GoogleString ToString(bool show_detached_contexts);
00654   void PrintState(bool show_detached_contexts); 
00655   void PrintStateToErrorLog(bool show_detached_contexts); 
00656 
00659   void WaitForCompletion();
00660 
00667   void WaitForShutDown();
00668 
00672   void BoundedWaitFor(WaitMode mode, int64 timeout_ms);
00673 
00681   void set_fully_rewrite_on_flush(bool x) {
00682     fully_rewrite_on_flush_ = x;
00683   }
00684 
00686   bool fully_rewrite_on_flush() const {
00687     return fully_rewrite_on_flush_;
00688   }
00689 
00693   void set_fast_blocking_rewrite(bool x) {
00694     fast_blocking_rewrite_ = x;
00695   }
00696 
00697   bool fast_blocking_rewrite() const {
00698     return fast_blocking_rewrite_;
00699   }
00700 
00703   void EnableBlockingRewrite(RequestHeaders* request_headers);
00704 
00711   void set_externally_managed(bool x) { externally_managed_ = x; }
00712 
00716   void DetachFetch();
00717 
00720   void DetachedFetchComplete();
00721 
00725   void FetchComplete();
00726 
00732   void DeleteRewriteContext(RewriteContext* rewrite_context);
00733 
00734   int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }
00735 
00740   void set_max_page_processing_delay_ms(int x) {
00741     max_page_processing_delay_ms_ = x;
00742   }
00743   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00744 
00746   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00747   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00748 
00754   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00755                                           RewriteContext* candidate);
00756 
00761   void DeregisterForPartitionKey(
00762       const GoogleString& partition_key, RewriteContext* candidate);
00763 
00766   void RequestFlush() { flush_requested_ = true; }
00767   bool flush_requested() const { return flush_requested_; }
00768 
00780   void ExecuteFlushIfRequested();
00781 
00785   void ExecuteFlushIfRequestedAsync(Function* callback);
00786 
00795   virtual void Flush();
00796 
00800   void FlushAsync(Function* done);
00801 
00803   void AddRewriteTask(Function* task);
00804 
00807   void AddLowPriorityRewriteTask(Function* task);
00808 
00809   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00810   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00811   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00812     return low_priority_rewrite_worker_;
00813   }
00814 
00815   Scheduler* scheduler() { return scheduler_; }
00816 
00819   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00820   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00821 
00829   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00830                                      const StringPiece& output_css_base,
00831                                      const StringPiece& contents,
00832                                      Writer* writer,
00833                                      MessageHandler* handler);
00834 
00842   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00843                            const GoogleUrl& output_base,
00844                            bool* proxy_mode) const;
00845 
00853   void UpdatePropertyValueInDomCohort(
00854       AbstractPropertyPage* page,
00855       StringPiece property_name,
00856       StringPiece property_value);
00857 
00860   PropertyPage* property_page() const;
00865   FallbackPropertyPage* fallback_property_page() const {
00866     return fallback_property_page_;
00867   }
00869   void set_property_page(PropertyPage* page);
00871   void set_fallback_property_page(FallbackPropertyPage* page);
00873   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00874 
00876   const CriticalLineInfo* critical_line_info() const;
00877 
00880   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00881 
00882   const SplitHtmlConfig* split_html_config();
00883 
00884   CriticalCssResult* critical_css_result() const;
00887   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00888 
00890   CriticalImagesInfo* critical_images_info() const {
00891     return critical_images_info_.get();
00892   }
00893 
00898   CriticalSelectorInfo* critical_selector_info() {
00899     return critical_selector_info_.get();
00900   }
00901 
00905   void set_critical_selector_info(CriticalSelectorInfo* info) {
00906     critical_selector_info_.reset(info);
00907   }
00908 
00913   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00914     critical_images_info_.reset(critical_images_info);
00915   }
00916 
00919   bool CriticalSelectorsEnabled() const;
00920 
00923   bool FlattenCssImportsEnabled() const {
00924     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00925             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00926              (CriticalSelectorsEnabled() ||
00927               options()->Enabled(RewriteOptions::kComputeCriticalCss))));
00928   }
00929 
00933   int num_inline_preview_images() const { return num_inline_preview_images_; }
00934 
00936   void increment_num_inline_preview_images();
00937 
00940   int num_flushed_early_pagespeed_resources() const {
00941     return num_flushed_early_pagespeed_resources_;
00942   }
00943 
00946   void increment_num_flushed_early_pagespeed_resources() {
00947     ++num_flushed_early_pagespeed_resources_;
00948   }
00949 
00952   void increment_async_events_count();
00953 
00955   void decrement_async_events_count();
00956 
00959   XhtmlStatus MimeTypeXhtmlStatus();
00960 
00961   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
00962   bool flushed_cached_html() { return flushed_cached_html_; }
00963 
00964   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
00965   bool flushing_cached_html() const { return flushing_cached_html_; }
00966 
00967   void set_flushed_early(bool x) { flushed_early_ = x; }
00968   bool flushed_early() const { return flushed_early_; }
00969 
00970   void set_flushing_early(bool x) { flushing_early_ = x; }
00971   bool flushing_early() const { return flushing_early_; }
00972 
00973   void set_is_lazyload_script_flushed(bool x) {
00974     is_lazyload_script_flushed_ = x;
00975   }
00976   bool is_lazyload_script_flushed() const {
00977     return is_lazyload_script_flushed_; }
00978 
00980   FlushEarlyInfo* flush_early_info();
00981 
00982   FlushEarlyRenderInfo* flush_early_render_info() const;
00983 
00986   void set_flush_early_render_info(
00987       FlushEarlyRenderInfo* flush_early_render_info);
00988 
00991   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
00992 
00995   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
00996 
00999   AbstractLogRecord* log_record();
01000 
01001   DomStatsFilter* dom_stats_filter() const {
01002     return dom_stats_filter_;
01003   }
01004 
01007   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01008 
01010   bool is_nested() const { return is_nested_; }
01011 
01015   bool MetadataRequested(const RequestHeaders& request_headers) const;
01016 
01018   bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; }
01019 
01032   bool Write(const ResourceVector& inputs,
01033              const StringPiece& contents,
01034              const ContentType* type,
01035              StringPiece charset,
01036              OutputResource* output);
01037 
01038   void set_defer_instrumentation_script(bool x) {
01039     defer_instrumentation_script_ = x;
01040   }
01041   bool defer_instrumentation_script() const {
01042     return defer_instrumentation_script_;
01043   }
01044 
01045  protected:
01046   virtual void DetermineEnabledFiltersImpl();
01047 
01048  private:
01049   friend class DistributedRewriteContextTest;
01050   friend class RewriteContext;
01051   friend class RewriteDriverTest;
01052   friend class RewriteTestBase;
01053   friend class ServerContextTest;
01054 
01055   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01056 
01058   bool ShouldDistributeFetch(const StringPiece& filter_id);
01059 
01071   bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id,
01072                        AsyncFetch* async_fetch);
01073 
01079   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01080                                Function* done);
01081 
01086   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01087                              Function* done);
01088 
01090   bool IsDone(WaitMode wait_mode, bool deadline_reached);
01091 
01094   bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
01095     return wait_mode == kWaitForShutDown ||
01096         (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
01097   }
01098 
01102   void FlushAsyncDone(int num_rewrites, Function* callback);
01103 
01108   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01109 
01111   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01112 
01115   void QueueFinishParseAfterFlush(Function* user_callback);
01116   void FinishParseAfterFlush(Function* user_callback);
01117 
01119   bool RewritesComplete() const;
01120 
01123   void SetBaseUrlIfUnset(const StringPiece& new_base);
01124 
01127   void SetBaseUrlForFetch(const StringPiece& url);
01128 
01131   void SetDecodedUrlFromBase();
01132 
01134   AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); }
01135 
01137   virtual void ParseTextInternal(const char* content, int size);
01138 
01140   bool ShouldSkipParsing();
01141 
01142   friend class ScanFilter;
01143 
01147   void RegisterRewriteFilter(RewriteFilter* filter);
01148 
01153   void EnableRewriteFilter(const char* id);
01154 
01160   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl,
01161                                            bool is_authorized_domain);
01162 
01163   void AddPreRenderFilters();
01164   void AddPostRenderFilters();
01165 
01167   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01168                                       ResourceNamer* name_out,
01169                                       OutputResourceKind* kind_out,
01170                                       RewriteFilter** filter_out,
01171                                       GoogleString* url_base,
01172                                       StringVector* urls) const;
01173 
01183   void WriteDomCohortIntoPropertyCache();
01184 
01186   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01187 
01194   void PossiblyPurgeCachedResponseAndReleaseDriver();
01195 
01200   bool ShouldPurgeRewrittenResponse();
01201 
01204   static bool GetPurgeUrl(const GoogleUrl& google_url,
01205                           const RewriteOptions* options,
01206                           GoogleString* purge_url,
01207                           GoogleString* purge_method);
01208 
01210   void PurgeDownstreamCache(const GoogleString& purge_url,
01211                             const GoogleString& purge_method);
01212 
01214   void LogStats();
01215 
01232   bool PrepareShouldSignal();
01233   void SignalIfRequired(bool result_of_prepare_should_signal);
01234 
01246   bool base_was_set_;
01247 
01252   bool refs_before_base_;
01253 
01255   GoogleString containing_charset_;
01256 
01259   void PopulateRequestContext();
01260 
01261   bool filters_added_;
01262   bool externally_managed_;
01263 
01272   enum RefCategory {
01273     kRefUser,  
01274     kRefParsing,  
01275 
01279     kRefPendingRewrites,
01280 
01284     kRefDetachedRewrites,
01285 
01293     kRefDeletingRewrites,
01294 
01296     kRefFetchUserFacing,
01297 
01299     kRefFetchBackground,
01300 
01305     kRefAsyncEvents,
01306 
01307     kNumRefCategories
01308   };
01309 
01310   friend class CategorizedRefcount<RewriteDriver, RefCategory>;
01311 
01313   CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;
01314 
01316   void LastRefRemoved();
01317   StringPiece RefCategoryName(RefCategory cat);
01318 
01321   void DropReference(RefCategory cat);
01322 
01325   bool release_driver_;
01326 
01329   bool parsing_; 
01330 
01334   WaitMode waiting_; 
01335 
01337   bool waiting_deadline_reached_; 
01338 
01343   bool fully_rewrite_on_flush_;
01344 
01347   bool fast_blocking_rewrite_;
01348 
01349   bool flush_requested_;
01350   bool flush_occurred_;
01351 
01353   bool flushed_cached_html_;
01354 
01356   bool flushing_cached_html_;
01357 
01360   bool flushed_early_;
01364   bool flushing_early_;
01365 
01368   bool is_lazyload_script_flushed_;
01369 
01372   bool made_downstream_purge_attempt_;
01373 
01377   bool write_property_cache_dom_cohort_;
01378 
01381   GoogleUrl base_url_;
01382 
01386   GoogleUrl decoded_base_url_;
01387 
01390   GoogleString fetch_url_;
01391 
01392   GoogleString user_agent_;
01393 
01394   LazyBool should_skip_parsing_;
01395 
01396   StringFilterMap resource_filter_map_;
01397 
01398   ResponseHeaders* response_headers_;
01399 
01402   scoped_ptr<const RequestHeaders> request_headers_;
01403 
01404   int status_code_; 
01405 
01408   typedef std::vector<RewriteContext*> RewriteContextVector;
01409   RewriteContextVector rewrites_; 
01410 
01413   int max_page_processing_delay_ms_;
01414 
01415   typedef std::set<RewriteContext*> RewriteContextSet;
01416 
01421   RewriteContextSet initiated_rewrites_; 
01422 
01424   int64 num_initiated_rewrites_; 
01425 
01433   int64 num_detached_rewrites_; 
01434 
01442   RewriteContextSet detached_rewrites_; 
01443 
01445   int possibly_quick_rewrites_; 
01446 
01449   RewriteContextVector fetch_rewrites_;
01450 
01453   FileSystem* file_system_;
01454   ServerContext* server_context_;
01455   Scheduler* scheduler_;
01456   UrlAsyncFetcher* default_url_async_fetcher_; 
01457 
01461   UrlAsyncFetcher* url_async_fetcher_;
01462 
01466   UrlAsyncFetcher* distributed_async_fetcher_;
01467 
01470   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01471 
01472   AddInstrumentationFilter* add_instrumentation_filter_;
01473   DomStatsFilter* dom_stats_filter_;
01474   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01475 
01476   ScanFilter scan_filter_;
01477   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01478   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01479 
01482   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01483   PrimaryRewriteContextMap primary_rewrite_context_map_;
01484 
01485   HtmlResourceSlotSet slots_;
01486 
01487   scoped_ptr<RewriteOptions> options_;
01488 
01489   RewriteDriverPool* controlling_pool_; 
01490 
01492   scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
01493       cache_url_async_fetcher_async_op_hooks_;
01494 
01496   UrlSegmentEncoder default_encoder_;
01497 
01499   FilterList early_pre_render_filters_;
01501   FilterList pre_render_filters_;
01502 
01506   FilterVector filters_to_delete_;
01507 
01508   QueuedWorkerPool::Sequence* html_worker_;
01509   QueuedWorkerPool::Sequence* rewrite_worker_;
01510   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01511 
01512   Writer* writer_;
01513 
01516   FallbackPropertyPage* fallback_property_page_;
01517 
01519   bool owns_property_page_;
01520 
01522   UserAgentMatcher::DeviceType device_type_;
01523 
01524   scoped_ptr<CriticalLineInfo> critical_line_info_;
01525 
01526   scoped_ptr<SplitHtmlConfig> split_html_config_;
01527 
01530   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01531   scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
01532 
01533   scoped_ptr<CriticalCssResult> critical_css_result_;
01534 
01536   bool xhtml_mimetype_computed_;
01537   XhtmlStatus xhtml_status_ : 8;
01538 
01541   int num_inline_preview_images_;
01542 
01544   int num_flushed_early_pagespeed_resources_;
01545 
01547   int num_bytes_in_;
01548 
01549   DebugFilter* debug_filter_;
01550 
01551   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01552   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01553 
01554   bool can_rewrite_resources_;
01555   bool is_nested_;
01556 
01559   RequestContextPtr request_context_;
01560 
01562   int64 start_time_ms_;
01563 
01564   scoped_ptr<RequestProperties> request_properties_;
01565 
01569   static int initialized_count_;
01570 
01573   bool tried_to_distribute_fetch_;
01574 
01577   bool defer_instrumentation_script_;
01578 
01579   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01580 };
01581 
01584 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01585  public:
01586   virtual ~OptionsAwareHTTPCacheCallback();
01587   virtual bool IsCacheValid(const GoogleString& key,
01588                             const ResponseHeaders& headers);
01589   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01590 
01594   static bool IsCacheValid(const GoogleString& key,
01595                            const RewriteOptions& rewrite_options,
01596                            const RequestContextPtr& request_ctx,
01597                            const ResponseHeaders& headers);
01598 
01599  protected:
01602   OptionsAwareHTTPCacheCallback(
01603       const RewriteOptions* rewrite_options,
01604       const RequestContextPtr& request_ctx);
01605 
01606  private:
01607   const RewriteOptions* rewrite_options_;
01608 
01609   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01610 };
01611 
01612 }  
01613 
01614 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines