Page Speed Optimization Libraries  1.6.29.3
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h"
00035 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/resource_slot.h"
00038 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
00039 #include "net/instaweb/rewriter/public/rewrite_options.h"
00040 #include "net/instaweb/rewriter/public/scan_filter.h"
00041 #include "net/instaweb/rewriter/public/server_context.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/google_url.h"
00044 #include "net/instaweb/util/public/printf_format.h"
00045 #include "net/instaweb/util/public/queued_worker_pool.h"
00046 #include "net/instaweb/util/public/scheduler.h"
00047 #include "net/instaweb/util/public/scoped_ptr.h"
00048 #include "net/instaweb/util/public/string.h"
00049 #include "net/instaweb/util/public/string_util.h"
00050 #include "net/instaweb/util/public/thread_system.h"
00051 #include "net/instaweb/util/public/url_segment_encoder.h"
00052 #include "pagespeed/kernel/http/content_type.h"
00053 
00054 namespace net_instaweb {
00055 
00056 class AbstractLogRecord;
00057 class AbstractMutex;
00058 class AbstractPropertyPage;
00059 class AddInstrumentationFilter;
00060 class AsyncFetch;
00061 class CommonFilter;
00062 class CriticalCssResult;
00063 class CriticalLineInfo;
00064 class DebugFilter;
00065 class DomStatsFilter;
00066 class DomainRewriteFilter;
00067 class FallbackPropertyPage;
00068 class FileSystem;
00069 class FlushEarlyInfo;
00070 class FlushEarlyRenderInfo;
00071 class Function;
00072 class HtmlFilter;
00073 class HtmlWriterFilter;
00074 class MessageHandler;
00075 class OutputResource;
00076 class PropertyPage;
00077 class RequestHeaders;
00078 class RequestProperties;
00079 class RequestTrace;
00080 class ResourceContext;
00081 class ResourceNamer;
00082 class ResponseHeaders;
00083 class RewriteContext;
00084 class RewriteDriverPool;
00085 class RewriteFilter;
00086 class ScopedMutex;
00087 class Statistics;
00088 class UrlAsyncFetcher;
00089 class UrlLeftTrimFilter;
00090 class Writer;
00091 
00094 class RewriteDriver : public HtmlParse {
00095  public:
00097   enum CssResolutionStatus {
00098     kWriteFailed,
00099     kNoResolutionNeeded,
00100     kSuccess
00101   };
00102 
00104   enum WaitMode {
00105     kNoWait,  
00106     kWaitForCompletion,    
00107     kWaitForCachedRender,  
00108 
00109     kWaitForShutDown       
00110 
00111   };
00112 
00124   enum XhtmlStatus {
00125     kXhtmlUnknown,
00126     kIsXhtml,
00127     kIsNotXhtml
00128   };
00129 
00136   static const char* kPassThroughRequestAttributes[7];
00137 
00141   static const char kDomCohort[];
00143   static const char kBeaconCohort[];
00144 
00147   static const char kLastRequestTimestamp[];
00149   static const char kParseSizeLimitExceeded[];
00151   static const char kSubresourcesPropertyName[];
00153   static const char kStatusCodePropertyName[];
00156   static const char kXPsaBlockingRewriteModeSlow[];
00157 
00158   RewriteDriver(MessageHandler* message_handler,
00159                 FileSystem* file_system,
00160                 UrlAsyncFetcher* url_async_fetcher);
00161 
00164   virtual ~RewriteDriver();
00165 
00168   RewriteDriver* Clone();
00169 
00175   void Clear();
00176 
00178   static void InitStats(Statistics* statistics);
00179 
00181   static void Initialize();
00182   static void Terminate();
00183 
00186   void SetServerContext(ServerContext* server_context);
00187 
00190   bool MayCacheExtendCss() const;
00191   bool MayCacheExtendImages() const;
00192   bool MayCacheExtendPdfs() const;
00193   bool MayCacheExtendScripts() const;
00194 
00195   const GoogleString& user_agent() const { return user_agent_; }
00196 
00197   void SetUserAgent(const StringPiece& user_agent_string);
00198 
00199   const RequestProperties* request_properties() const {
00200     return request_properties_.get();
00201   }
00202 
00204   void ClearRequestProperties();
00205 
00207   bool using_spdy() const { return request_context_->using_spdy(); }
00208 
00209   bool write_property_cache_dom_cohort() const {
00210     return write_property_cache_dom_cohort_;
00211   }
00212   void set_write_property_cache_dom_cohort(bool x) {
00213     write_property_cache_dom_cohort_ = x;
00214   }
00215 
00216   RequestContextPtr request_context() { return request_context_; }
00217   void set_request_context(const RequestContextPtr& x);
00218 
00221   RequestTrace* trace_context();
00222 
00225   void TracePrintf(const char* fmt, ...);
00226 
00229   ResponseHeaders* mutable_response_headers() {
00230     return flush_occurred_ ? NULL : response_headers_;
00231   }
00232 
00239   const ResponseHeaders* response_headers() {
00240     return response_headers_;
00241   }
00242 
00246   void set_response_headers_ptr(ResponseHeaders* headers) {
00247     response_headers_ = headers;
00248   }
00249 
00255   void SetRequestHeaders(const RequestHeaders& headers);
00256 
00257   const RequestHeaders* request_headers() const {
00258     return request_headers_.get();
00259   }
00260 
00261   UserAgentMatcher* user_agent_matcher() const {
00262     DCHECK(server_context() != NULL);
00263     return server_context()->user_agent_matcher();
00264   }
00265 
00270   void AddFilters();
00271 
00275   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00276 
00278   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00280   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00281 
00283   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00285   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00286 
00298   void AppendRewriteFilter(RewriteFilter* filter);
00299 
00302   void PrependRewriteFilter(RewriteFilter* filter);
00303 
00309   void SetWriter(Writer* writer);
00310 
00311   Writer* writer() const { return writer_; }
00312 
00341   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00342 
00360   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00361                             AsyncFetch* async_fetch);
00362 
00369   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00370                            RewriteFilter* filter,
00371                            AsyncFetch* async_fetch);
00372 
00379   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00380                                          RewriteFilter** filter) const;
00381 
00385   bool DecodeOutputResourceName(const GoogleUrl& url,
00386                                 ResourceNamer* name_out,
00387                                 OutputResourceKind* kind_out,
00388                                 RewriteFilter** filter_out) const;
00389 
00391   bool DecodeUrl(const GoogleUrl& url,
00392                  StringVector* decoded_urls) const;
00393 
00394   FileSystem* file_system() { return file_system_; }
00395   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00396 
00401   void SetSessionFetcher(UrlAsyncFetcher* f);
00402 
00403   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00405   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00406     distributed_async_fetcher_ = fetcher;
00407   }
00408 
00411   CacheUrlAsyncFetcher* CreateCacheFetcher();
00413   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00414 
00415   ServerContext* server_context() const { return server_context_; }
00416   Statistics* statistics() const;
00417 
00418   AddInstrumentationFilter* add_instrumentation_filter() {
00419     return add_instrumentation_filter_;
00420   }
00421 
00423   void set_custom_options(RewriteOptions* options) {
00424     set_options_for_pool(NULL, options);
00425   }
00426 
00429   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00430     controlling_pool_ = pool;
00431     options_.reset(options);
00432   }
00433 
00435   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00436 
00438   const RewriteOptions* options() const { return options_.get(); }
00439 
00441   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00442                             const ContentType& content_type);
00443 
00450   virtual void FinishParse();
00451 
00454   void FinishParseAsync(Function* callback);
00455 
00459   void InfoAt(const RewriteContext* context,
00460               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00461 
00466 
00473   OutputResourcePtr CreateOutputResourceFromResource(
00474       const StringPiece& filter_id,
00475       const UrlSegmentEncoder* encoder,
00476       const ResourceContext* data,
00477       const ResourcePtr& input_resource,
00478       OutputResourceKind kind);
00479 
00496   OutputResourcePtr CreateOutputResourceWithPath(
00497       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00498       const StringPiece& base_url, const StringPiece& filter_id,
00499       const StringPiece& name, OutputResourceKind kind);
00500 
00503   void PopulateResourceNamer(
00504     const StringPiece& filter_id,
00505     const StringPiece& name,
00506     ResourceNamer* full_name);
00507 
00511   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00512       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00513       const StringPiece& name, OutputResourceKind kind);
00514 
00517   OutputResourcePtr CreateOutputResourceWithMappedPath(
00518       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00519       const StringPiece& filter_id, const StringPiece& name,
00520       OutputResourceKind kind) {
00521     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00522                                         decoded_base_url_.AllExceptLeaf(),
00523                                         filter_id, name, kind);
00524   }
00525 
00528   OutputResourcePtr CreateOutputResourceWithPath(
00529       const StringPiece& path, const StringPiece& filter_id,
00530       const StringPiece& name, OutputResourceKind kind) {
00531     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00532                                         kind);
00533   }
00534 
00538   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00539 
00543   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00544       const StringPiece& absolute_url);
00545 
00550   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00551 
00555   bool MayRewriteUrl(const GoogleUrl& domain_url,
00556                      const GoogleUrl& input_url) const;
00557 
00561   const GoogleUrl& base_url() const { return base_url_; }
00562 
00564   StringPiece fetch_url() const { return fetch_url_; }
00565 
00569   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00570   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00571 
00573   bool IsHttps() const { return google_url().SchemeIs("https"); }
00574 
00575   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00576 
00578   RewriteFilter* FindFilter(const StringPiece& id) const;
00579 
00581   bool refs_before_base() { return refs_before_base_; }
00582 
00587   void set_refs_before_base() { refs_before_base_ = true; }
00588 
00593   StringPiece containing_charset() { return containing_charset_; }
00594   void set_containing_charset(const StringPiece charset) {
00595     charset.CopyToString(&containing_charset_);
00596   }
00597 
00599   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00600                               HtmlElement* elt,
00601                               HtmlElement::Attribute* attr);
00602 
00607   bool InitiateRewrite(RewriteContext* rewrite_context);
00608   void InitiateFetch(RewriteContext* rewrite_context);
00609 
00620   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00621 
00625   void ReportSlowRewrites(int num);
00626 
00631   void Cleanup();
00632 
00634   GoogleString ToString(bool show_detached_contexts);
00635   void PrintState(bool show_detached_contexts); 
00636   void PrintStateToErrorLog(bool show_detached_contexts); 
00637 
00640   void WaitForCompletion();
00641 
00648   void WaitForShutDown();
00649 
00653   void BoundedWaitFor(WaitMode mode, int64 timeout_ms);
00654 
00662   void set_fully_rewrite_on_flush(bool x) {
00663     fully_rewrite_on_flush_ = x;
00664   }
00665 
00667   bool fully_rewrite_on_flush() const {
00668     return fully_rewrite_on_flush_;
00669   }
00670 
00674   void set_fast_blocking_rewrite(bool x) {
00675     fast_blocking_rewrite_ = x;
00676   }
00677 
00678   bool fast_blocking_rewrite() const {
00679     return fast_blocking_rewrite_;
00680   }
00681 
00684   void EnableBlockingRewrite(RequestHeaders* request_headers);
00685 
00692   void set_externally_managed(bool x) { externally_managed_ = x; }
00693 
00697   void DetachFetch();
00698 
00701   void DetachedFetchComplete();
00702 
00706   void FetchComplete();
00707 
00713   void DeleteRewriteContext(RewriteContext* rewrite_context);
00714 
00715   int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }
00716 
00721   void set_max_page_processing_delay_ms(int x) {
00722     max_page_processing_delay_ms_ = x;
00723   }
00724   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00725 
00727   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00728   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00729 
00735   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00736                                           RewriteContext* candidate);
00737 
00742   void DeregisterForPartitionKey(
00743       const GoogleString& partition_key, RewriteContext* candidate);
00744 
00747   void RequestFlush() { flush_requested_ = true; }
00748   bool flush_requested() const { return flush_requested_; }
00749 
00761   void ExecuteFlushIfRequested();
00762 
00766   void ExecuteFlushIfRequestedAsync(Function* callback);
00767 
00776   virtual void Flush();
00777 
00781   void FlushAsync(Function* done);
00782 
00784   void AddRewriteTask(Function* task);
00785 
00788   void AddLowPriorityRewriteTask(Function* task);
00789 
00790   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00791   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00792   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00793     return low_priority_rewrite_worker_;
00794   }
00795 
00796   Scheduler* scheduler() { return scheduler_; }
00797 
00800   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00801   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00802 
00810   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00811                                      const StringPiece& output_css_base,
00812                                      const StringPiece& contents,
00813                                      Writer* writer,
00814                                      MessageHandler* handler);
00815 
00823   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00824                            const GoogleUrl& output_base,
00825                            bool* proxy_mode) const;
00826 
00834   void UpdatePropertyValueInDomCohort(
00835       AbstractPropertyPage* page,
00836       StringPiece property_name,
00837       StringPiece property_value);
00838 
00839   void set_client_id(const StringPiece& id) { client_id_ = id.as_string(); }
00840   const GoogleString& client_id() const { return client_id_; }
00841 
00844   PropertyPage* property_page() const;
00849   FallbackPropertyPage* fallback_property_page() const {
00850     return fallback_property_page_;
00851   }
00853   void set_property_page(PropertyPage* page);
00855   void set_fallback_property_page(FallbackPropertyPage* page);
00857   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00858 
00860   const CriticalLineInfo* critical_line_info() const;
00861 
00864   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00865 
00866   CriticalCssResult* critical_css_result() const;
00869   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00870 
00872   CriticalImagesInfo* critical_images_info() const {
00873     return critical_images_info_.get();
00874   }
00875 
00880   CriticalSelectorInfo* critical_selector_info() {
00881     return critical_selector_info_.get();
00882   }
00883 
00887   void set_critical_selector_info(CriticalSelectorInfo* info) {
00888     critical_selector_info_.reset(info);
00889   }
00890 
00895   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00896     critical_images_info_.reset(critical_images_info);
00897   }
00898 
00901   bool CriticalSelectorsEnabled() const {
00902     return (options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) &&
00903             server_context()->factory()->UseSelectorFilterForCriticalCss());
00904   }
00905 
00908   bool FlattenCssImportsEnabled() const {
00909     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00910             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00911              (CriticalSelectorsEnabled() ||
00912               options()->Enabled(RewriteOptions::kComputeCriticalCss))));
00913   }
00914 
00918   int num_inline_preview_images() const { return num_inline_preview_images_; }
00919 
00921   void increment_num_inline_preview_images();
00922 
00925   int num_flushed_early_pagespeed_resources() const {
00926     return num_flushed_early_pagespeed_resources_;
00927   }
00928 
00931   void increment_num_flushed_early_pagespeed_resources() {
00932     ++num_flushed_early_pagespeed_resources_;
00933   }
00934 
00938   void increment_async_events_count();
00939 
00941   void decrement_async_events_count();
00942 
00945   XhtmlStatus MimeTypeXhtmlStatus();
00946 
00947   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
00948   bool flushed_cached_html() { return flushed_cached_html_; }
00949 
00950   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
00951   bool flushing_cached_html() const { return flushing_cached_html_; }
00952 
00953   void set_flushed_early(bool x) { flushed_early_ = x; }
00954   bool flushed_early() const { return flushed_early_; }
00955 
00956   void set_flushing_early(bool x) { flushing_early_ = x; }
00957   bool flushing_early() const { return flushing_early_; }
00958 
00959   void set_is_lazyload_script_flushed(bool x) {
00960     is_lazyload_script_flushed_ = x;
00961   }
00962   bool is_lazyload_script_flushed() const {
00963     return is_lazyload_script_flushed_; }
00964 
00966   FlushEarlyInfo* flush_early_info();
00967 
00968   FlushEarlyRenderInfo* flush_early_render_info() const;
00969 
00972   void set_flush_early_render_info(
00973       FlushEarlyRenderInfo* flush_early_render_info);
00974 
00975   void set_serve_blink_non_critical(bool x) { serve_blink_non_critical_ = x; }
00976   bool serve_blink_non_critical() const { return serve_blink_non_critical_; }
00977 
00978   void set_is_blink_request(bool x) { is_blink_request_ = x; }
00979   bool is_blink_request() const { return is_blink_request_; }
00980 
00983   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
00984 
00987   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
00988 
00991   AbstractLogRecord* log_record();
00992 
00993   DomStatsFilter* dom_stats_filter() const {
00994     return dom_stats_filter_;
00995   }
00996 
00999   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01000 
01002   bool is_nested() const { return is_nested_; }
01003 
01007   bool MetadataRequested(const RequestHeaders& request_headers) const;
01008 
01021   bool Write(const ResourceVector& inputs,
01022              const StringPiece& contents,
01023              const ContentType* type,
01024              StringPiece charset,
01025              OutputResource* output);
01026 
01027  protected:
01028   virtual void DetermineEnabledFilters();
01029 
01030  private:
01031   friend class RewriteContext;
01032   friend class RewriteDriverTest;
01033   friend class RewriteTestBase;
01034   friend class ServerContextTest;
01035 
01036   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01037 
01042   void FetchCompleteImpl(bool signal, ScopedMutex* lock);
01043 
01048   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01049                                Function* done);
01050 
01054   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01055                              Function* done);
01056 
01058   bool IsDone(WaitMode wait_mode, bool deadline_reached);
01059 
01062   bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
01063     return wait_mode == kWaitForShutDown ||
01064         (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
01065   }
01066 
01070   void FlushAsyncDone(int num_rewrites, Function* callback);
01071 
01076   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01077 
01079   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01080 
01083   void QueueFinishParseAfterFlush(Function* user_callback);
01084   void FinishParseAfterFlush(Function* user_callback);
01085 
01087   bool RewritesComplete() const;
01088 
01093   bool HaveBackgroundFetchRewrite() const;
01094 
01097   void SetBaseUrlIfUnset(const StringPiece& new_base);
01098 
01101   void SetBaseUrlForFetch(const StringPiece& url);
01102 
01105   void SetDecodedUrlFromBase();
01106 
01108   AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); }
01109 
01111   virtual void ParseTextInternal(const char* content, int size);
01112 
01114   bool ShouldSkipParsing();
01115 
01116   friend class ScanFilter;
01117 
01121   void AddCommonFilter(CommonFilter* filter);
01122 
01126   void RegisterRewriteFilter(RewriteFilter* filter);
01127 
01132   void EnableRewriteFilter(const char* id);
01133 
01137   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl);
01138 
01139   void AddPreRenderFilters();
01140   void AddPostRenderFilters();
01141 
01143   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01144                                       ResourceNamer* name_out,
01145                                       OutputResourceKind* kind_out,
01146                                       RewriteFilter** filter_out,
01147                                       GoogleString* url_base,
01148                                       StringVector* urls) const;
01149 
01159   void WriteDomCohortIntoPropertyCache();
01160 
01161   void FinalizeFilterLogging();
01162 
01164   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01165 
01172   void PossiblyPurgeCachedResponseAndReleaseDriver();
01173 
01178   bool ShouldPurgeRewrittenResponse();
01179 
01182   static bool GetPurgeUrl(const GoogleUrl& google_url,
01183                           const RewriteOptions* options,
01184                           GoogleString* purge_url,
01185                           GoogleString* purge_method);
01186 
01188   void PurgeDownstreamCache(const GoogleString& purge_url,
01189                             const GoogleString& purge_method);
01190 
01192   void LogStats();
01193 
01205   bool base_was_set_;
01206 
01211   bool refs_before_base_;
01212 
01214   GoogleString containing_charset_;
01215 
01216   bool filters_added_;
01217   bool externally_managed_;
01218 
01222   bool fetch_queued_; 
01223 
01228   bool fetch_detached_; 
01229 
01236   bool detached_fetch_main_path_complete_;
01237   bool detached_fetch_detached_path_complete_;
01238 
01241   bool parsing_; 
01242 
01246   WaitMode waiting_; 
01247 
01252   bool fully_rewrite_on_flush_;
01253 
01256   bool fast_blocking_rewrite_;
01257 
01260   bool cleanup_on_fetch_complete_;
01261 
01262   bool flush_requested_;
01263   bool flush_occurred_;
01264 
01266   bool flushed_cached_html_;
01267 
01269   bool flushing_cached_html_;
01270 
01273   bool flushed_early_;
01277   bool flushing_early_;
01278 
01281   bool is_lazyload_script_flushed_;
01282 
01284   bool release_driver_;
01285 
01288   bool made_downstream_purge_attempt_;
01289 
01293   bool write_property_cache_dom_cohort_;
01294 
01302   int rewrites_to_delete_; 
01303 
01306   GoogleUrl base_url_;
01307 
01311   GoogleUrl decoded_base_url_;
01312 
01315   GoogleString fetch_url_;
01316 
01317   GoogleString user_agent_;
01318 
01319   LazyBool should_skip_parsing_;
01320 
01321   StringFilterMap resource_filter_map_;
01322 
01323   ResponseHeaders* response_headers_;
01324 
01327   scoped_ptr<const RequestHeaders> request_headers_;
01328 
01329   int status_code_; 
01330 
01333   typedef std::vector<RewriteContext*> RewriteContextVector;
01334   RewriteContextVector rewrites_; 
01335 
01338   int max_page_processing_delay_ms_;
01339 
01340   typedef std::set<RewriteContext*> RewriteContextSet;
01341 
01346   RewriteContextSet initiated_rewrites_; 
01347 
01349   int64 num_initiated_rewrites_; 
01350 
01358   RewriteContextSet detached_rewrites_; 
01359 
01362   int64 num_detached_rewrites_; 
01363 
01368   int pending_rewrites_; 
01369 
01371   int possibly_quick_rewrites_; 
01372 
01377   int pending_async_events_; 
01378 
01381   FileSystem* file_system_;
01382   ServerContext* server_context_;
01383   Scheduler* scheduler_;
01384   UrlAsyncFetcher* default_url_async_fetcher_; 
01385 
01389   UrlAsyncFetcher* url_async_fetcher_;
01390 
01394   UrlAsyncFetcher* distributed_async_fetcher_;
01395 
01398   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01399 
01400   AddInstrumentationFilter* add_instrumentation_filter_;
01401   DomStatsFilter* dom_stats_filter_;
01402   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01403 
01404   ScanFilter scan_filter_;
01405   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01406   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01407 
01410   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01411   PrimaryRewriteContextMap primary_rewrite_context_map_;
01412 
01413   HtmlResourceSlotSet slots_;
01414 
01415   scoped_ptr<RewriteOptions> options_;
01416 
01417   RewriteDriverPool* controlling_pool_; 
01418 
01420   scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
01421       cache_url_async_fetcher_async_op_hooks_;
01422 
01424   UrlSegmentEncoder default_encoder_;
01425 
01427   FilterList early_pre_render_filters_;
01429   FilterList pre_render_filters_;
01430 
01434   FilterVector filters_to_delete_;
01435 
01436   QueuedWorkerPool::Sequence* html_worker_;
01437   QueuedWorkerPool::Sequence* rewrite_worker_;
01438   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01439 
01440   Writer* writer_;
01441 
01443   GoogleString client_id_;
01444 
01447   FallbackPropertyPage* fallback_property_page_;
01448 
01450   bool owns_property_page_;
01451 
01453   UserAgentMatcher::DeviceType device_type_;
01454 
01455   scoped_ptr<CriticalLineInfo> critical_line_info_;
01456 
01459   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01460   scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
01461 
01462   scoped_ptr<CriticalCssResult> critical_css_result_;
01463 
01465   bool xhtml_mimetype_computed_;
01466   XhtmlStatus xhtml_status_ : 8;
01467 
01470   int num_inline_preview_images_;
01471 
01473   int num_flushed_early_pagespeed_resources_;
01474 
01476   int num_bytes_in_;
01477 
01478   DebugFilter* debug_filter_;
01479 
01480   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01481   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01482 
01486   bool serve_blink_non_critical_;
01488   bool is_blink_request_;
01489   bool can_rewrite_resources_;
01490   bool is_nested_;
01491 
01494   RequestContextPtr request_context_;
01495 
01497   int64 start_time_ms_;
01498 
01499   scoped_ptr<RequestProperties> request_properties_;
01500 
01504   static int initialized_count_;
01505 
01506   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01507 };
01508 
01511 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01512  public:
01513   virtual ~OptionsAwareHTTPCacheCallback();
01514   virtual bool IsCacheValid(const GoogleString& key,
01515                             const ResponseHeaders& headers);
01516   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01517  protected:
01520   OptionsAwareHTTPCacheCallback(
01521       const RewriteOptions* rewrite_options,
01522       const RequestContextPtr& request_ctx);
01523 
01524  private:
01525   const RewriteOptions* rewrite_options_;
01526 
01527   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01528 };
01529 
01530 }  
01531 
01532 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines