Page Speed Optimization Libraries  1.7.30.1
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h"
00035 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/resource_slot.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/scan_filter.h"
00040 #include "net/instaweb/rewriter/public/server_context.h"
00041 #include "net/instaweb/util/public/basictypes.h"
00042 #include "net/instaweb/util/public/google_url.h"
00043 #include "net/instaweb/util/public/printf_format.h"
00044 #include "net/instaweb/util/public/queued_worker_pool.h"
00045 #include "net/instaweb/util/public/scheduler.h"
00046 #include "net/instaweb/util/public/scoped_ptr.h"
00047 #include "net/instaweb/util/public/string.h"
00048 #include "net/instaweb/util/public/string_util.h"
00049 #include "net/instaweb/util/public/thread_system.h"
00050 #include "net/instaweb/util/public/url_segment_encoder.h"
00051 #include "pagespeed/kernel/http/content_type.h"
00052 #include "pagespeed/kernel/util/categorized_refcount.h"
00053 
00054 namespace net_instaweb {
00055 
00056 class AbstractLogRecord;
00057 class AbstractMutex;
00058 class AbstractPropertyPage;
00059 class AddInstrumentationFilter;
00060 class AsyncFetch;
00061 class CommonFilter;
00062 class CriticalCssResult;
00063 class CriticalLineInfo;
00064 class DebugFilter;
00065 class DomStatsFilter;
00066 class DomainRewriteFilter;
00067 class FallbackPropertyPage;
00068 class FileSystem;
00069 class FlushEarlyInfo;
00070 class FlushEarlyRenderInfo;
00071 class Function;
00072 class HtmlFilter;
00073 class HtmlWriterFilter;
00074 class MessageHandler;
00075 class OutputResource;
00076 class PropertyPage;
00077 class RequestHeaders;
00078 class RequestProperties;
00079 class RequestTrace;
00080 class ResourceContext;
00081 class ResourceNamer;
00082 class ResponseHeaders;
00083 class RewriteContext;
00084 class RewriteDriverPool;
00085 class RewriteFilter;
00086 class SplitHtmlConfig;
00087 class Statistics;
00088 class UrlAsyncFetcher;
00089 class UrlLeftTrimFilter;
00090 class Writer;
00091 
00094 class RewriteDriver : public HtmlParse {
00095  public:
00097   enum CssResolutionStatus {
00098     kWriteFailed,
00099     kNoResolutionNeeded,
00100     kSuccess
00101   };
00102 
00104   enum WaitMode {
00105     kNoWait,  
00106     kWaitForCompletion,    
00107     kWaitForCachedRender,  
00108 
00109     kWaitForShutDown       
00110 
00111   };
00112 
00124   enum XhtmlStatus {
00125     kXhtmlUnknown,
00126     kIsXhtml,
00127     kIsNotXhtml
00128   };
00129 
00136   static const char* kPassThroughRequestAttributes[7];
00137 
00141   static const char kDomCohort[];
00143   static const char kBeaconCohort[];
00144 
00147   static const char kLastRequestTimestamp[];
00149   static const char kParseSizeLimitExceeded[];
00151   static const char kSubresourcesPropertyName[];
00153   static const char kStatusCodePropertyName[];
00154 
00155   RewriteDriver(MessageHandler* message_handler,
00156                 FileSystem* file_system,
00157                 UrlAsyncFetcher* url_async_fetcher);
00158 
00161   virtual ~RewriteDriver();
00162 
00165   RewriteDriver* Clone();
00166 
00172   void Clear();
00173 
00175   static void InitStats(Statistics* statistics);
00176 
00178   static void Initialize();
00179   static void Terminate();
00180 
00183   void SetServerContext(ServerContext* server_context);
00184 
00187   bool MayCacheExtendCss() const;
00188   bool MayCacheExtendImages() const;
00189   bool MayCacheExtendPdfs() const;
00190   bool MayCacheExtendScripts() const;
00191 
00192   const GoogleString& user_agent() const { return user_agent_; }
00193 
00194   void SetUserAgent(const StringPiece& user_agent_string);
00195 
00196   const RequestProperties* request_properties() const {
00197     return request_properties_.get();
00198   }
00199 
00201   void ClearRequestProperties();
00202 
00204   bool using_spdy() const { return request_context_->using_spdy(); }
00205 
00206   bool write_property_cache_dom_cohort() const {
00207     return write_property_cache_dom_cohort_;
00208   }
00209   void set_write_property_cache_dom_cohort(bool x) {
00210     write_property_cache_dom_cohort_ = x;
00211   }
00212 
00213   RequestContextPtr request_context() { return request_context_; }
00214   void set_request_context(const RequestContextPtr& x);
00215 
00218   RequestTrace* trace_context();
00219 
00222   void TracePrintf(const char* fmt, ...);
00223 
00226   ResponseHeaders* mutable_response_headers() {
00227     return flush_occurred_ ? NULL : response_headers_;
00228   }
00229 
00236   const ResponseHeaders* response_headers() {
00237     return response_headers_;
00238   }
00239 
00243   void set_response_headers_ptr(ResponseHeaders* headers) {
00244     response_headers_ = headers;
00245   }
00246 
00252   void SetRequestHeaders(const RequestHeaders& headers);
00253 
00254   const RequestHeaders* request_headers() const {
00255     return request_headers_.get();
00256   }
00257 
00258   UserAgentMatcher* user_agent_matcher() const {
00259     DCHECK(server_context() != NULL);
00260     return server_context()->user_agent_matcher();
00261   }
00262 
00267   void AddFilters();
00268 
00272   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00273 
00275   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00277   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00278 
00280   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00282   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00283 
00295   void AppendRewriteFilter(RewriteFilter* filter);
00296 
00299   void PrependRewriteFilter(RewriteFilter* filter);
00300 
00306   void SetWriter(Writer* writer);
00307 
00308   Writer* writer() const { return writer_; }
00309 
00338   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00339 
00357   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00358                             AsyncFetch* async_fetch);
00359 
00366   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00367                            RewriteFilter* filter,
00368                            AsyncFetch* async_fetch);
00369 
00376   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00377                                          RewriteFilter** filter) const;
00378 
00382   bool DecodeOutputResourceName(const GoogleUrl& url,
00383                                 ResourceNamer* name_out,
00384                                 OutputResourceKind* kind_out,
00385                                 RewriteFilter** filter_out) const;
00386 
00388   bool DecodeUrl(const GoogleUrl& url,
00389                  StringVector* decoded_urls) const;
00390 
00391   FileSystem* file_system() { return file_system_; }
00392   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00393 
00398   void SetSessionFetcher(UrlAsyncFetcher* f);
00399 
00400   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00402   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00403     distributed_async_fetcher_ = fetcher;
00404   }
00405 
00408   CacheUrlAsyncFetcher* CreateCacheFetcher();
00410   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00411 
00412   ServerContext* server_context() const { return server_context_; }
00413   Statistics* statistics() const;
00414 
00415   AddInstrumentationFilter* add_instrumentation_filter() {
00416     return add_instrumentation_filter_;
00417   }
00418 
00420   void set_custom_options(RewriteOptions* options) {
00421     set_options_for_pool(NULL, options);
00422   }
00423 
00426   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00427     controlling_pool_ = pool;
00428     options_.reset(options);
00429   }
00430 
00432   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00433 
00435   const RewriteOptions* options() const { return options_.get(); }
00436 
00440   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00441                             const ContentType& content_type);
00442 
00449   virtual void FinishParse();
00450 
00453   void FinishParseAsync(Function* callback);
00454 
00458   void InfoAt(const RewriteContext* context,
00459               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00460 
00465 
00472   OutputResourcePtr CreateOutputResourceFromResource(
00473       const StringPiece& filter_id,
00474       const UrlSegmentEncoder* encoder,
00475       const ResourceContext* data,
00476       const ResourcePtr& input_resource,
00477       OutputResourceKind kind);
00478 
00495   OutputResourcePtr CreateOutputResourceWithPath(
00496       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00497       const StringPiece& base_url, const StringPiece& filter_id,
00498       const StringPiece& name, OutputResourceKind kind);
00499 
00502   void PopulateResourceNamer(
00503     const StringPiece& filter_id,
00504     const StringPiece& name,
00505     ResourceNamer* full_name);
00506 
00510   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00511       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00512       const StringPiece& name, OutputResourceKind kind);
00513 
00516   OutputResourcePtr CreateOutputResourceWithMappedPath(
00517       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00518       const StringPiece& filter_id, const StringPiece& name,
00519       OutputResourceKind kind) {
00520     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00521                                         decoded_base_url_.AllExceptLeaf(),
00522                                         filter_id, name, kind);
00523   }
00524 
00527   OutputResourcePtr CreateOutputResourceWithPath(
00528       const StringPiece& path, const StringPiece& filter_id,
00529       const StringPiece& name, OutputResourceKind kind) {
00530     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00531                                         kind);
00532   }
00533 
00537   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00538 
00542   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00543       const StringPiece& absolute_url);
00544 
00549   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00550 
00554   bool MayRewriteUrl(const GoogleUrl& domain_url,
00555                      const GoogleUrl& input_url) const;
00556 
00560   const GoogleUrl& base_url() const { return base_url_; }
00561 
00563   StringPiece fetch_url() const { return fetch_url_; }
00564 
00568   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00569   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00570 
00572   bool IsHttps() const { return google_url().SchemeIs("https"); }
00573 
00574   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00575 
00577   RewriteFilter* FindFilter(const StringPiece& id) const;
00578 
00580   bool refs_before_base() { return refs_before_base_; }
00581 
00586   void set_refs_before_base() { refs_before_base_ = true; }
00587 
00592   StringPiece containing_charset() { return containing_charset_; }
00593   void set_containing_charset(const StringPiece charset) {
00594     charset.CopyToString(&containing_charset_);
00595   }
00596 
00598   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00599                               HtmlElement* elt,
00600                               HtmlElement::Attribute* attr);
00601 
00606   bool InitiateRewrite(RewriteContext* rewrite_context);
00607   void InitiateFetch(RewriteContext* rewrite_context);
00608 
00619   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00620 
00624   void ReportSlowRewrites(int num);
00625 
00630   void Cleanup();
00631 
00636   void AddUserReference();
00637 
00639   GoogleString ToString(bool show_detached_contexts);
00640   void PrintState(bool show_detached_contexts); 
00641   void PrintStateToErrorLog(bool show_detached_contexts); 
00642 
00645   void WaitForCompletion();
00646 
00653   void WaitForShutDown();
00654 
00658   void BoundedWaitFor(WaitMode mode, int64 timeout_ms);
00659 
00667   void set_fully_rewrite_on_flush(bool x) {
00668     fully_rewrite_on_flush_ = x;
00669   }
00670 
00672   bool fully_rewrite_on_flush() const {
00673     return fully_rewrite_on_flush_;
00674   }
00675 
00679   void set_fast_blocking_rewrite(bool x) {
00680     fast_blocking_rewrite_ = x;
00681   }
00682 
00683   bool fast_blocking_rewrite() const {
00684     return fast_blocking_rewrite_;
00685   }
00686 
00689   void EnableBlockingRewrite(RequestHeaders* request_headers);
00690 
00697   void set_externally_managed(bool x) { externally_managed_ = x; }
00698 
00702   void DetachFetch();
00703 
00706   void DetachedFetchComplete();
00707 
00711   void FetchComplete();
00712 
00718   void DeleteRewriteContext(RewriteContext* rewrite_context);
00719 
00720   int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }
00721 
00726   void set_max_page_processing_delay_ms(int x) {
00727     max_page_processing_delay_ms_ = x;
00728   }
00729   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00730 
00732   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00733   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00734 
00740   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00741                                           RewriteContext* candidate);
00742 
00747   void DeregisterForPartitionKey(
00748       const GoogleString& partition_key, RewriteContext* candidate);
00749 
00752   void RequestFlush() { flush_requested_ = true; }
00753   bool flush_requested() const { return flush_requested_; }
00754 
00766   void ExecuteFlushIfRequested();
00767 
00771   void ExecuteFlushIfRequestedAsync(Function* callback);
00772 
00781   virtual void Flush();
00782 
00786   void FlushAsync(Function* done);
00787 
00789   void AddRewriteTask(Function* task);
00790 
00793   void AddLowPriorityRewriteTask(Function* task);
00794 
00795   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00796   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00797   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00798     return low_priority_rewrite_worker_;
00799   }
00800 
00801   Scheduler* scheduler() { return scheduler_; }
00802 
00805   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00806   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00807 
00815   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00816                                      const StringPiece& output_css_base,
00817                                      const StringPiece& contents,
00818                                      Writer* writer,
00819                                      MessageHandler* handler);
00820 
00828   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00829                            const GoogleUrl& output_base,
00830                            bool* proxy_mode) const;
00831 
00839   void UpdatePropertyValueInDomCohort(
00840       AbstractPropertyPage* page,
00841       StringPiece property_name,
00842       StringPiece property_value);
00843 
00844   void set_client_id(const StringPiece& id) { client_id_ = id.as_string(); }
00845   const GoogleString& client_id() const { return client_id_; }
00846 
00849   PropertyPage* property_page() const;
00854   FallbackPropertyPage* fallback_property_page() const {
00855     return fallback_property_page_;
00856   }
00858   void set_property_page(PropertyPage* page);
00860   void set_fallback_property_page(FallbackPropertyPage* page);
00862   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00863 
00865   const CriticalLineInfo* critical_line_info() const;
00866 
00869   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00870 
00871   const SplitHtmlConfig* split_html_config();
00872 
00873   CriticalCssResult* critical_css_result() const;
00876   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00877 
00879   CriticalImagesInfo* critical_images_info() const {
00880     return critical_images_info_.get();
00881   }
00882 
00887   CriticalSelectorInfo* critical_selector_info() {
00888     return critical_selector_info_.get();
00889   }
00890 
00894   void set_critical_selector_info(CriticalSelectorInfo* info) {
00895     critical_selector_info_.reset(info);
00896   }
00897 
00902   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00903     critical_images_info_.reset(critical_images_info);
00904   }
00905 
00908   bool CriticalSelectorsEnabled() const;
00909 
00912   bool FlattenCssImportsEnabled() const {
00913     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00914             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00915              (CriticalSelectorsEnabled() ||
00916               options()->Enabled(RewriteOptions::kComputeCriticalCss))));
00917   }
00918 
00922   int num_inline_preview_images() const { return num_inline_preview_images_; }
00923 
00925   void increment_num_inline_preview_images();
00926 
00929   int num_flushed_early_pagespeed_resources() const {
00930     return num_flushed_early_pagespeed_resources_;
00931   }
00932 
00935   void increment_num_flushed_early_pagespeed_resources() {
00936     ++num_flushed_early_pagespeed_resources_;
00937   }
00938 
00941   void increment_async_events_count();
00942 
00944   void decrement_async_events_count();
00945 
00948   XhtmlStatus MimeTypeXhtmlStatus();
00949 
00950   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
00951   bool flushed_cached_html() { return flushed_cached_html_; }
00952 
00953   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
00954   bool flushing_cached_html() const { return flushing_cached_html_; }
00955 
00956   void set_flushed_early(bool x) { flushed_early_ = x; }
00957   bool flushed_early() const { return flushed_early_; }
00958 
00959   void set_flushing_early(bool x) { flushing_early_ = x; }
00960   bool flushing_early() const { return flushing_early_; }
00961 
00962   void set_is_lazyload_script_flushed(bool x) {
00963     is_lazyload_script_flushed_ = x;
00964   }
00965   bool is_lazyload_script_flushed() const {
00966     return is_lazyload_script_flushed_; }
00967 
00969   FlushEarlyInfo* flush_early_info();
00970 
00971   FlushEarlyRenderInfo* flush_early_render_info() const;
00972 
00975   void set_flush_early_render_info(
00976       FlushEarlyRenderInfo* flush_early_render_info);
00977 
00978   void set_is_blink_request(bool x) { is_blink_request_ = x; }
00979   bool is_blink_request() const { return is_blink_request_; }
00980 
00983   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
00984 
00987   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
00988 
00991   AbstractLogRecord* log_record();
00992 
00993   DomStatsFilter* dom_stats_filter() const {
00994     return dom_stats_filter_;
00995   }
00996 
00999   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01000 
01002   bool is_nested() const { return is_nested_; }
01003 
01007   bool MetadataRequested(const RequestHeaders& request_headers) const;
01008 
01010   bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; }
01011 
01024   bool Write(const ResourceVector& inputs,
01025              const StringPiece& contents,
01026              const ContentType* type,
01027              StringPiece charset,
01028              OutputResource* output);
01029 
01030   void set_defer_instrumentation_script(bool x) {
01031     defer_instrumentation_script_ = x;
01032   }
01033   bool defer_instrumentation_script() const {
01034     return defer_instrumentation_script_;
01035   }
01036 
01037  protected:
01038   virtual void DetermineEnabledFiltersImpl();
01039 
01040  private:
01041   friend class DistributedRewriteContextTest;
01042   friend class RewriteContext;
01043   friend class RewriteDriverTest;
01044   friend class RewriteTestBase;
01045   friend class ServerContextTest;
01046 
01047   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01048 
01050   bool ShouldDistributeFetch(const StringPiece& filter_id);
01051 
01063   bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id,
01064                        AsyncFetch* async_fetch);
01065 
01070   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01071                                Function* done);
01072 
01076   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01077                              Function* done);
01078 
01080   bool IsDone(WaitMode wait_mode, bool deadline_reached);
01081 
01084   bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
01085     return wait_mode == kWaitForShutDown ||
01086         (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
01087   }
01088 
01092   void FlushAsyncDone(int num_rewrites, Function* callback);
01093 
01098   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01099 
01101   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01102 
01105   void QueueFinishParseAfterFlush(Function* user_callback);
01106   void FinishParseAfterFlush(Function* user_callback);
01107 
01109   bool RewritesComplete() const;
01110 
01113   void SetBaseUrlIfUnset(const StringPiece& new_base);
01114 
01117   void SetBaseUrlForFetch(const StringPiece& url);
01118 
01121   void SetDecodedUrlFromBase();
01122 
01124   AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); }
01125 
01127   virtual void ParseTextInternal(const char* content, int size);
01128 
01130   bool ShouldSkipParsing();
01131 
01132   friend class ScanFilter;
01133 
01137   void AddCommonFilter(CommonFilter* filter);
01138 
01142   void RegisterRewriteFilter(RewriteFilter* filter);
01143 
01148   void EnableRewriteFilter(const char* id);
01149 
01153   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl);
01154 
01155   void AddPreRenderFilters();
01156   void AddPostRenderFilters();
01157 
01159   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01160                                       ResourceNamer* name_out,
01161                                       OutputResourceKind* kind_out,
01162                                       RewriteFilter** filter_out,
01163                                       GoogleString* url_base,
01164                                       StringVector* urls) const;
01165 
01175   void WriteDomCohortIntoPropertyCache();
01176 
01178   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01179 
01186   void PossiblyPurgeCachedResponseAndReleaseDriver();
01187 
01192   bool ShouldPurgeRewrittenResponse();
01193 
01196   static bool GetPurgeUrl(const GoogleUrl& google_url,
01197                           const RewriteOptions* options,
01198                           GoogleString* purge_url,
01199                           GoogleString* purge_method);
01200 
01202   void PurgeDownstreamCache(const GoogleString& purge_url,
01203                             const GoogleString& purge_method);
01204 
01206   void LogStats();
01207 
01224   bool PrepareShouldSignal();
01225   void SignalIfRequired(bool result_of_prepare_should_signal);
01226 
01238   bool base_was_set_;
01239 
01244   bool refs_before_base_;
01245 
01247   GoogleString containing_charset_;
01248 
01249   bool filters_added_;
01250   bool externally_managed_;
01251 
01260   enum RefCategory {
01261     kRefUser,  
01262     kRefParsing,  
01263 
01267     kRefPendingRewrites,
01268 
01272     kRefDetachedRewrites,
01273 
01281     kRefDeletingRewrites,
01282 
01284     kRefFetchUserFacing,
01285 
01287     kRefFetchBackground,
01288 
01293     kRefAsyncEvents,
01294 
01295     kNumRefCategories
01296   };
01297 
01298   friend class CategorizedRefcount<RewriteDriver, RefCategory>;
01299 
01301   CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;
01302 
01304   void LastRefRemoved();
01305   StringPiece RefCategoryName(RefCategory cat);
01306 
01309   void DropReference(RefCategory cat);
01310 
01313   bool release_driver_;
01314 
01317   bool parsing_; 
01318 
01322   WaitMode waiting_; 
01323 
01325   bool waiting_deadline_reached_; 
01326 
01331   bool fully_rewrite_on_flush_;
01332 
01335   bool fast_blocking_rewrite_;
01336 
01337   bool flush_requested_;
01338   bool flush_occurred_;
01339 
01341   bool flushed_cached_html_;
01342 
01344   bool flushing_cached_html_;
01345 
01348   bool flushed_early_;
01352   bool flushing_early_;
01353 
01356   bool is_lazyload_script_flushed_;
01357 
01360   bool made_downstream_purge_attempt_;
01361 
01365   bool write_property_cache_dom_cohort_;
01366 
01369   GoogleUrl base_url_;
01370 
01374   GoogleUrl decoded_base_url_;
01375 
01378   GoogleString fetch_url_;
01379 
01380   GoogleString user_agent_;
01381 
01382   LazyBool should_skip_parsing_;
01383 
01384   StringFilterMap resource_filter_map_;
01385 
01386   ResponseHeaders* response_headers_;
01387 
01390   scoped_ptr<const RequestHeaders> request_headers_;
01391 
01392   int status_code_; 
01393 
01396   typedef std::vector<RewriteContext*> RewriteContextVector;
01397   RewriteContextVector rewrites_; 
01398 
01401   int max_page_processing_delay_ms_;
01402 
01403   typedef std::set<RewriteContext*> RewriteContextSet;
01404 
01409   RewriteContextSet initiated_rewrites_; 
01410 
01412   int64 num_initiated_rewrites_; 
01413 
01421   int64 num_detached_rewrites_; 
01422 
01430   RewriteContextSet detached_rewrites_; 
01431 
01433   int possibly_quick_rewrites_; 
01434 
01437   RewriteContextVector fetch_rewrites_;
01438 
01441   FileSystem* file_system_;
01442   ServerContext* server_context_;
01443   Scheduler* scheduler_;
01444   UrlAsyncFetcher* default_url_async_fetcher_; 
01445 
01449   UrlAsyncFetcher* url_async_fetcher_;
01450 
01454   UrlAsyncFetcher* distributed_async_fetcher_;
01455 
01458   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01459 
01460   AddInstrumentationFilter* add_instrumentation_filter_;
01461   DomStatsFilter* dom_stats_filter_;
01462   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01463 
01464   ScanFilter scan_filter_;
01465   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01466   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01467 
01470   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01471   PrimaryRewriteContextMap primary_rewrite_context_map_;
01472 
01473   HtmlResourceSlotSet slots_;
01474 
01475   scoped_ptr<RewriteOptions> options_;
01476 
01477   RewriteDriverPool* controlling_pool_; 
01478 
01480   scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
01481       cache_url_async_fetcher_async_op_hooks_;
01482 
01484   UrlSegmentEncoder default_encoder_;
01485 
01487   FilterList early_pre_render_filters_;
01489   FilterList pre_render_filters_;
01490 
01494   FilterVector filters_to_delete_;
01495 
01496   QueuedWorkerPool::Sequence* html_worker_;
01497   QueuedWorkerPool::Sequence* rewrite_worker_;
01498   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01499 
01500   Writer* writer_;
01501 
01503   GoogleString client_id_;
01504 
01507   FallbackPropertyPage* fallback_property_page_;
01508 
01510   bool owns_property_page_;
01511 
01513   UserAgentMatcher::DeviceType device_type_;
01514 
01515   scoped_ptr<CriticalLineInfo> critical_line_info_;
01516 
01517   scoped_ptr<SplitHtmlConfig> split_html_config_;
01518 
01521   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01522   scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
01523 
01524   scoped_ptr<CriticalCssResult> critical_css_result_;
01525 
01527   bool xhtml_mimetype_computed_;
01528   XhtmlStatus xhtml_status_ : 8;
01529 
01532   int num_inline_preview_images_;
01533 
01535   int num_flushed_early_pagespeed_resources_;
01536 
01538   int num_bytes_in_;
01539 
01540   DebugFilter* debug_filter_;
01541 
01542   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01543   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01544 
01546   bool is_blink_request_;
01547   bool can_rewrite_resources_;
01548   bool is_nested_;
01549 
01552   RequestContextPtr request_context_;
01553 
01555   int64 start_time_ms_;
01556 
01557   scoped_ptr<RequestProperties> request_properties_;
01558 
01562   static int initialized_count_;
01563 
01566   bool tried_to_distribute_fetch_;
01567 
01570   bool defer_instrumentation_script_;
01571 
01572   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01573 };
01574 
01577 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01578  public:
01579   virtual ~OptionsAwareHTTPCacheCallback();
01580   virtual bool IsCacheValid(const GoogleString& key,
01581                             const ResponseHeaders& headers);
01582   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01583  protected:
01586   OptionsAwareHTTPCacheCallback(
01587       const RewriteOptions* rewrite_options,
01588       const RequestContextPtr& request_ctx);
01589 
01590  private:
01591   const RewriteOptions* rewrite_options_;
01592 
01593   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01594 };
01595 
01596 }  
01597 
01598 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines