Page Speed Optimization Libraries  1.5.27.2
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/content_type.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00035 #include "net/instaweb/rewriter/public/resource.h"
00036 #include "net/instaweb/rewriter/public/resource_slot.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver_factory.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/scan_filter.h"
00040 #include "net/instaweb/rewriter/public/server_context.h"
00041 #include "net/instaweb/util/public/abstract_client_state.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/google_url.h"
00044 #include "net/instaweb/util/public/printf_format.h"
00045 #include "net/instaweb/util/public/queued_worker_pool.h"
00046 #include "net/instaweb/util/public/scheduler.h"
00047 #include "net/instaweb/util/public/scoped_ptr.h"
00048 #include "net/instaweb/util/public/string.h"
00049 #include "net/instaweb/util/public/string_util.h"
00050 #include "net/instaweb/util/public/thread_system.h"
00051 #include "net/instaweb/util/public/url_segment_encoder.h"
00052 
00053 namespace net_instaweb {
00054 
00055 class AbstractLogRecord;
00056 class AbstractMutex;
00057 class AbstractPropertyPage;
00058 class AddInstrumentationFilter;
00059 class AsyncFetch;
00060 class CacheUrlAsyncFetcher;
00061 class CommonFilter;
00062 class CriticalCssResult;
00063 class CriticalLineInfo;
00064 class CriticalSelectorSet;
00065 class DebugFilter;
00066 class DeviceProperties;
00067 class DomainRewriteFilter;
00068 class DomStatsFilter;
00069 class FallbackPropertyPage;
00070 class FileSystem;
00071 class FlushEarlyInfo;
00072 class FlushEarlyRenderInfo;
00073 class Function;
00074 class HtmlFilter;
00075 class HtmlWriterFilter;
00076 class MessageHandler;
00077 class OutputResource;
00078 class PropertyPage;
00079 class RequestHeaders;
00080 class RequestTrace;
00081 class ResourceContext;
00082 class ResourceNamer;
00083 class ResponseHeaders;
00084 class RewriteContext;
00085 class RewriteDriverPool;
00086 class RewriteFilter;
00087 class ScopedMutex;
00088 class Statistics;
00089 class UrlAsyncFetcher;
00090 class UrlLeftTrimFilter;
00091 class Writer;
00092 
00095 class RewriteDriver : public HtmlParse {
00096  public:
00098   enum CssResolutionStatus {
00099     kWriteFailed,
00100     kNoResolutionNeeded,
00101     kSuccess
00102   };
00103 
00105   enum WaitMode {
00106     kNoWait,  
00107     kWaitForCompletion,    
00108     kWaitForCachedRender,  
00109 
00110     kWaitForShutDown       
00111 
00112   };
00113 
00125   enum XhtmlStatus {
00126     kXhtmlUnknown,
00127     kIsXhtml,
00128     kIsNotXhtml
00129   };
00130 
00137   static const char* kPassThroughRequestAttributes[5];
00138 
00142   static const char kDomCohort[];
00144   static const char kBeaconCohort[];
00145 
00148   static const char kLastRequestTimestamp[];
00150   static const char kParseSizeLimitExceeded[];
00152   static const char kSubresourcesPropertyName[];
00154   static const char kStatusCodePropertyName[];
00155 
00156   RewriteDriver(MessageHandler* message_handler,
00157                 FileSystem* file_system,
00158                 UrlAsyncFetcher* url_async_fetcher);
00159 
00162   virtual ~RewriteDriver();
00163 
00166   RewriteDriver* Clone();
00167 
00173   void Clear();
00174 
00176   static void InitStats(Statistics* statistics);
00177 
00179   static void Initialize();
00180   static void Terminate();
00181 
00184   void SetServerContext(ServerContext* server_context);
00185 
00188   bool MayCacheExtendCss() const;
00189   bool MayCacheExtendImages() const;
00190   bool MayCacheExtendPdfs() const;
00191   bool MayCacheExtendScripts() const;
00192 
00193   const GoogleString& user_agent() const { return user_agent_; }
00194 
00195   void SetUserAgent(const StringPiece& user_agent_string);
00196 
00197   const DeviceProperties* device_properties() const {
00198     return device_properties_.get();
00199   }
00200 
00202   void ClearDeviceProperties();
00203 
00205   bool using_spdy() const { return request_context_->using_spdy(); }
00206 
00207   bool write_property_cache_dom_cohort() const {
00208     return write_property_cache_dom_cohort_;
00209   }
00210   void set_write_property_cache_dom_cohort(bool x) {
00211     write_property_cache_dom_cohort_ = x;
00212   }
00213 
00214   RequestContextPtr request_context() { return request_context_; }
00215   void set_request_context(const RequestContextPtr& x);
00216 
00219   RequestTrace* trace_context();
00220 
00223   void TracePrintf(const char* fmt, ...);
00224 
00227   ResponseHeaders* mutable_response_headers() {
00228     return flush_occurred_ ? NULL : response_headers_;
00229   }
00230 
00237   const ResponseHeaders* response_headers() {
00238     return response_headers_;
00239   }
00240 
00244   void set_response_headers_ptr(ResponseHeaders* headers) {
00245     response_headers_ = headers;
00246   }
00247 
00248   void set_request_headers(const RequestHeaders* headers) {
00249     request_headers_ = headers;
00250   }
00251 
00252   const RequestHeaders* request_headers() const {
00253     return request_headers_;
00254   }
00255 
00256   UserAgentMatcher* user_agent_matcher() const {
00257     DCHECK(server_context() != NULL);
00258     return server_context()->user_agent_matcher();
00259   }
00260 
00265   void AddFilters();
00266 
00270   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00271 
00273   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00275   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00276 
00278   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00280   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00281 
00293   void AppendRewriteFilter(RewriteFilter* filter);
00294 
00297   void PrependRewriteFilter(RewriteFilter* filter);
00298 
00304   void SetWriter(Writer* writer);
00305 
00306   Writer* writer() const { return writer_; }
00307 
00336   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00337 
00355   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00356                             AsyncFetch* async_fetch);
00357 
00364   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00365                            RewriteFilter* filter,
00366                            AsyncFetch* async_fetch);
00367 
00374   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00375                                          RewriteFilter** filter) const;
00376 
00380   bool DecodeOutputResourceName(const GoogleUrl& url,
00381                                 ResourceNamer* name_out,
00382                                 OutputResourceKind* kind_out,
00383                                 RewriteFilter** filter_out) const;
00384 
00386   bool DecodeUrl(const GoogleUrl& url,
00387                  StringVector* decoded_urls) const;
00388 
00389   FileSystem* file_system() { return file_system_; }
00390   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00391 
00396   void SetSessionFetcher(UrlAsyncFetcher* f);
00397 
00398   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00400   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00401     distributed_async_fetcher_ = fetcher;
00402   }
00403 
00406   CacheUrlAsyncFetcher* CreateCacheFetcher();
00408   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00409 
00410   ServerContext* server_context() const { return server_context_; }
00411   Statistics* statistics() const;
00412 
00413   AddInstrumentationFilter* add_instrumentation_filter() {
00414     return add_instrumentation_filter_;
00415   }
00416 
00418   void set_custom_options(RewriteOptions* options) {
00419     set_options_for_pool(NULL, options);
00420   }
00421 
00424   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00425     controlling_pool_ = pool;
00426     options_.reset(options);
00427   }
00428 
00430   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00431 
00433   const RewriteOptions* options() const { return options_.get(); }
00434 
00436   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00437                             const ContentType& content_type);
00438 
00445   virtual void FinishParse();
00446 
00449   void FinishParseAsync(Function* callback);
00450 
00454   void InfoAt(const RewriteContext* context,
00455               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00456 
00461 
00468   OutputResourcePtr CreateOutputResourceFromResource(
00469       const StringPiece& filter_id,
00470       const UrlSegmentEncoder* encoder,
00471       const ResourceContext* data,
00472       const ResourcePtr& input_resource,
00473       OutputResourceKind kind);
00474 
00491   OutputResourcePtr CreateOutputResourceWithPath(
00492       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00493       const StringPiece& base_url, const StringPiece& filter_id,
00494       const StringPiece& name, OutputResourceKind kind);
00495 
00498   void PopulateResourceNamer(
00499     const StringPiece& filter_id,
00500     const StringPiece& name,
00501     ResourceNamer* full_name);
00502 
00506   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00507       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00508       const StringPiece& name, OutputResourceKind kind);
00509 
00512   OutputResourcePtr CreateOutputResourceWithMappedPath(
00513       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00514       const StringPiece& filter_id, const StringPiece& name,
00515       OutputResourceKind kind) {
00516     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00517                                         decoded_base_url_.AllExceptLeaf(),
00518                                         filter_id, name, kind);
00519   }
00520 
00523   OutputResourcePtr CreateOutputResourceWithPath(
00524       const StringPiece& path, const StringPiece& filter_id,
00525       const StringPiece& name, OutputResourceKind kind) {
00526     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00527                                         kind);
00528   }
00529 
00533   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00534 
00538   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00539       const StringPiece& absolute_url);
00540 
00545   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00546 
00550   bool MayRewriteUrl(const GoogleUrl& domain_url,
00551                      const GoogleUrl& input_url) const;
00552 
00557   void ReadAsync(Resource::AsyncCallback* callback,
00558                  MessageHandler* message_handler);
00559 
00563   const GoogleUrl& base_url() const { return base_url_; }
00564 
00566   StringPiece fetch_url() const { return fetch_url_; }
00567 
00571   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00572   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00573 
00575   bool IsHttps() const { return google_url().SchemeIs("https"); }
00576 
00577   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00578 
00580   RewriteFilter* FindFilter(const StringPiece& id) const;
00581 
00583   bool refs_before_base() { return refs_before_base_; }
00584 
00589   void set_refs_before_base() { refs_before_base_ = true; }
00590 
00595   StringPiece containing_charset() { return containing_charset_; }
00596   void set_containing_charset(const StringPiece charset) {
00597     charset.CopyToString(&containing_charset_);
00598   }
00599 
00601   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00602                               HtmlElement* elt,
00603                               HtmlElement::Attribute* attr);
00604 
00609   bool InitiateRewrite(RewriteContext* rewrite_context);
00610   void InitiateFetch(RewriteContext* rewrite_context);
00611 
00622   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00623 
00627   void ReportSlowRewrites(int num);
00628 
00633   void Cleanup();
00634 
00636   GoogleString ToString(bool show_detached_contexts);
00637   void PrintState(bool show_detached_contexts); 
00638   void PrintStateToErrorLog(bool show_detached_contexts); 
00639 
00642   void WaitForCompletion();
00643 
00650   void WaitForShutDown();
00651 
00655   void BoundedWaitFor(WaitMode mode, int64 timeout_ms);
00656 
00664   void set_fully_rewrite_on_flush(bool x) {
00665     fully_rewrite_on_flush_ = x;
00666   }
00667 
00670   void EnableBlockingRewrite(RequestHeaders* request_headers);
00671 
00678   void set_externally_managed(bool x) { externally_managed_ = x; }
00679 
00683   void DetachFetch();
00684 
00687   void DetachedFetchComplete();
00688 
00692   void FetchComplete();
00693 
00699   void DeleteRewriteContext(RewriteContext* rewrite_context);
00700 
00706   void set_rewrite_deadline_ms(int x) { options_->set_rewrite_deadline_ms(x); }
00707   int rewrite_deadline_ms() { return options_->rewrite_deadline_ms(); }
00708 
00713   void set_max_page_processing_delay_ms(int x) {
00714     max_page_processing_delay_ms_ = x;
00715   }
00716   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00717 
00719   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00720   UserAgentMatcher::DeviceType device_type() { return device_type_; }
00721 
00727   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00728                                           RewriteContext* candidate);
00729 
00734   void DeregisterForPartitionKey(
00735       const GoogleString& partition_key, RewriteContext* candidate);
00736 
00739   void RequestFlush() { flush_requested_ = true; }
00740   bool flush_requested() const { return flush_requested_; }
00741 
00753   void ExecuteFlushIfRequested();
00754 
00758   void ExecuteFlushIfRequestedAsync(Function* callback);
00759 
00768   virtual void Flush();
00769 
00773   void FlushAsync(Function* done);
00774 
00776   void AddRewriteTask(Function* task);
00777 
00780   void AddLowPriorityRewriteTask(Function* task);
00781 
00782   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00783   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00784   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00785     return low_priority_rewrite_worker_;
00786   }
00787 
00788   Scheduler* scheduler() { return scheduler_; }
00789 
00792   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00793   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00794 
00802   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00803                                      const StringPiece& output_css_base,
00804                                      const StringPiece& contents,
00805                                      Writer* writer,
00806                                      MessageHandler* handler);
00807 
00815   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00816                            const GoogleUrl& output_base,
00817                            bool* proxy_mode) const;
00818 
00826   void UpdatePropertyValueInDomCohort(
00827       AbstractPropertyPage* page,
00828       StringPiece property_name,
00829       StringPiece property_value);
00830 
00833   void set_client_state(AbstractClientState* client_state) {
00834       client_state_.reset(client_state);
00835   }
00836 
00840   AbstractClientState* client_state() const { return client_state_.get(); }
00841 
00842   void set_client_id(const StringPiece& id) { client_id_ = id.as_string(); }
00843   const GoogleString& client_id() const { return client_id_; }
00844 
00847   PropertyPage* property_page() const;
00852   FallbackPropertyPage* fallback_property_page() const {
00853     return fallback_property_page_;
00854   }
00856   void set_property_page(PropertyPage* page);
00858   void set_fallback_property_page(FallbackPropertyPage* page);
00860   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00861 
00863   const CriticalLineInfo* critical_line_info() const;
00864 
00867   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00868 
00869   CriticalCssResult* critical_css_result() const;
00872   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00873 
00875   CriticalImagesInfo* critical_images_info() const {
00876     return critical_images_info_.get();
00877   }
00878 
00881   bool critical_images_info_was_set() const {
00882     return critical_images_info_was_set_;
00883   }
00884 
00889   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00890     critical_images_info_.reset(critical_images_info);
00891     critical_images_info_was_set_ = true;
00892   }
00893 
00896   bool CriticalSelectorsEnabled() const {
00897     return (options()->Enabled(RewriteOptions::kPrioritizeCriticalCss) &&
00898             server_context()->factory()->UseBeaconResultsInFilters());
00899   }
00900 
00903   bool FlattenCssImportsEnabled() const {
00904     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00905             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00906              CriticalSelectorsEnabled()));
00907   }
00908 
00913   CriticalSelectorSet* CriticalSelectors();
00914 
00918   int num_inline_preview_images() const { return num_inline_preview_images_; }
00919 
00921   void increment_num_inline_preview_images();
00922 
00925   int num_flushed_early_pagespeed_resources() const {
00926     return num_flushed_early_pagespeed_resources_;
00927   }
00928 
00931   void increment_num_flushed_early_pagespeed_resources() {
00932     ++num_flushed_early_pagespeed_resources_;
00933   }
00934 
00938   void increment_async_events_count();
00939 
00941   void decrement_async_events_count();
00942 
00945   XhtmlStatus MimeTypeXhtmlStatus();
00946 
00947   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
00948   bool flushed_cached_html() { return flushed_cached_html_; }
00949 
00950   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
00951   bool flushing_cached_html() const { return flushing_cached_html_; }
00952 
00953   void set_flushed_early(bool x) { flushed_early_ = x; }
00954   bool flushed_early() const { return flushed_early_; }
00955 
00956   void set_flushing_early(bool x) { flushing_early_ = x; }
00957   bool flushing_early() const { return flushing_early_; }
00958 
00959   void set_is_lazyload_script_flushed(bool x) {
00960     is_lazyload_script_flushed_ = x;
00961   }
00962   bool is_lazyload_script_flushed() const {
00963     return is_lazyload_script_flushed_; }
00964 
00966   FlushEarlyInfo* flush_early_info();
00967 
00968   FlushEarlyRenderInfo* flush_early_render_info() const;
00969 
00972   void set_flush_early_render_info(
00973       FlushEarlyRenderInfo* flush_early_render_info);
00974 
00975   void set_serve_blink_non_critical(bool x) { serve_blink_non_critical_ = x; }
00976   bool serve_blink_non_critical() const { return serve_blink_non_critical_; }
00977 
00978   void set_is_blink_request(bool x) { is_blink_request_ = x; }
00979   bool is_blink_request() const { return is_blink_request_; }
00980 
00983   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
00984 
00987   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
00988 
00991   AbstractLogRecord* log_record();
00992 
00993   DomStatsFilter* dom_stats_filter() const {
00994     return dom_stats_filter_;
00995   }
00996 
00999   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01000 
01002   void set_is_nested(bool n) { is_nested_ = n; }
01003   bool is_nested() const { return is_nested_; }
01004 
01017   bool Write(const ResourceVector& inputs,
01018              const StringPiece& contents,
01019              const ContentType* type,
01020              StringPiece charset,
01021              OutputResource* output);
01022 
01023  protected:
01024   virtual void DetermineEnabledFilters();
01025 
01026  private:
01027   friend class RewriteContext;
01028   friend class RewriteDriverTest;
01029   friend class RewriteTestBase;
01030   friend class ServerContextTest;
01031 
01032   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01033 
01038   void FetchCompleteImpl(bool signal, ScopedMutex* lock);
01039 
01044   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01045                                Function* done);
01046 
01050   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01051                              Function* done);
01052 
01054   bool IsDone(WaitMode wait_mode, bool deadline_reached);
01055 
01059   void FlushAsyncDone(int num_rewrites, Function* callback);
01060 
01065   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01066 
01068   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01069 
01072   void QueueFinishParseAfterFlush(Function* user_callback);
01073   void FinishParseAfterFlush(Function* user_callback);
01074 
01076   bool RewritesComplete() const;
01077 
01082   bool HaveBackgroundFetchRewrite() const;
01083 
01086   void SetBaseUrlIfUnset(const StringPiece& new_base);
01087 
01090   void SetBaseUrlForFetch(const StringPiece& url);
01091 
01094   void SetDecodedUrlFromBase();
01095 
01097   AbstractMutex* rewrite_mutex() { return scheduler_->mutex(); }
01098 
01100   virtual void ParseTextInternal(const char* content, int size);
01101 
01103   bool ShouldSkipParsing();
01104 
01105   friend class ScanFilter;
01106 
01110   void AddCommonFilter(CommonFilter* filter);
01111 
01115   void RegisterRewriteFilter(RewriteFilter* filter);
01116 
01121   void EnableRewriteFilter(const char* id);
01122 
01126   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl);
01127 
01128   void AddPreRenderFilters();
01129   void AddPostRenderFilters();
01130 
01132   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01133                                       ResourceNamer* name_out,
01134                                       OutputResourceKind* kind_out,
01135                                       RewriteFilter** filter_out,
01136                                       GoogleString* url_base,
01137                                       StringVector* urls) const;
01138 
01148   void WriteDomCohortIntoPropertyCache();
01149 
01152   void WriteClientStateIntoPropertyCache();
01153 
01154   void FinalizeFilterLogging();
01155 
01157   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01158 
01160   void LogStats();
01161 
01173   bool base_was_set_;
01174 
01179   bool refs_before_base_;
01180 
01182   GoogleString containing_charset_;
01183 
01184   bool filters_added_;
01185   bool externally_managed_;
01186 
01190   bool fetch_queued_; 
01191 
01196   bool fetch_detached_; 
01197 
01204   bool detached_fetch_main_path_complete_;
01205   bool detached_fetch_detached_path_complete_;
01206 
01209   bool parsing_; 
01210 
01214   WaitMode waiting_; 
01215 
01220   bool fully_rewrite_on_flush_;
01221 
01224   bool cleanup_on_fetch_complete_;
01225 
01226   bool flush_requested_;
01227   bool flush_occurred_;
01228 
01230   bool flushed_cached_html_;
01231 
01233   bool flushing_cached_html_;
01234 
01237   bool flushed_early_;
01241   bool flushing_early_;
01242 
01245   bool is_lazyload_script_flushed_;
01246 
01248   bool release_driver_;
01249 
01253   bool write_property_cache_dom_cohort_;
01254 
01262   int rewrites_to_delete_; 
01263 
01266   GoogleUrl base_url_;
01267 
01271   GoogleUrl decoded_base_url_;
01272 
01275   GoogleString fetch_url_;
01276 
01277   GoogleString user_agent_;
01278 
01279   LazyBool should_skip_parsing_;
01280 
01281   StringFilterMap resource_filter_map_;
01282 
01283   ResponseHeaders* response_headers_;
01284   const RequestHeaders* request_headers_;
01285   int status_code_; 
01286 
01289   typedef std::vector<RewriteContext*> RewriteContextVector;
01290   RewriteContextVector rewrites_; 
01291 
01294   int max_page_processing_delay_ms_;
01295 
01296   typedef std::set<RewriteContext*> RewriteContextSet;
01297 
01302   RewriteContextSet initiated_rewrites_; 
01303 
01311   RewriteContextSet detached_rewrites_; 
01312 
01317   int pending_rewrites_; 
01318 
01320   int possibly_quick_rewrites_; 
01321 
01326   int pending_async_events_; 
01327 
01330   FileSystem* file_system_;
01331   ServerContext* server_context_;
01332   Scheduler* scheduler_;
01333   UrlAsyncFetcher* default_url_async_fetcher_; 
01334 
01338   UrlAsyncFetcher* url_async_fetcher_;
01339 
01343   UrlAsyncFetcher* distributed_async_fetcher_;
01344 
01347   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01348 
01349   AddInstrumentationFilter* add_instrumentation_filter_;
01350   DomStatsFilter* dom_stats_filter_;
01351   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01352 
01353   ScanFilter scan_filter_;
01354   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01355   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01356 
01359   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01360   PrimaryRewriteContextMap primary_rewrite_context_map_;
01361 
01362   HtmlResourceSlotSet slots_;
01363 
01364   scoped_ptr<RewriteOptions> options_;
01365   RewriteDriverPool* controlling_pool_; 
01366 
01368   UrlSegmentEncoder default_encoder_;
01369 
01371   FilterList early_pre_render_filters_;
01373   FilterList pre_render_filters_;
01374 
01378   FilterVector filters_to_delete_;
01379 
01380   QueuedWorkerPool::Sequence* html_worker_;
01381   QueuedWorkerPool::Sequence* rewrite_worker_;
01382   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01383 
01384   Writer* writer_;
01385 
01387   GoogleString client_id_;
01388 
01390   scoped_ptr<AbstractClientState> client_state_;
01391 
01394   FallbackPropertyPage* fallback_property_page_;
01395 
01397   bool owns_property_page_;
01398 
01400   UserAgentMatcher::DeviceType device_type_;
01401 
01402   scoped_ptr<CriticalLineInfo> critical_line_info_;
01403 
01405   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01406 
01409   bool critical_images_info_was_set_;
01410 
01411   scoped_ptr<CriticalCssResult> critical_css_result_;
01412 
01414   bool critical_selector_info_computed_;
01415   scoped_ptr<CriticalSelectorSet> critical_selector_info_;
01416 
01418   bool xhtml_mimetype_computed_;
01419   XhtmlStatus xhtml_status_ : 8;
01420 
01423   int num_inline_preview_images_;
01424 
01426   int num_flushed_early_pagespeed_resources_;
01427 
01429   int num_bytes_in_;
01430 
01431   DebugFilter* debug_filter_;
01432 
01433   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01434   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01435 
01439   bool serve_blink_non_critical_;
01441   bool is_blink_request_;
01442   bool can_rewrite_resources_;
01443 
01446   RequestContextPtr request_context_;
01447 
01449   int64 start_time_ms_;
01450 
01455   bool is_nested_;
01456 
01457   scoped_ptr<DeviceProperties> device_properties_;
01458 
01462   static int initialized_count_;
01463 
01464   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01465 };
01466 
01469 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01470  public:
01471   virtual ~OptionsAwareHTTPCacheCallback();
01472   virtual bool IsCacheValid(const GoogleString& key,
01473                             const ResponseHeaders& headers);
01474   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01475  protected:
01478   OptionsAwareHTTPCacheCallback(
01479       const RewriteOptions* rewrite_options,
01480       const RequestContextPtr& request_ctx);
01481 
01482  private:
01483   const RewriteOptions* rewrite_options_;
01484 
01485   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01486 };
01487 
01488 }  
01489 
01490 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines