Page Speed Optimization Libraries  1.7.30.4
net/instaweb/rewriter/public/rewrite_driver.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
00021 
00022 #include <map>
00023 #include <set>
00024 #include <vector>
00025 
00026 #include "base/logging.h"
00027 #include "net/instaweb/htmlparse/public/html_element.h"
00028 #include "net/instaweb/htmlparse/public/html_parse.h"
00029 #include "net/instaweb/http/public/cache_url_async_fetcher.h"
00030 #include "net/instaweb/http/public/http_cache.h"
00031 #include "net/instaweb/http/public/request_context.h"
00032 #include "net/instaweb/http/public/user_agent_matcher.h"
00033 #include "net/instaweb/rewriter/public/critical_images_finder.h"
00034 #include "net/instaweb/rewriter/public/critical_selector_finder.h"
00035 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/resource_slot.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/scan_filter.h"
00040 #include "net/instaweb/rewriter/public/server_context.h"
00041 #include "net/instaweb/util/public/basictypes.h"
00042 #include "net/instaweb/util/public/google_url.h"
00043 #include "net/instaweb/util/public/printf_format.h"
00044 #include "net/instaweb/util/public/queued_worker_pool.h"
00045 #include "net/instaweb/util/public/scheduler.h"
00046 #include "net/instaweb/util/public/scoped_ptr.h"
00047 #include "net/instaweb/util/public/string.h"
00048 #include "net/instaweb/util/public/string_util.h"
00049 #include "net/instaweb/util/public/thread_system.h"
00050 #include "net/instaweb/util/public/url_segment_encoder.h"
00051 #include "pagespeed/kernel/base/thread_annotations.h"
00052 #include "pagespeed/kernel/http/content_type.h"
00053 #include "pagespeed/kernel/http/response_headers.h"
00054 #include "pagespeed/kernel/util/categorized_refcount.h"
00055 
00056 namespace net_instaweb {
00057 
00058 class AbstractLogRecord;
00059 class AbstractMutex;
00060 class AbstractPropertyPage;
00061 class AsyncFetch;
00062 class CriticalCssResult;
00063 class CriticalKeys;
00064 class CriticalLineInfo;
00065 class DebugFilter;
00066 class DomStatsFilter;
00067 class DomainRewriteFilter;
00068 class FallbackPropertyPage;
00069 class FileSystem;
00070 class FlushEarlyInfo;
00071 class FlushEarlyRenderInfo;
00072 class Function;
00073 class HtmlFilter;
00074 class HtmlWriterFilter;
00075 class MessageHandler;
00076 class OutputResource;
00077 class PropertyPage;
00078 class RequestHeaders;
00079 class RequestProperties;
00080 class RequestTrace;
00081 class ResourceContext;
00082 class ResourceNamer;
00083 class RewriteContext;
00084 class RewriteDriverPool;
00085 class RewriteFilter;
00086 class SplitHtmlConfig;
00087 class Statistics;
00088 class UrlAsyncFetcher;
00089 class UrlLeftTrimFilter;
00090 class UrlNamer;
00091 class Writer;
00092 
00095 class RewriteDriver : public HtmlParse {
00096  public:
00098   enum CssResolutionStatus {
00099     kWriteFailed,
00100     kNoResolutionNeeded,
00101     kSuccess
00102   };
00103 
00105   enum WaitMode {
00106     kNoWait,  
00107     kWaitForCompletion,    
00108     kWaitForCachedRender,  
00109 
00110     kWaitForShutDown       
00111 
00112   };
00113 
00125   enum XhtmlStatus {
00126     kXhtmlUnknown,
00127     kIsXhtml,
00128     kIsNotXhtml
00129   };
00130 
00132   enum InlineAuthorizationPolicy {
00133     kInlineUnauthorizedResources,
00134     kInlineOnlyAuthorizedResources
00135   };
00136 
00138   enum IntendedFor {
00139     kIntendedForInlining,
00140     kIntendedForGeneral
00141   };
00142 
00146   static const char kDomCohort[];
00148   static const char kBeaconCohort[];
00149 
00152   static const char kLastRequestTimestamp[];
00154   static const char kParseSizeLimitExceeded[];
00156   static const char kSubresourcesPropertyName[];
00158   static const char kStatusCodePropertyName[];
00159 
00160   RewriteDriver(MessageHandler* message_handler,
00161                 FileSystem* file_system,
00162                 UrlAsyncFetcher* url_async_fetcher);
00163 
00166   virtual ~RewriteDriver();
00167 
00170   RewriteDriver* Clone();
00171 
00177   void Clear();
00178 
00180   static void InitStats(Statistics* statistics);
00181 
00183   static void Initialize();
00184   static void Terminate();
00185 
00188   void SetServerContext(ServerContext* server_context);
00189 
00192   bool MayCacheExtendCss() const;
00193   bool MayCacheExtendImages() const;
00194   bool MayCacheExtendPdfs() const;
00195   bool MayCacheExtendScripts() const;
00196 
00197   const GoogleString& user_agent() const { return user_agent_; }
00198 
00199   void SetUserAgent(const StringPiece& user_agent_string);
00200 
00201   const RequestProperties* request_properties() const {
00202     return request_properties_.get();
00203   }
00204 
00206   void ClearRequestProperties();
00207 
00209   bool using_spdy() const { return request_context_->using_spdy(); }
00210 
00211   bool write_property_cache_dom_cohort() const {
00212     return write_property_cache_dom_cohort_;
00213   }
00214   void set_write_property_cache_dom_cohort(bool x) {
00215     write_property_cache_dom_cohort_ = x;
00216   }
00217 
00218   RequestContextPtr request_context() { return request_context_; }
00219   void set_request_context(const RequestContextPtr& x);
00220 
00223   RequestTrace* trace_context();
00224 
00227   void TracePrintf(const char* fmt, ...);
00228 
00231   ResponseHeaders* mutable_response_headers() {
00232     return flush_occurred_ ? NULL : response_headers_;
00233   }
00234 
00241   const ResponseHeaders* response_headers() {
00242     return response_headers_;
00243   }
00244 
00248   void set_response_headers_ptr(ResponseHeaders* headers) {
00249     response_headers_ = headers;
00250   }
00251 
00257   void SetRequestHeaders(const RequestHeaders& headers);
00258 
00259   const RequestHeaders* request_headers() const {
00260     return request_headers_.get();
00261   }
00262 
00263   UserAgentMatcher* user_agent_matcher() const {
00264     DCHECK(server_context() != NULL);
00265     return server_context()->user_agent_matcher();
00266   }
00267 
00272   void AddFilters();
00273 
00277   void AddOwnedEarlyPreRenderFilter(HtmlFilter* filter);
00278 
00280   void PrependOwnedPreRenderFilter(HtmlFilter* filter);
00282   void AppendOwnedPreRenderFilter(HtmlFilter* filter);
00283 
00285   void AddOwnedPostRenderFilter(HtmlFilter* filter);
00287   void AddUnownedPostRenderFilter(HtmlFilter* filter);
00288 
00300   void AppendRewriteFilter(RewriteFilter* filter);
00301 
00304   void PrependRewriteFilter(RewriteFilter* filter);
00305 
00313   void AddResourceUrlClaimant(ResourceUrlClaimant* claimant);
00314 
00320   void SetWriter(Writer* writer);
00321 
00322   Writer* writer() const { return writer_; }
00323 
00352   bool FetchResource(const StringPiece& url, AsyncFetch* fetch);
00353 
00371   void FetchInPlaceResource(const GoogleUrl& gurl, bool proxy_mode,
00372                             AsyncFetch* async_fetch);
00373 
00380   bool FetchOutputResource(const OutputResourcePtr& output_resource,
00381                            RewriteFilter* filter,
00382                            AsyncFetch* async_fetch);
00383 
00390   OutputResourcePtr DecodeOutputResource(const GoogleUrl& url,
00391                                          RewriteFilter** filter) const;
00392 
00398   bool DecodeOutputResourceName(const GoogleUrl& url,
00399                                 const RewriteOptions* options_to_use,
00400                                 const UrlNamer* url_namer,
00401                                 ResourceNamer* name_out,
00402                                 OutputResourceKind* kind_out,
00403                                 RewriteFilter** filter_out) const;
00404 
00406   bool DecodeUrl(const GoogleUrl& url,
00407                  StringVector* decoded_urls) const;
00408 
00411   bool DecodeUrlGivenOptions(const GoogleUrl& url,
00412                              const RewriteOptions* options,
00413                              const UrlNamer* url_namer,
00414                              StringVector* decoded_urls) const;
00415 
00416   FileSystem* file_system() { return file_system_; }
00417   UrlAsyncFetcher* async_fetcher() { return url_async_fetcher_; }
00418 
00423   void SetSessionFetcher(UrlAsyncFetcher* f);
00424 
00425   UrlAsyncFetcher* distributed_fetcher() { return distributed_async_fetcher_; }
00427   void set_distributed_fetcher(UrlAsyncFetcher* fetcher) {
00428     distributed_async_fetcher_ = fetcher;
00429   }
00430 
00433   CacheUrlAsyncFetcher* CreateCacheFetcher();
00435   CacheUrlAsyncFetcher* CreateCacheOnlyFetcher();
00436 
00437   ServerContext* server_context() const { return server_context_; }
00438   Statistics* statistics() const;
00439 
00441   void set_custom_options(RewriteOptions* options) {
00442     set_options_for_pool(NULL, options);
00443   }
00444 
00447   void set_options_for_pool(RewriteDriverPool* pool, RewriteOptions* options) {
00448     controlling_pool_ = pool;
00449     options_.reset(options);
00450   }
00451 
00453   RewriteDriverPool* controlling_pool() { return controlling_pool_; }
00454 
00456   const RewriteOptions* options() const { return options_.get(); }
00457 
00461   virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
00462                             const ContentType& content_type);
00463 
00470   virtual void FinishParse();
00471 
00475   void FinishParseAsync(Function* callback);
00476 
00480   void InfoAt(const RewriteContext* context,
00481               const char* msg, ...) INSTAWEB_PRINTF_FORMAT(3, 4);
00482 
00487 
00494   OutputResourcePtr CreateOutputResourceFromResource(
00495       const StringPiece& filter_id,
00496       const UrlSegmentEncoder* encoder,
00497       const ResourceContext* data,
00498       const ResourcePtr& input_resource,
00499       OutputResourceKind kind);
00500 
00517   OutputResourcePtr CreateOutputResourceWithPath(
00518       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00519       const StringPiece& base_url, const StringPiece& filter_id,
00520       const StringPiece& name, OutputResourceKind kind);
00521 
00524   void PopulateResourceNamer(
00525     const StringPiece& filter_id,
00526     const StringPiece& name,
00527     ResourceNamer* full_name);
00528 
00532   OutputResourcePtr CreateOutputResourceWithUnmappedUrl(
00533       const GoogleUrl& unmapped_gurl, const StringPiece& filter_id,
00534       const StringPiece& name, OutputResourceKind kind);
00535 
00538   OutputResourcePtr CreateOutputResourceWithMappedPath(
00539       const StringPiece& mapped_path, const StringPiece& unmapped_path,
00540       const StringPiece& filter_id, const StringPiece& name,
00541       OutputResourceKind kind) {
00542     return CreateOutputResourceWithPath(mapped_path, unmapped_path,
00543                                         decoded_base_url_.AllExceptLeaf(),
00544                                         filter_id, name, kind);
00545   }
00546 
00549   OutputResourcePtr CreateOutputResourceWithPath(
00550       const StringPiece& path, const StringPiece& filter_id,
00551       const StringPiece& name, OutputResourceKind kind) {
00552     return CreateOutputResourceWithPath(path, path, path, filter_id, name,
00553                                         kind);
00554   }
00555 
00560   ResourcePtr CreateInputResource(const GoogleUrl& input_url);
00561 
00575   ResourcePtr CreateInputResource(
00576       const GoogleUrl& input_url,
00577       InlineAuthorizationPolicy inline_authorization_policy,
00578       IntendedFor intended_for);
00579 
00583   ResourcePtr CreateInputResourceAbsoluteUnchecked(
00584       const StringPiece& absolute_url);
00585 
00589   bool IsResourceUrlClaimed(const GoogleUrl& url) const;
00590 
00595   bool MatchesBaseUrl(const GoogleUrl& input_url) const;
00596 
00602   bool MayRewriteUrl(const GoogleUrl& domain_url,
00603                      const GoogleUrl& input_url,
00604                      InlineAuthorizationPolicy inline_authorization_policy,
00605                      IntendedFor intended_for,
00606                      bool* is_authorized_domain) const;
00607 
00611   const GoogleUrl& base_url() const { return base_url_; }
00612 
00614   StringPiece fetch_url() const { return fetch_url_; }
00615 
00619   const GoogleUrl& decoded_base_url() const { return decoded_base_url_; }
00620   StringPiece decoded_base() const { return decoded_base_url_.Spec(); }
00621 
00623   bool IsHttps() const { return google_url().SchemeIs("https"); }
00624 
00625   const UrlSegmentEncoder* default_encoder() const { return &default_encoder_; }
00626 
00628   RewriteFilter* FindFilter(const StringPiece& id) const;
00629 
00631   bool refs_before_base() { return refs_before_base_; }
00632 
00637   void set_refs_before_base() { refs_before_base_ = true; }
00638 
00643   StringPiece containing_charset() { return containing_charset_; }
00644   void set_containing_charset(const StringPiece charset) {
00645     charset.CopyToString(&containing_charset_);
00646   }
00647 
00649   HtmlResourceSlotPtr GetSlot(const ResourcePtr& resource,
00650                               HtmlElement* elt,
00651                               HtmlElement::Attribute* attr);
00652 
00657   bool InitiateRewrite(RewriteContext* rewrite_context)
00658       LOCKS_EXCLUDED(rewrite_mutex());
00659   void InitiateFetch(RewriteContext* rewrite_context);
00660 
00671   void RewriteComplete(RewriteContext* rewrite_context, bool permit_render);
00672 
00676   void ReportSlowRewrites(int num);
00677 
00682   void Cleanup();
00683 
00688   void AddUserReference();
00689 
00691   GoogleString ToString(bool show_detached_contexts);
00692   void PrintState(bool show_detached_contexts); 
00693   void PrintStateToErrorLog(bool show_detached_contexts); 
00694 
00697   void WaitForCompletion();
00698 
00705   void WaitForShutDown();
00706 
00710   void BoundedWaitFor(WaitMode mode, int64 timeout_ms)
00711       LOCKS_EXCLUDED(rewrite_mutex());
00712 
00720   void set_fully_rewrite_on_flush(bool x) {
00721     fully_rewrite_on_flush_ = x;
00722   }
00723 
00725   bool fully_rewrite_on_flush() const {
00726     return fully_rewrite_on_flush_;
00727   }
00728 
00732   void set_fast_blocking_rewrite(bool x) {
00733     fast_blocking_rewrite_ = x;
00734   }
00735 
00736   bool fast_blocking_rewrite() const {
00737     return fast_blocking_rewrite_;
00738   }
00739 
00742   void EnableBlockingRewrite(RequestHeaders* request_headers);
00743 
00750   void set_externally_managed(bool x) { externally_managed_ = x; }
00751 
00755   void DetachFetch();
00756 
00759   void DetachedFetchComplete();
00760 
00764   void FetchComplete();
00765 
00771   void DeleteRewriteContext(RewriteContext* rewrite_context);
00772 
00773   int rewrite_deadline_ms() { return options()->rewrite_deadline_ms(); }
00774 
00779   void set_max_page_processing_delay_ms(int x) {
00780     max_page_processing_delay_ms_ = x;
00781   }
00782   int max_page_processing_delay_ms() { return max_page_processing_delay_ms_; }
00783 
00785   void set_device_type(UserAgentMatcher::DeviceType x) { device_type_ = x; }
00786   UserAgentMatcher::DeviceType device_type() const { return device_type_; }
00787 
00793   RewriteContext* RegisterForPartitionKey(const GoogleString& partition_key,
00794                                           RewriteContext* candidate);
00795 
00800   void DeregisterForPartitionKey(
00801       const GoogleString& partition_key, RewriteContext* candidate);
00802 
00805   void RequestFlush() { flush_requested_ = true; }
00806   bool flush_requested() const { return flush_requested_; }
00807 
00819   void ExecuteFlushIfRequested();
00820 
00824   void ExecuteFlushIfRequestedAsync(Function* callback);
00825 
00834   virtual void Flush();
00835 
00839   void FlushAsync(Function* done);
00840 
00842   void AddRewriteTask(Function* task);
00843 
00846   void AddLowPriorityRewriteTask(Function* task);
00847 
00848   QueuedWorkerPool::Sequence* html_worker() { return html_worker_; }
00849   QueuedWorkerPool::Sequence* rewrite_worker() { return rewrite_worker_; }
00850   QueuedWorkerPool::Sequence* low_priority_rewrite_worker() {
00851     return low_priority_rewrite_worker_;
00852   }
00853 
00854   Scheduler* scheduler() { return scheduler_; }
00855 
00858   DomainRewriteFilter* domain_rewriter() { return domain_rewriter_.get(); }
00859   UrlLeftTrimFilter* url_trim_filter() { return url_trim_filter_.get(); }
00860 
00868   CssResolutionStatus ResolveCssUrls(const GoogleUrl& input_css_base,
00869                                      const StringPiece& output_css_base,
00870                                      const StringPiece& contents,
00871                                      Writer* writer,
00872                                      MessageHandler* handler);
00873 
00881   bool ShouldAbsolutifyUrl(const GoogleUrl& input_base,
00882                            const GoogleUrl& output_base,
00883                            bool* proxy_mode) const;
00884 
00892   void UpdatePropertyValueInDomCohort(
00893       AbstractPropertyPage* page,
00894       StringPiece property_name,
00895       StringPiece property_value);
00896 
00899   PropertyPage* property_page() const;
00904   FallbackPropertyPage* fallback_property_page() const {
00905     return fallback_property_page_;
00906   }
00908   void set_property_page(PropertyPage* page);
00910   void set_fallback_property_page(FallbackPropertyPage* page);
00912   void set_unowned_fallback_property_page(FallbackPropertyPage* page);
00913 
00915   const CriticalLineInfo* critical_line_info() const;
00916 
00919   void set_critical_line_info(CriticalLineInfo* critical_line_info);
00920 
00921   CriticalKeys* beacon_critical_line_info() const;
00922   void set_beacon_critical_line_info(CriticalKeys* beacon_critical_line_info);
00923 
00924   const SplitHtmlConfig* split_html_config();
00925 
00926   CriticalCssResult* critical_css_result() const;
00929   void set_critical_css_result(CriticalCssResult* critical_css_rules);
00930 
00932   CriticalImagesInfo* critical_images_info() const {
00933     return critical_images_info_.get();
00934   }
00935 
00940   CriticalSelectorInfo* critical_selector_info() {
00941     return critical_selector_info_.get();
00942   }
00943 
00947   void set_critical_selector_info(CriticalSelectorInfo* info) {
00948     critical_selector_info_.reset(info);
00949   }
00950 
00955   void set_critical_images_info(CriticalImagesInfo* critical_images_info) {
00956     critical_images_info_.reset(critical_images_info);
00957   }
00958 
00961   bool CriticalSelectorsEnabled() const;
00962 
00965   bool FlattenCssImportsEnabled() const {
00966     return (options()->Enabled(RewriteOptions::kFlattenCssImports) ||
00967             (!options()->Forbidden(RewriteOptions::kFlattenCssImports) &&
00968              (CriticalSelectorsEnabled() ||
00969               options()->Enabled(RewriteOptions::kComputeCriticalCss))));
00970   }
00971 
00975   int num_inline_preview_images() const { return num_inline_preview_images_; }
00976 
00978   void increment_num_inline_preview_images();
00979 
00982   int num_flushed_early_pagespeed_resources() const {
00983     return num_flushed_early_pagespeed_resources_;
00984   }
00985 
00988   void increment_num_flushed_early_pagespeed_resources() {
00989     ++num_flushed_early_pagespeed_resources_;
00990   }
00991 
00994   void increment_async_events_count();
00995 
00997   void decrement_async_events_count();
00998 
01001   XhtmlStatus MimeTypeXhtmlStatus();
01002 
01003   void set_flushed_cached_html(bool x) { flushed_cached_html_ = x; }
01004   bool flushed_cached_html() { return flushed_cached_html_; }
01005 
01006   void set_flushing_cached_html(bool x) { flushing_cached_html_ = x; }
01007   bool flushing_cached_html() const { return flushing_cached_html_; }
01008 
01009   void set_flushed_early(bool x) { flushed_early_ = x; }
01010   bool flushed_early() const { return flushed_early_; }
01011 
01012   void set_flushing_early(bool x) { flushing_early_ = x; }
01013   bool flushing_early() const { return flushing_early_; }
01014 
01015   void set_is_lazyload_script_flushed(bool x) {
01016     is_lazyload_script_flushed_ = x;
01017   }
01018   bool is_lazyload_script_flushed() const {
01019     return is_lazyload_script_flushed_; }
01020 
01022   FlushEarlyInfo* flush_early_info();
01023 
01024   FlushEarlyRenderInfo* flush_early_render_info() const;
01025 
01028   void set_flush_early_render_info(
01029       FlushEarlyRenderInfo* flush_early_render_info);
01030 
01033   bool DebugMode() const { return options()->Enabled(RewriteOptions::kDebug); }
01034 
01037   void SaveOriginalHeaders(const ResponseHeaders& response_headers);
01038 
01041   AbstractLogRecord* log_record();
01042 
01043   DomStatsFilter* dom_stats_filter() const {
01044     return dom_stats_filter_;
01045   }
01046 
01049   bool can_rewrite_resources() const { return can_rewrite_resources_; }
01050 
01052   bool is_nested() const { return is_nested_; }
01053 
01057   bool MetadataRequested(const RequestHeaders& request_headers) const;
01058 
01060   bool tried_to_distribute_fetch() const { return tried_to_distribute_fetch_; }
01061 
01074   bool Write(const ResourceVector& inputs,
01075              const StringPiece& contents,
01076              const ContentType* type,
01077              StringPiece charset,
01078              OutputResource* output);
01079 
01080   void set_defer_instrumentation_script(bool x) {
01081     defer_instrumentation_script_ = x;
01082   }
01083   bool defer_instrumentation_script() const {
01084     return defer_instrumentation_script_;
01085   }
01086 
01089   const GoogleString& CacheFragment() const;
01090 
01091  protected:
01092   virtual void DetermineEnabledFiltersImpl();
01093 
01094  private:
01095   friend class DistributedRewriteContextTest;
01096   friend class RewriteContext;
01097   friend class RewriteDriverTest;
01098   friend class RewriteTestBase;
01099   friend class ServerContextTest;
01100 
01101   typedef std::map<GoogleString, RewriteFilter*> StringFilterMap;
01102 
01104   bool ShouldDistributeFetch(const StringPiece& filter_id);
01105 
01117   bool DistributeFetch(const StringPiece& url, const StringPiece& filter_id,
01118                        AsyncFetch* async_fetch);
01119 
01124   void CheckForCompletionAsync(WaitMode wait_mode, int64 timeout_ms,
01125                                Function* done)
01126       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01127 
01131   void TryCheckForCompletion(WaitMode wait_mode, int64 end_time_ms,
01132                              Function* done)
01133       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01134 
01136   bool IsDone(WaitMode wait_mode, bool deadline_reached)
01137       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01138 
01141   bool WaitForPendingAsyncEvents(WaitMode wait_mode) {
01142     return wait_mode == kWaitForShutDown ||
01143         (fully_rewrite_on_flush_ && !fast_blocking_rewrite_);
01144   }
01145 
01149   void FlushAsyncDone(int num_rewrites, Function* callback);
01150 
01155   int64 ComputeCurrentFlushWindowRewriteDelayMs();
01156 
01158   void QueueFlushAsyncDone(int num_rewrites, Function* callback);
01159 
01162   void QueueFinishParseAfterFlush(Function* user_callback);
01163   void FinishParseAfterFlush(Function* user_callback);
01164 
01165   bool RewritesComplete() const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01166 
01169   void SetBaseUrlIfUnset(const StringPiece& new_base);
01170 
01173   void SetBaseUrlForFetch(const StringPiece& url);
01174 
01177   void SetDecodedUrlFromBase();
01178 
01180   AbstractMutex* rewrite_mutex() const LOCK_RETURNED(scheduler_->mutex()) {
01181     return scheduler_->mutex();
01182   }
01183 
01185   virtual void ParseTextInternal(const char* content, int size);
01186 
01188   bool ShouldSkipParsing();
01189 
01190   friend class ScanFilter;
01191 
01195   void RegisterRewriteFilter(RewriteFilter* filter);
01196 
01201   void EnableRewriteFilter(const char* id);
01202 
01208   ResourcePtr CreateInputResourceUnchecked(const GoogleUrl& gurl,
01209                                            bool is_authorized_domain);
01210 
01211   void AddPreRenderFilters();
01212   void AddPostRenderFilters();
01213 
01215   bool DecodeOutputResourceNameHelper(const GoogleUrl& url,
01216                                       const RewriteOptions* options_to_use,
01217                                       const UrlNamer* url_namer,
01218                                       ResourceNamer* name_out,
01219                                       OutputResourceKind* kind_out,
01220                                       RewriteFilter** filter_out,
01221                                       GoogleString* url_base,
01222                                       StringVector* urls) const;
01223 
01233   void WriteDomCohortIntoPropertyCache();
01234 
01236   CacheUrlAsyncFetcher* CreateCustomCacheFetcher(UrlAsyncFetcher* base_fetcher);
01237 
01244   void PossiblyPurgeCachedResponseAndReleaseDriver();
01245 
01250   bool ShouldPurgeRewrittenResponse();
01251 
01254   static bool GetPurgeUrl(const GoogleUrl& google_url,
01255                           const RewriteOptions* options,
01256                           GoogleString* purge_url,
01257                           GoogleString* purge_method);
01258 
01260   void PurgeDownstreamCache(const GoogleString& purge_url,
01261                             const GoogleString& purge_method);
01262 
01264   void LogStats();
01265 
01281   bool PrepareShouldSignal() EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01282   void SignalIfRequired(bool result_of_prepare_should_signal)
01283       EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex());
01284 
01296   bool base_was_set_;
01297 
01302   bool refs_before_base_;
01303 
01305   GoogleString containing_charset_;
01306 
01309   void PopulateRequestContext();
01310 
01311   bool filters_added_;
01312   bool externally_managed_;
01313 
01322   enum RefCategory {
01323     kRefUser,  
01324     kRefParsing,  
01325 
01329     kRefPendingRewrites,
01330 
01334     kRefDetachedRewrites,
01335 
01343     kRefDeletingRewrites,
01344 
01346     kRefFetchUserFacing,
01347 
01349     kRefFetchBackground,
01350 
01355     kRefAsyncEvents,
01356 
01357     kNumRefCategories
01358   };
01359 
01360   friend class CategorizedRefcount<RewriteDriver, RefCategory>;
01361 
01363   CategorizedRefcount<RewriteDriver, RefCategory> ref_counts_;
01364 
01366   void LastRefRemoved();
01367   StringPiece RefCategoryName(RefCategory cat);
01368 
01371   void DropReference(RefCategory cat);
01372 
01375   bool release_driver_;
01376 
01379   bool parsing_ GUARDED_BY(rewrite_mutex());
01380 
01384   WaitMode waiting_ GUARDED_BY(rewrite_mutex());
01385 
01387   bool waiting_deadline_reached_ GUARDED_BY(rewrite_mutex());
01388 
01393   bool fully_rewrite_on_flush_;
01394 
01397   bool fast_blocking_rewrite_;
01398 
01399   bool flush_requested_;
01400   bool flush_occurred_;
01401 
01403   bool flushed_cached_html_;
01404 
01406   bool flushing_cached_html_;
01407 
01410   bool flushed_early_;
01414   bool flushing_early_;
01415 
01418   bool is_lazyload_script_flushed_;
01419 
01422   bool made_downstream_purge_attempt_;
01423 
01427   bool write_property_cache_dom_cohort_;
01428 
01431   GoogleUrl base_url_;
01432 
01436   GoogleUrl decoded_base_url_;
01437 
01440   GoogleString fetch_url_;
01441 
01442   GoogleString user_agent_;
01443 
01444   LazyBool should_skip_parsing_;
01445 
01446   StringFilterMap resource_filter_map_;
01447 
01448   ResponseHeaders* response_headers_;
01449 
01452   scoped_ptr<const RequestHeaders> request_headers_;
01453 
01454   int status_code_; 
01455 
01458   typedef std::vector<RewriteContext*> RewriteContextVector;
01459   RewriteContextVector rewrites_; 
01460 
01463   int max_page_processing_delay_ms_;
01464 
01465   typedef std::set<RewriteContext*> RewriteContextSet;
01466 
01471   RewriteContextSet initiated_rewrites_ GUARDED_BY(rewrite_mutex());
01472 
01474   int64 num_initiated_rewrites_ GUARDED_BY(rewrite_mutex());
01475 
01483   int64 num_detached_rewrites_ GUARDED_BY(rewrite_mutex());
01484 
01492   RewriteContextSet detached_rewrites_ GUARDED_BY(rewrite_mutex());
01493 
01495   int possibly_quick_rewrites_ GUARDED_BY(rewrite_mutex());
01496 
01499   RewriteContextVector fetch_rewrites_;
01500 
01503   FileSystem* file_system_;
01504   ServerContext* server_context_;
01505   Scheduler* scheduler_;
01506   UrlAsyncFetcher* default_url_async_fetcher_; 
01507 
01511   UrlAsyncFetcher* url_async_fetcher_;
01512 
01516   UrlAsyncFetcher* distributed_async_fetcher_;
01517 
01520   std::vector<UrlAsyncFetcher*> owned_url_async_fetchers_;
01521 
01522   DomStatsFilter* dom_stats_filter_;
01523   scoped_ptr<HtmlWriterFilter> html_writer_filter_;
01524 
01525   ScanFilter scan_filter_;
01526   scoped_ptr<DomainRewriteFilter> domain_rewriter_;
01527   scoped_ptr<UrlLeftTrimFilter> url_trim_filter_;
01528 
01531   typedef std::map<GoogleString, RewriteContext*> PrimaryRewriteContextMap;
01532   PrimaryRewriteContextMap primary_rewrite_context_map_;
01533 
01534   HtmlResourceSlotSet slots_;
01535 
01536   scoped_ptr<RewriteOptions> options_;
01537 
01538   RewriteDriverPool* controlling_pool_; 
01539 
01541   scoped_ptr<CacheUrlAsyncFetcher::AsyncOpHooks>
01542       cache_url_async_fetcher_async_op_hooks_;
01543 
01545   UrlSegmentEncoder default_encoder_;
01546 
01548   FilterList early_pre_render_filters_;
01550   FilterList pre_render_filters_;
01551 
01553   std::vector<ResourceUrlClaimant*> resource_claimants_;
01554 
01558   FilterVector filters_to_delete_;
01559 
01560   QueuedWorkerPool::Sequence* html_worker_;
01561   QueuedWorkerPool::Sequence* rewrite_worker_;
01562   QueuedWorkerPool::Sequence* low_priority_rewrite_worker_;
01563 
01564   Writer* writer_;
01565 
01568   FallbackPropertyPage* fallback_property_page_;
01569 
01571   bool owns_property_page_;
01572 
01574   UserAgentMatcher::DeviceType device_type_;
01575 
01576   scoped_ptr<CriticalLineInfo> critical_line_info_;
01577   scoped_ptr<CriticalKeys> beacon_critical_line_info_;
01578 
01579   scoped_ptr<SplitHtmlConfig> split_html_config_;
01580 
01583   scoped_ptr<CriticalImagesInfo> critical_images_info_;
01584   scoped_ptr<CriticalSelectorInfo> critical_selector_info_;
01585 
01586   scoped_ptr<CriticalCssResult> critical_css_result_;
01587 
01589   bool xhtml_mimetype_computed_;
01590   XhtmlStatus xhtml_status_ : 8;
01591 
01594   int num_inline_preview_images_;
01595 
01597   int num_flushed_early_pagespeed_resources_;
01598 
01600   int num_bytes_in_;
01601 
01602   DebugFilter* debug_filter_;
01603 
01604   scoped_ptr<FlushEarlyInfo> flush_early_info_;
01605   scoped_ptr<FlushEarlyRenderInfo> flush_early_render_info_;
01606 
01607   bool can_rewrite_resources_;
01608   bool is_nested_;
01609 
01612   RequestContextPtr request_context_;
01613 
01615   int64 start_time_ms_;
01616 
01617   scoped_ptr<RequestProperties> request_properties_;
01618 
01622   static int initialized_count_;
01623 
01626   bool tried_to_distribute_fetch_;
01627 
01630   bool defer_instrumentation_script_;
01631 
01632   DISALLOW_COPY_AND_ASSIGN(RewriteDriver);
01633 };
01634 
01637 class OptionsAwareHTTPCacheCallback : public HTTPCache::Callback {
01638  public:
01639   virtual ~OptionsAwareHTTPCacheCallback();
01640   virtual bool IsCacheValid(const GoogleString& key,
01641                             const ResponseHeaders& headers);
01642   virtual int64 OverrideCacheTtlMs(const GoogleString& key);
01643   virtual ResponseHeaders::VaryOption RespectVaryOnResources() const;
01644 
01648   static bool IsCacheValid(const GoogleString& key,
01649                            const RewriteOptions& rewrite_options,
01650                            const RequestContextPtr& request_ctx,
01651                            const ResponseHeaders& headers);
01652 
01653  protected:
01656   OptionsAwareHTTPCacheCallback(
01657       const RewriteOptions* rewrite_options,
01658       const RequestContextPtr& request_ctx);
01659 
01660  private:
01661   const RewriteOptions* rewrite_options_;
01662 
01663   DISALLOW_COPY_AND_ASSIGN(OptionsAwareHTTPCacheCallback);
01664 };
01665 
01666 }  
01667 
01668 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_DRIVER_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines