Page Speed Optimization Libraries  1.8.31.3
net/instaweb/rewriter/public/rewrite_context.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00021 
00022 #include <set>
00023 #include <vector>
00024 
00025 #include "net/instaweb/http/public/http_cache.h"
00026 #include "net/instaweb/rewriter/cached_result.pb.h"
00027 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00028 #include "net/instaweb/rewriter/public/resource.h"
00029 #include "net/instaweb/rewriter/public/server_context.h"
00030 #include "net/instaweb/rewriter/public/resource_slot.h"
00031 #include "net/instaweb/rewriter/public/rewrite_result.h"
00032 #include "net/instaweb/util/public/basictypes.h"
00033 #include "net/instaweb/util/public/scoped_ptr.h"
00034 #include "net/instaweb/util/public/string.h"
00035 #include "net/instaweb/util/public/string_util.h"
00036 #include "net/instaweb/util/public/url_segment_encoder.h"
00037 
00038 namespace net_instaweb {
00039 
00040 class AsyncFetch;
00041 class GoogleUrl;
00042 class MessageHandler;
00043 class NamedLock;
00044 class RequestTrace;
00045 class ResponseHeaders;
00046 class RewriteDriver;
00047 class RewriteOptions;
00048 class Statistics;
00049 class Variable;
00050 class FreshenMetadataUpdateManager;
00051 
00144 class RewriteContext {
00145  public:
00146   typedef std::vector<InputInfo*> InputInfoStarVector;
00147   static const char kNumRewritesAbandonedForLockContention[];
00148   static const char kNumDeadlineAlarmInvocations[];
00149   static const char kNumDistributedRewriteSuccesses[];
00150   static const char kNumDistributedRewriteFailures[];
00151   static const char kNumDistributedMetadataFailures[];
00153   static const char kDistributedExt[];
00155   static const char kDistributedHash[];
00158   struct CacheLookupResult {
00159     CacheLookupResult()
00160         : cache_ok(false),
00161           can_revalidate(false),
00162           useable_cache_content(false),
00163           is_stale_rewrite(false),
00164           partitions(new OutputPartitions) {}
00165 
00166     bool cache_ok;
00167     bool can_revalidate;
00168     bool useable_cache_content;
00169     bool is_stale_rewrite;
00170     InputInfoStarVector revalidate;
00171     scoped_ptr<OutputPartitions> partitions;
00172   };
00173 
00175   class CacheLookupResultCallback {
00176    public:
00177     CacheLookupResultCallback() {}
00178     virtual ~CacheLookupResultCallback();
00179     virtual void Done(const GoogleString& cache_key,
00180                       CacheLookupResult* result) = 0;
00181    private:
00182     DISALLOW_COPY_AND_ASSIGN(CacheLookupResultCallback);
00183   };
00184 
00187   RewriteContext(RewriteDriver* driver,   
00188                  RewriteContext* parent,  
00189                  ResourceContext* resource_context);
00190   virtual ~RewriteContext();
00191 
00196   int num_slots() const { return slots_.size(); }
00197   ResourceSlotPtr slot(int index) const { return slots_[index]; }
00198 
00201   int num_outputs() const { return outputs_.size(); }
00202   OutputResourcePtr output(int i) const { return outputs_[i]; }
00203 
00206   int num_output_partitions() const;
00207   const CachedResult* output_partition(int i) const;
00208   CachedResult* output_partition(int i);
00209 
00212   bool chained() const { return chained_; }
00213 
00217   void AddSlot(const ResourceSlotPtr& slot);
00218 
00221   void RemoveLastSlot();
00222 
00228   void AddNestedContext(RewriteContext* context);
00229 
00240   void Initiate();
00241 
00247   bool Fetch(const OutputResourcePtr& output_resource,
00248              AsyncFetch* fetch,
00249              MessageHandler* message_handler);
00250 
00253   bool slow() const { return slow_; }
00254 
00256   bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }
00257 
00259   bool has_parent() const { return parent_ != NULL; }
00260 
00263   bool IsNestedIn(StringPiece id) const;
00264 
00266   RewriteContext* parent() { return parent_; }
00267   const RewriteContext* parent() const { return parent_; }
00268 
00270   void set_force_rewrite(bool x) { force_rewrite_ = x; }
00271 
00272   bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
00273   void set_rewrite_uncacheable(bool rewrite_uncacheable) {
00274     rewrite_uncacheable_ = rewrite_uncacheable;
00275   }
00276 
00277   const ResourceContext* resource_context() const {
00278     return resource_context_.get();
00279   }
00280 
00282   GoogleString ToString(StringPiece prefix) const;
00283 
00285   static void InitStats(Statistics* stats);
00286 
00287  protected:
00288   typedef std::vector<GoogleUrl*> GoogleUrlStarVector;
00289 
00294 
00298   ServerContext* FindServerContext() const;
00299   const RewriteOptions* Options() const;
00300   RewriteDriver* Driver() const {
00301     return driver_;
00302   }
00303 
00305   int num_nested() const { return nested_.size(); }
00306   RewriteContext* nested(int i) const { return nested_[i]; }
00307 
00308   OutputPartitions* partitions() { return partitions_.get(); }
00309 
00312   void AddRecheckDependency();
00313 
00317   virtual bool OptimizationOnly() const { return true; }
00318 
00336   virtual bool Partition(OutputPartitions* partitions,
00337                          OutputResourceVector* outputs);
00338 
00344   virtual void PartitionAsync(OutputPartitions* partitions,
00345                               OutputResourceVector* outputs);
00346 
00350   void PartitionDone(RewriteResult result);
00351 
00354   void CrossThreadPartitionDone(RewriteResult result);
00355 
00374   virtual void Rewrite(int partition_index,
00375                        CachedResult* partition,
00376                        const OutputResourcePtr& output) = 0;
00377 
00382   void RewriteDone(RewriteResult result, int partition_index);
00383 
00393   virtual bool SendFallbackResponse(StringPiece output_url_base,
00394                                     StringPiece contents,
00395                                     AsyncFetch* async_fetch,
00396                                     MessageHandler* handler);
00397 
00401   void StartNestedTasks();
00402 
00407   virtual void Harvest();
00408 
00420   virtual void Render();
00421 
00432   virtual void WillNotRender();
00433 
00437   virtual void Cancel();
00438 
00441 
00448   virtual const UrlSegmentEncoder* encoder() const;
00449 
00452   virtual GoogleString CacheKeySuffix() const;
00453 
00460   virtual GoogleString UserAgentCacheKey(
00461       const ResourceContext* context) const {
00462     return "";
00463   }
00464 
00469   virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}
00470 
00472   virtual const char* id() const = 0;
00473 
00482   virtual OutputResourceKind kind() const = 0;
00483 
00487 
00490   void AttachDependentRequestTrace(const StringPiece& label);
00491 
00495   RequestTrace* dependent_request_trace() { return dependent_request_trace_; }
00496 
00499   void TracePrintf(const char* fmt, ...);
00500 
00508 
00518   virtual void StartFetchReconstruction();
00519 
00525   bool ShouldDistributeRewrite() const;
00526 
00529   bool IsDistributedRewriteForHtml() const;
00530 
00534   void DistributeRewrite();
00535 
00541   void DetachFetch();
00542 
00547   virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
00548                                MessageHandler* message_handler,
00549                                GoogleUrlStarVector* url_vector);
00550 
00555   virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
00556                                        ResponseHeaders* headers);
00557 
00560   virtual void FetchCallbackDone(bool success);
00561 
00566   virtual void FetchTryFallback(const GoogleString& url,
00567                                 const StringPiece& hash);
00568 
00570   void Freshen();
00571 
00572   bool notify_driver_on_fetch_done() const {
00573     return notify_driver_on_fetch_done_;
00574   }
00575   void set_notify_driver_on_fetch_done(bool value) {
00576     notify_driver_on_fetch_done_ = value;
00577   }
00578 
00583   bool block_distribute_rewrite() const { return block_distribute_rewrite_; }
00584   void set_block_distribute_rewrite(const bool x) {
00585     block_distribute_rewrite_ = x;
00586   }
00587 
00589   AsyncFetch* async_fetch();
00590 
00592   bool FetchContextDetached();
00593 
00595   MessageHandler* fetch_message_handler();
00596 
00598   bool stale_rewrite() const { return stale_rewrite_; }
00599 
00603   virtual int64 GetRewriteDeadlineAlarmMs() const;
00604 
00606   virtual bool CreationLockBeforeStartFetch() { return true; }
00607 
00611   static bool LookupMetadataForOutputResourceImpl(
00612       OutputResourcePtr output_resource,
00613       const GoogleUrl& gurl,
00614       RewriteContext* rewrite_context,
00615       RewriteDriver* driver,
00616       GoogleString* error_out,
00617       CacheLookupResultCallback* callback);
00618 
00619  private:
00620   class DistributedRewriteCallback;
00621   class DistributedRewriteFetch;
00622   class OutputCacheCallback;
00623   class LookupMetadataForOutputResourceCallback;
00624   class HTTPCacheCallback;
00625   class ResourceCallbackUtils;
00626   class ResourceFetchCallback;
00627   class ResourceReconstructCallback;
00628   class ResourceRevalidateCallback;
00629   class InvokeRewriteFunction;
00630   class RewriteFreshenCallback;
00631   friend class RewriteDriver;
00632 
00633   typedef std::set<RewriteContext*> ContextSet;
00634 
00638   enum FallbackCondition {
00639     kFallbackDiscretional,   
00640 
00641     kFallbackEmergency    
00642 
00643   };
00644 
00646   void Start();
00647   void SetPartitionKey();
00648   void StartFetch();
00649   void StartFetchImpl();
00650   void CancelFetch();
00651   void OutputCacheDone(CacheLookupResult* cache_result);
00652   void OutputCacheHit(bool write_partitions);
00653   void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
00654   void OutputCacheMiss();
00655   void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
00656   void ResourceRevalidateDone(InputInfo* input_info, bool success);
00657   void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);
00658 
00663   void RepeatedSuccess(const RewriteContext* primary);
00664   void RepeatedFailure();
00665 
00674   void Finalize();
00675 
00677   NamedLock* Lock();
00678 
00688   void FetchInputs();
00689 
00694   void DistributeRewriteDone(bool success);
00695 
00699   bool ParseAndRemoveMetadataFromResponseHeaders(
00700       ResponseHeaders* response_headers, CacheLookupResult* cache_result);
00701 
00704   bool CreateOutputResourceFromContent(const CachedResult& cached_result,
00705                                        const ResponseHeaders& response_headers,
00706                                        StringPiece content,
00707                                        OutputResourcePtr* output_resource);
00708 
00718   GoogleString DistributedFetchUrl(StringPiece url);
00719 
00722   bool IsFetchRewrite() const { return fetch_.get() != NULL; }
00723 
00728   void NestedRewriteDone(const RewriteContext* context);
00729 
00733   void Activate();
00734 
00758   void Propagate(bool render_slots);
00759 
00766   void StartRewriteForHtml();
00767   void StartRewriteForFetch();
00768 
00775   bool ReadyToRewrite() const;
00776 
00781   void DetachSlots();
00782 
00785   void RunSuccessors();
00786 
00789   void WritePartition();
00790 
00800   void FinalizeRewriteForHtml();
00801 
00807   void RetireRewriteForHtml(bool permit_render);
00808 
00811   void MarkSlow();
00812 
00815   void MarkTooBusy();
00816 
00820   void CollectDependentTopLevel(ContextSet* contexts);
00821 
00824   void RewriteDoneImpl(RewriteResult result, int partition_index);
00825 
00828   void StartNestedTasksImpl();
00829 
00833   void RenderPartitionOnDetach(int partition_index);
00834 
00837   bool PrepareFetch(
00838       const OutputResourcePtr& output_resource,
00839       AsyncFetch* fetch,
00840       MessageHandler* message_handler);
00841 
00844   bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
00845                                            OutputResourcePtr* output_resource);
00846 
00848   void FetchCacheDone(CacheLookupResult* cache_result);
00849 
00852   void FetchFallbackCacheDone(HTTPCache::FindResult result,
00853                               HTTPCache::Callback* data);
00854 
00859   bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;
00860 
00863   bool HasDuplicateOtherDependency(const InputInfo& input);
00864 
00868   void CheckAndAddOtherDependency(const InputInfo& input);
00869 
00872   void CheckAndFreshenResource(const InputInfo& input_info,
00873                                ResourcePtr resource, int partition_index,
00874                                int input_index,
00875                                FreshenMetadataUpdateManager* freshen_manager);
00876   ResourcePtr CreateUrlResource(const StringPiece& input_url);
00877 
00879   ResourceSlotVector slots_;
00880 
00884   std::vector<bool> render_slots_;
00885 
00896 
00897   bool started_;
00898   scoped_ptr<OutputPartitions> partitions_;
00899   OutputResourceVector outputs_;
00900   int outstanding_fetches_;
00901   int outstanding_rewrites_;
00902   scoped_ptr<ResourceContext> resource_context_;
00903   GoogleString partition_key_;
00904 
00905   UrlSegmentEncoder default_encoder_;
00906 
00909   scoped_ptr<NamedLock> lock_;
00910 
00914   class FetchContext;
00915   scoped_ptr<FetchContext> fetch_;
00916 
00919   std::vector<RewriteContext*> successors_;
00920 
00924   std::vector<RewriteContext*> repeated_;
00925 
00929   int num_pending_nested_;
00930   std::vector<RewriteContext*> nested_;
00931 
00933   RewriteContext* parent_;
00934 
00943   RewriteDriver* driver_;
00944 
00946   int num_predecessors_;
00947 
00950   bool chained_;
00951 
00969 
00971   bool rewrite_done_;
00972 
00984   bool ok_to_write_output_partitions_;
00985 
00989   bool was_too_busy_;
00990 
00994   bool slow_;
00995 
00997   bool revalidate_ok_;
00998 
01001   bool notify_driver_on_fetch_done_;
01002 
01005   bool force_rewrite_;
01006 
01009   bool stale_rewrite_;
01010 
01013   bool is_metadata_cache_miss_;
01014 
01017   bool rewrite_uncacheable_;
01018 
01021   RequestTrace* dependent_request_trace_;
01022 
01025   bool block_distribute_rewrite_;
01026 
01028   scoped_ptr<DistributedRewriteFetch> distributed_fetch_;
01029 
01031   StringIntMap other_dependency_map_;
01032 
01033   Variable* const num_rewrites_abandoned_for_lock_contention_;
01034   Variable* const num_distributed_rewrite_failures_;
01035   Variable* const num_distributed_rewrite_successes_;
01036   Variable* const num_distributed_metadata_failures_;
01037   DISALLOW_COPY_AND_ASSIGN(RewriteContext);
01038 };
01039 
01040 }  
01041 
01042 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines