Page Speed Optimization Libraries  1.8.31.2
net/instaweb/rewriter/public/rewrite_context.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00021 
00022 #include <set>
00023 #include <vector>
00024 
00025 #include "net/instaweb/http/public/http_cache.h"
00026 #include "net/instaweb/rewriter/cached_result.pb.h"
00027 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00028 #include "net/instaweb/rewriter/public/resource.h"
00029 #include "net/instaweb/rewriter/public/server_context.h"
00030 #include "net/instaweb/rewriter/public/resource_slot.h"
00031 #include "net/instaweb/rewriter/public/rewrite_result.h"
00032 #include "net/instaweb/util/public/basictypes.h"
00033 #include "net/instaweb/util/public/scoped_ptr.h"
00034 #include "net/instaweb/util/public/string.h"
00035 #include "net/instaweb/util/public/string_util.h"
00036 #include "net/instaweb/util/public/url_segment_encoder.h"
00037 
00038 namespace net_instaweb {
00039 
00040 class AsyncFetch;
00041 class GoogleUrl;
00042 class MessageHandler;
00043 class NamedLock;
00044 class RequestTrace;
00045 class ResponseHeaders;
00046 class RewriteDriver;
00047 class RewriteOptions;
00048 class Statistics;
00049 class Variable;
00050 class Writer;
00051 class FreshenMetadataUpdateManager;
00052 
00145 class RewriteContext {
00146  public:
00147   typedef std::vector<InputInfo*> InputInfoStarVector;
00148   static const char kNumRewritesAbandonedForLockContention[];
00149   static const char kNumDeadlineAlarmInvocations[];
00150   static const char kNumDistributedRewriteSuccesses[];
00151   static const char kNumDistributedRewriteFailures[];
00152   static const char kNumDistributedMetadataFailures[];
00154   static const char kDistributedExt[];
00156   static const char kDistributedHash[];
00159   struct CacheLookupResult {
00160     CacheLookupResult()
00161         : cache_ok(false),
00162           can_revalidate(false),
00163           useable_cache_content(false),
00164           is_stale_rewrite(false),
00165           partitions(new OutputPartitions) {}
00166 
00167     bool cache_ok;
00168     bool can_revalidate;
00169     bool useable_cache_content;
00170     bool is_stale_rewrite;
00171     InputInfoStarVector revalidate;
00172     scoped_ptr<OutputPartitions> partitions;
00173   };
00174 
00176   class CacheLookupResultCallback {
00177    public:
00178     CacheLookupResultCallback() {}
00179     virtual ~CacheLookupResultCallback();
00180     virtual void Done(const GoogleString& cache_key,
00181                       CacheLookupResult* result) = 0;
00182    private:
00183     DISALLOW_COPY_AND_ASSIGN(CacheLookupResultCallback);
00184   };
00185 
00188   RewriteContext(RewriteDriver* driver,   
00189                  RewriteContext* parent,  
00190                  ResourceContext* resource_context);
00191   virtual ~RewriteContext();
00192 
00197   int num_slots() const { return slots_.size(); }
00198   ResourceSlotPtr slot(int index) const { return slots_[index]; }
00199 
00202   int num_outputs() const { return outputs_.size(); }
00203   OutputResourcePtr output(int i) const { return outputs_[i]; }
00204 
00207   int num_output_partitions() const;
00208   const CachedResult* output_partition(int i) const;
00209   CachedResult* output_partition(int i);
00210 
00213   bool chained() const { return chained_; }
00214 
00218   void AddSlot(const ResourceSlotPtr& slot);
00219 
00222   void RemoveLastSlot();
00223 
00229   void AddNestedContext(RewriteContext* context);
00230 
00241   void Initiate();
00242 
00248   bool Fetch(const OutputResourcePtr& output_resource,
00249              AsyncFetch* fetch,
00250              MessageHandler* message_handler);
00251 
00254   bool slow() const { return slow_; }
00255 
00257   bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }
00258 
00260   bool has_parent() const { return parent_ != NULL; }
00261 
00264   bool IsNestedIn(StringPiece id) const;
00265 
00267   RewriteContext* parent() { return parent_; }
00268   const RewriteContext* parent() const { return parent_; }
00269 
00271   void set_force_rewrite(bool x) { force_rewrite_ = x; }
00272 
00273   bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
00274   void set_rewrite_uncacheable(bool rewrite_uncacheable) {
00275     rewrite_uncacheable_ = rewrite_uncacheable;
00276   }
00277 
00278   const ResourceContext* resource_context() const {
00279     return resource_context_.get();
00280   }
00281 
00283   GoogleString ToString(StringPiece prefix) const;
00284 
00286   static void InitStats(Statistics* stats);
00287 
00288  protected:
00289   typedef std::vector<GoogleUrl*> GoogleUrlStarVector;
00290 
00295 
00299   ServerContext* FindServerContext() const;
00300   const RewriteOptions* Options() const;
00301   RewriteDriver* Driver() const {
00302     return driver_;
00303   }
00304 
00306   int num_nested() const { return nested_.size(); }
00307   RewriteContext* nested(int i) const { return nested_[i]; }
00308 
00309   OutputPartitions* partitions() { return partitions_.get(); }
00310 
00313   void AddRecheckDependency();
00314 
00318   virtual bool OptimizationOnly() const { return true; }
00319 
00337   virtual bool Partition(OutputPartitions* partitions,
00338                          OutputResourceVector* outputs);
00339 
00345   virtual void PartitionAsync(OutputPartitions* partitions,
00346                               OutputResourceVector* outputs);
00347 
00351   void PartitionDone(RewriteResult result);
00352 
00355   void CrossThreadPartitionDone(RewriteResult result);
00356 
00375   virtual void Rewrite(int partition_index,
00376                        CachedResult* partition,
00377                        const OutputResourcePtr& output) = 0;
00378 
00383   void RewriteDone(RewriteResult result, int partition_index);
00384 
00391   virtual bool AbsolutifyIfNeeded(const StringPiece& output_url_base,
00392                                   const StringPiece& input_contents,
00393                                   Writer* writer, MessageHandler* handler);
00394 
00398   void StartNestedTasks();
00399 
00404   virtual void Harvest();
00405 
00417   virtual void Render();
00418 
00429   virtual void WillNotRender();
00430 
00434   virtual void Cancel();
00435 
00438 
00445   virtual const UrlSegmentEncoder* encoder() const;
00446 
00449   virtual GoogleString CacheKeySuffix() const;
00450 
00457   virtual GoogleString UserAgentCacheKey(
00458       const ResourceContext* context) const {
00459     return "";
00460   }
00461 
00466   virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}
00467 
00469   virtual const char* id() const = 0;
00470 
00479   virtual OutputResourceKind kind() const = 0;
00480 
00484 
00487   void AttachDependentRequestTrace(const StringPiece& label);
00488 
00492   RequestTrace* dependent_request_trace() { return dependent_request_trace_; }
00493 
00496   void TracePrintf(const char* fmt, ...);
00497 
00505 
00515   virtual void StartFetchReconstruction();
00516 
00522   bool ShouldDistributeRewrite() const;
00523 
00526   bool IsDistributedRewriteForHtml() const;
00527 
00531   void DistributeRewrite();
00532 
00538   void DetachFetch();
00539 
00544   virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
00545                                MessageHandler* message_handler,
00546                                GoogleUrlStarVector* url_vector);
00547 
00552   virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
00553                                        ResponseHeaders* headers);
00554 
00557   virtual void FetchCallbackDone(bool success);
00558 
00563   virtual void FetchTryFallback(const GoogleString& url,
00564                                 const StringPiece& hash);
00565 
00567   void Freshen();
00568 
00569   bool notify_driver_on_fetch_done() const {
00570     return notify_driver_on_fetch_done_;
00571   }
00572   void set_notify_driver_on_fetch_done(bool value) {
00573     notify_driver_on_fetch_done_ = value;
00574   }
00575 
00580   bool block_distribute_rewrite() const { return block_distribute_rewrite_; }
00581   void set_block_distribute_rewrite(const bool x) {
00582     block_distribute_rewrite_ = x;
00583   }
00584 
00586   AsyncFetch* async_fetch();
00587 
00589   bool FetchContextDetached();
00590 
00592   MessageHandler* fetch_message_handler();
00593 
00595   bool stale_rewrite() const { return stale_rewrite_; }
00596 
00600   virtual int64 GetRewriteDeadlineAlarmMs() const;
00601 
00603   virtual bool CreationLockBeforeStartFetch() { return true; }
00604 
00608   static bool LookupMetadataForOutputResourceImpl(
00609       OutputResourcePtr output_resource,
00610       const GoogleUrl& gurl,
00611       RewriteContext* rewrite_context,
00612       RewriteDriver* driver,
00613       GoogleString* error_out,
00614       CacheLookupResultCallback* callback);
00615 
00616  private:
00617   class DistributedRewriteCallback;
00618   class DistributedRewriteFetch;
00619   class OutputCacheCallback;
00620   class LookupMetadataForOutputResourceCallback;
00621   class HTTPCacheCallback;
00622   class ResourceCallbackUtils;
00623   class ResourceFetchCallback;
00624   class ResourceReconstructCallback;
00625   class ResourceRevalidateCallback;
00626   class InvokeRewriteFunction;
00627   class RewriteFreshenCallback;
00628   friend class RewriteDriver;
00629 
00630   typedef std::set<RewriteContext*> ContextSet;
00631 
00635   enum FallbackCondition {
00636     kFallbackDiscretional,   
00637 
00638     kFallbackEmergency    
00639 
00640   };
00641 
00643   void Start();
00644   void SetPartitionKey();
00645   void StartFetch();
00646   void StartFetchImpl();
00647   void CancelFetch();
00648   void OutputCacheDone(CacheLookupResult* cache_result);
00649   void OutputCacheHit(bool write_partitions);
00650   void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
00651   void OutputCacheMiss();
00652   void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
00653   void ResourceRevalidateDone(InputInfo* input_info, bool success);
00654   void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);
00655 
00660   void RepeatedSuccess(const RewriteContext* primary);
00661   void RepeatedFailure();
00662 
00671   void Finalize();
00672 
00674   NamedLock* Lock();
00675 
00685   void FetchInputs();
00686 
00691   void DistributeRewriteDone(bool success);
00692 
00696   bool ParseAndRemoveMetadataFromResponseHeaders(
00697       ResponseHeaders* response_headers, CacheLookupResult* cache_result);
00698 
00701   bool CreateOutputResourceFromContent(const CachedResult& cached_result,
00702                                        const ResponseHeaders& response_headers,
00703                                        StringPiece content,
00704                                        OutputResourcePtr* output_resource);
00705 
00715   GoogleString DistributedFetchUrl(StringPiece url);
00716 
00719   bool IsFetchRewrite() const { return fetch_.get() != NULL; }
00720 
00725   void NestedRewriteDone(const RewriteContext* context);
00726 
00730   void Activate();
00731 
00755   void Propagate(bool render_slots);
00756 
00763   void StartRewriteForHtml();
00764   void StartRewriteForFetch();
00765 
00772   bool ReadyToRewrite() const;
00773 
00778   void DetachSlots();
00779 
00782   void RunSuccessors();
00783 
00786   void WritePartition();
00787 
00797   void FinalizeRewriteForHtml();
00798 
00804   void RetireRewriteForHtml(bool permit_render);
00805 
00808   void MarkSlow();
00809 
00812   void MarkTooBusy();
00813 
00817   void CollectDependentTopLevel(ContextSet* contexts);
00818 
00821   void RewriteDoneImpl(RewriteResult result, int partition_index);
00822 
00825   void StartNestedTasksImpl();
00826 
00830   void RenderPartitionOnDetach(int partition_index);
00831 
00834   bool PrepareFetch(
00835       const OutputResourcePtr& output_resource,
00836       AsyncFetch* fetch,
00837       MessageHandler* message_handler);
00838 
00841   bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
00842                                            OutputResourcePtr* output_resource);
00843 
00845   void FetchCacheDone(CacheLookupResult* cache_result);
00846 
00849   void FetchFallbackCacheDone(HTTPCache::FindResult result,
00850                               HTTPCache::Callback* data);
00851 
00856   bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;
00857 
00860   bool HasDuplicateOtherDependency(const InputInfo& input);
00861 
00865   void CheckAndAddOtherDependency(const InputInfo& input);
00866 
00869   void CheckAndFreshenResource(const InputInfo& input_info,
00870                                ResourcePtr resource, int partition_index,
00871                                int input_index,
00872                                FreshenMetadataUpdateManager* freshen_manager);
00873   ResourcePtr CreateUrlResource(const StringPiece& input_url);
00874 
00876   ResourceSlotVector slots_;
00877 
00881   std::vector<bool> render_slots_;
00882 
00893 
00894   bool started_;
00895   scoped_ptr<OutputPartitions> partitions_;
00896   OutputResourceVector outputs_;
00897   int outstanding_fetches_;
00898   int outstanding_rewrites_;
00899   scoped_ptr<ResourceContext> resource_context_;
00900   GoogleString partition_key_;
00901 
00902   UrlSegmentEncoder default_encoder_;
00903 
00906   scoped_ptr<NamedLock> lock_;
00907 
00911   class FetchContext;
00912   scoped_ptr<FetchContext> fetch_;
00913 
00916   std::vector<RewriteContext*> successors_;
00917 
00921   std::vector<RewriteContext*> repeated_;
00922 
00926   int num_pending_nested_;
00927   std::vector<RewriteContext*> nested_;
00928 
00930   RewriteContext* parent_;
00931 
00940   RewriteDriver* driver_;
00941 
00943   int num_predecessors_;
00944 
00947   bool chained_;
00948 
00966 
00968   bool rewrite_done_;
00969 
00981   bool ok_to_write_output_partitions_;
00982 
00986   bool was_too_busy_;
00987 
00991   bool slow_;
00992 
00994   bool revalidate_ok_;
00995 
00998   bool notify_driver_on_fetch_done_;
00999 
01002   bool force_rewrite_;
01003 
01006   bool stale_rewrite_;
01007 
01010   bool is_metadata_cache_miss_;
01011 
01014   bool rewrite_uncacheable_;
01015 
01018   RequestTrace* dependent_request_trace_;
01019 
01022   bool block_distribute_rewrite_;
01023 
01025   scoped_ptr<DistributedRewriteFetch> distributed_fetch_;
01026 
01028   StringIntMap other_dependency_map_;
01029 
01030   Variable* const num_rewrites_abandoned_for_lock_contention_;
01031   Variable* const num_distributed_rewrite_failures_;
01032   Variable* const num_distributed_rewrite_successes_;
01033   Variable* const num_distributed_metadata_failures_;
01034   DISALLOW_COPY_AND_ASSIGN(RewriteContext);
01035 };
01036 
01037 }  
01038 
01039 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines