Page Speed Optimization Libraries  1.7.30.4
net/instaweb/rewriter/public/rewrite_context.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2011 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00019 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00020 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
00021 
00022 #include <set>
00023 #include <vector>
00024 
00025 #include "net/instaweb/http/public/http_cache.h"
00026 #include "net/instaweb/rewriter/cached_result.pb.h"
00027 #include "net/instaweb/rewriter/public/output_resource_kind.h"
00028 #include "net/instaweb/rewriter/public/resource.h"
00029 #include "net/instaweb/rewriter/public/server_context.h"
00030 #include "net/instaweb/rewriter/public/resource_slot.h"
00031 #include "net/instaweb/rewriter/public/rewrite_context.h"
00032 #include "net/instaweb/rewriter/public/rewrite_result.h"
00033 #include "net/instaweb/util/public/basictypes.h"
00034 #include "net/instaweb/util/public/scoped_ptr.h"
00035 #include "net/instaweb/util/public/string.h"
00036 #include "net/instaweb/util/public/string_util.h"
00037 #include "net/instaweb/util/public/url_segment_encoder.h"
00038 
00039 namespace net_instaweb {
00040 
00041 class AsyncFetch;
00042 class GoogleUrl;
00043 class MessageHandler;
00044 class NamedLock;
00045 class RequestTrace;
00046 class ResponseHeaders;
00047 class RewriteDriver;
00048 class RewriteOptions;
00049 class Statistics;
00050 class Variable;
00051 class Writer;
00052 class FreshenMetadataUpdateManager;
00053 
00146 class RewriteContext {
00147  public:
00148   typedef std::vector<InputInfo*> InputInfoStarVector;
00149   static const char kNumRewritesAbandonedForLockContention[];
00150   static const char kNumDeadlineAlarmInvocations[];
00151   static const char kNumDistributedRewriteSuccesses[];
00152   static const char kNumDistributedRewriteFailures[];
00153   static const char kNumDistributedMetadataFailures[];
00155   static const char kDistributedExt[];
00157   static const char kDistributedHash[];
00160   struct CacheLookupResult {
00161     CacheLookupResult()
00162         : cache_ok(false),
00163           can_revalidate(false),
00164           useable_cache_content(false),
00165           is_stale_rewrite(false),
00166           partitions(new OutputPartitions) {}
00167 
00168     bool cache_ok;
00169     bool can_revalidate;
00170     bool useable_cache_content;
00171     bool is_stale_rewrite;
00172     InputInfoStarVector revalidate;
00173     scoped_ptr<OutputPartitions> partitions;
00174   };
00175 
00177   class CacheLookupResultCallback {
00178    public:
00179     CacheLookupResultCallback() {}
00180     virtual ~CacheLookupResultCallback();
00181     virtual void Done(const GoogleString& cache_key,
00182                       CacheLookupResult* result) = 0;
00183    private:
00184     DISALLOW_COPY_AND_ASSIGN(CacheLookupResultCallback);
00185   };
00186 
00189   RewriteContext(RewriteDriver* driver,   
00190                  RewriteContext* parent,  
00191                  ResourceContext* resource_context);
00192   virtual ~RewriteContext();
00193 
00198   int num_slots() const { return slots_.size(); }
00199   ResourceSlotPtr slot(int index) const { return slots_[index]; }
00200 
00203   int num_outputs() const { return outputs_.size(); }
00204   OutputResourcePtr output(int i) const { return outputs_[i]; }
00205 
00208   int num_output_partitions() const;
00209   const CachedResult* output_partition(int i) const;
00210   CachedResult* output_partition(int i);
00211 
00214   bool chained() const { return chained_; }
00215 
00219   void AddSlot(const ResourceSlotPtr& slot);
00220 
00223   void RemoveLastSlot();
00224 
00230   void AddNestedContext(RewriteContext* context);
00231 
00242   void Initiate();
00243 
00249   bool Fetch(const OutputResourcePtr& output_resource,
00250              AsyncFetch* fetch,
00251              MessageHandler* message_handler);
00252 
00266   static bool LookupMetadataForOutputResource(
00267       const GoogleString& url,
00268       RewriteDriver* driver,
00269       GoogleString* error_out,
00270       CacheLookupResultCallback* callback);
00271 
00274   bool slow() const { return slow_; }
00275 
00277   bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }
00278 
00280   bool has_parent() const { return parent_ != NULL; }
00281 
00284   bool IsNestedIn(StringPiece id) const;
00285 
00287   RewriteContext* parent() { return parent_; }
00288   const RewriteContext* parent() const { return parent_; }
00289 
00291   void set_force_rewrite(bool x) { force_rewrite_ = x; }
00292 
00293   bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
00294   void set_rewrite_uncacheable(bool rewrite_uncacheable) {
00295     rewrite_uncacheable_ = rewrite_uncacheable;
00296   }
00297 
00298   const ResourceContext* resource_context() const {
00299     return resource_context_.get();
00300   }
00301 
00303   GoogleString ToString(StringPiece prefix) const;
00304 
00306   static void InitStats(Statistics* stats);
00307 
00308  protected:
00309   typedef std::vector<GoogleUrl*> GoogleUrlStarVector;
00310 
00315 
00319   ServerContext* FindServerContext() const;
00320   const RewriteOptions* Options() const;
00321   RewriteDriver* Driver() const;
00322 
00324   int num_nested() const { return nested_.size(); }
00325   RewriteContext* nested(int i) const { return nested_[i]; }
00326 
00327   OutputPartitions* partitions() { return partitions_.get(); }
00328 
00331   void AddRecheckDependency();
00332 
00336   virtual bool OptimizationOnly() const { return true; }
00337 
00355   virtual bool Partition(OutputPartitions* partitions,
00356                          OutputResourceVector* outputs);
00357 
00363   virtual void PartitionAsync(OutputPartitions* partitions,
00364                               OutputResourceVector* outputs);
00365 
00369   void PartitionDone(bool result);
00370 
00373   void CrossThreadPartitionDone(bool result);
00374 
00393   virtual void Rewrite(int partition_index,
00394                        CachedResult* partition,
00395                        const OutputResourcePtr& output) = 0;
00396 
00401   void RewriteDone(RewriteResult result, int partition_index);
00402 
00409   virtual bool AbsolutifyIfNeeded(const StringPiece& output_url_base,
00410                                   const StringPiece& input_contents,
00411                                   Writer* writer, MessageHandler* handler);
00412 
00416   void StartNestedTasks();
00417 
00422   virtual void Harvest();
00423 
00435   virtual void Render();
00436 
00447   virtual void WillNotRender();
00448 
00452   virtual void Cancel();
00453 
00456 
00463   virtual const UrlSegmentEncoder* encoder() const;
00464 
00467   virtual GoogleString CacheKeySuffix() const;
00468 
00475   virtual GoogleString UserAgentCacheKey(
00476       const ResourceContext* context) const {
00477     return "";
00478   }
00479 
00484   virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}
00485 
00487   virtual const char* id() const = 0;
00488 
00497   virtual OutputResourceKind kind() const = 0;
00498 
00502 
00505   void AttachDependentRequestTrace(const StringPiece& label);
00506 
00510   RequestTrace* dependent_request_trace() { return dependent_request_trace_; }
00511 
00514   void TracePrintf(const char* fmt, ...);
00515 
00523 
00533   virtual void StartFetchReconstruction();
00534 
00540   bool ShouldDistributeRewrite() const;
00541 
00544   bool IsDistributedRewriteForHtml() const;
00545 
00549   void DistributeRewrite();
00550 
00556   void DetachFetch();
00557 
00562   virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
00563                                MessageHandler* message_handler,
00564                                GoogleUrlStarVector* url_vector);
00565 
00570   virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
00571                                        ResponseHeaders* headers);
00572 
00575   virtual void FetchCallbackDone(bool success);
00576 
00581   virtual void FetchTryFallback(const GoogleString& url,
00582                                 const StringPiece& hash);
00583 
00585   void Freshen();
00586 
00587   bool notify_driver_on_fetch_done() const {
00588     return notify_driver_on_fetch_done_;
00589   }
00590   void set_notify_driver_on_fetch_done(bool value) {
00591     notify_driver_on_fetch_done_ = value;
00592   }
00593 
00598   bool block_distribute_rewrite() const { return block_distribute_rewrite_; }
00599   void set_block_distribute_rewrite(const bool x) {
00600     block_distribute_rewrite_ = x;
00601   }
00602 
00604   AsyncFetch* async_fetch();
00605 
00607   bool FetchContextDetached();
00608 
00610   MessageHandler* fetch_message_handler();
00611 
00613   bool stale_rewrite() const { return stale_rewrite_; }
00614 
00618   virtual int64 GetRewriteDeadlineAlarmMs() const;
00619 
00621   virtual bool CreationLockBeforeStartFetch() { return true; }
00622 
00623  private:
00624   class DistributedRewriteCallback;
00625   class DistributedRewriteFetch;
00626   class OutputCacheCallback;
00627   class LookupMetadataForOutputResourceCallback;
00628   class HTTPCacheCallback;
00629   class ResourceCallbackUtils;
00630   class ResourceFetchCallback;
00631   class ResourceReconstructCallback;
00632   class ResourceRevalidateCallback;
00633   class InvokeRewriteFunction;
00634   class RewriteFreshenCallback;
00635   friend class RewriteDriver;
00636 
00637   typedef std::set<RewriteContext*> ContextSet;
00638 
00642   enum FallbackCondition {
00643     kFallbackDiscretional,   
00644 
00645     kFallbackEmergency    
00646 
00647   };
00648 
00650   void Start();
00651   void SetPartitionKey();
00652   void StartFetch();
00653   void StartFetchImpl();
00654   void CancelFetch();
00655   void OutputCacheDone(CacheLookupResult* cache_result);
00656   void OutputCacheHit(bool write_partitions);
00657   void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
00658   void OutputCacheMiss();
00659   void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
00660   void ResourceRevalidateDone(InputInfo* input_info, bool success);
00661   void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);
00662 
00667   void RepeatedSuccess(const RewriteContext* primary);
00668   void RepeatedFailure();
00669 
00678   void Finalize();
00679 
00681   NamedLock* Lock();
00682 
00692   void FetchInputs();
00693 
00698   void DistributeRewriteDone(bool success);
00699 
00703   bool ParseAndRemoveMetadataFromResponseHeaders(
00704       ResponseHeaders* response_headers, CacheLookupResult* cache_result);
00705 
00708   bool CreateOutputResourceFromContent(const CachedResult& cached_result,
00709                                        const ResponseHeaders& response_headers,
00710                                        StringPiece content,
00711                                        OutputResourcePtr* output_resource);
00712 
00722   GoogleString DistributedFetchUrl(StringPiece url);
00723 
00726   bool IsFetchRewrite() const { return fetch_.get() != NULL; }
00727 
00732   void NestedRewriteDone(const RewriteContext* context);
00733 
00737   void Activate();
00738 
00762   void Propagate(bool render_slots);
00763 
00770   void StartRewriteForHtml();
00771   void StartRewriteForFetch();
00772 
00779   bool ReadyToRewrite() const;
00780 
00785   void DetachSlots();
00786 
00789   void RunSuccessors();
00790 
00793   void WritePartition();
00794 
00804   void FinalizeRewriteForHtml();
00805 
00811   void RetireRewriteForHtml(bool permit_render);
00812 
00815   void MarkSlow();
00816 
00819   void MarkTooBusy();
00820 
00824   void CollectDependentTopLevel(ContextSet* contexts);
00825 
00828   void RewriteDoneImpl(RewriteResult result, int partition_index);
00829 
00832   void StartNestedTasksImpl();
00833 
00837   void RenderPartitionOnDetach(int partition_index);
00838 
00841   bool PrepareFetch(
00842       const OutputResourcePtr& output_resource,
00843       AsyncFetch* fetch,
00844       MessageHandler* message_handler);
00845 
00848   bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
00849                                            OutputResourcePtr* output_resource);
00850 
00852   void FetchCacheDone(CacheLookupResult* cache_result);
00853 
00856   void FetchFallbackCacheDone(HTTPCache::FindResult result,
00857                               HTTPCache::Callback* data);
00858 
00863   bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;
00864 
00867   bool HasDuplicateOtherDependency(const InputInfo& input);
00868 
00872   void CheckAndAddOtherDependency(const InputInfo& input);
00873 
00876   void CheckAndFreshenResource(const InputInfo& input_info,
00877                                ResourcePtr resource, int partition_index,
00878                                int input_index,
00879                                FreshenMetadataUpdateManager* freshen_manager);
00880   ResourcePtr CreateUrlResource(const StringPiece& input_url);
00881 
00883   ResourceSlotVector slots_;
00884 
00888   std::vector<bool> render_slots_;
00889 
00900 
00901   bool started_;
00902   scoped_ptr<OutputPartitions> partitions_;
00903   OutputResourceVector outputs_;
00904   int outstanding_fetches_;
00905   int outstanding_rewrites_;
00906   scoped_ptr<ResourceContext> resource_context_;
00907   GoogleString partition_key_;
00908 
00909   UrlSegmentEncoder default_encoder_;
00910 
00913   scoped_ptr<NamedLock> lock_;
00914 
00918   class FetchContext;
00919   scoped_ptr<FetchContext> fetch_;
00920 
00923   std::vector<RewriteContext*> successors_;
00924 
00928   std::vector<RewriteContext*> repeated_;
00929 
00933   int num_pending_nested_;
00934   std::vector<RewriteContext*> nested_;
00935 
00937   RewriteContext* parent_;
00938 
00948   RewriteDriver* driver_;
00949 
00951   int num_predecessors_;
00952 
00955   bool chained_;
00956 
00974 
00976   bool rewrite_done_;
00977 
00989   bool ok_to_write_output_partitions_;
00990 
00994   bool was_too_busy_;
00995 
00999   bool slow_;
01000 
01002   bool revalidate_ok_;
01003 
01006   bool notify_driver_on_fetch_done_;
01007 
01010   bool force_rewrite_;
01011 
01014   bool stale_rewrite_;
01015 
01018   bool is_metadata_cache_miss_;
01019 
01022   bool rewrite_uncacheable_;
01023 
01026   RequestTrace* dependent_request_trace_;
01027 
01030   bool block_distribute_rewrite_;
01031 
01033   scoped_ptr<DistributedRewriteFetch> distributed_fetch_;
01034 
01036   StringIntMap other_dependency_map_;
01037 
01038   Variable* const num_rewrites_abandoned_for_lock_contention_;
01039   Variable* const num_distributed_rewrite_failures_;
01040   Variable* const num_distributed_rewrite_successes_;
01041   Variable* const num_distributed_metadata_failures_;
01042   DISALLOW_COPY_AND_ASSIGN(RewriteContext);
01043 };
01044 
01045 }  
01046 
01047 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines