Page Speed Optimization Libraries  1.9.32.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
rewrite_context.h
Go to the documentation of this file.
1 /*
2  * Copyright 2011 Google Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http:///www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
18 
19 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
20 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
21 
22 #include <set>
23 #include <vector>
24 
26 #include "net/instaweb/rewriter/cached_result.pb.h"
36 #include "net/instaweb/util/public/url_segment_encoder.h"
37 
38 namespace net_instaweb {
39 
40 class AsyncFetch;
41 class GoogleUrl;
42 class MessageHandler;
43 class NamedLock;
44 class RequestTrace;
45 class ResponseHeaders;
46 class RewriteDriver;
47 class RewriteOptions;
48 class Statistics;
49 class Variable;
50 class FreshenMetadataUpdateManager;
51 
145  public:
146  typedef std::vector<InputInfo*> InputInfoStarVector;
147  static const char kNumRewritesAbandonedForLockContention[];
148  static const char kNumDeadlineAlarmInvocations[];
149  static const char kNumDistributedRewriteSuccesses[];
150  static const char kNumDistributedRewriteFailures[];
151  static const char kNumDistributedMetadataFailures[];
153  static const char kDistributedExt[];
155  static const char kDistributedHash[];
160  : cache_ok(false),
161  can_revalidate(false),
162  useable_cache_content(false),
163  is_stale_rewrite(false),
164  partitions(new OutputPartitions) {}
165 
166  bool cache_ok;
167  bool can_revalidate;
168  bool useable_cache_content;
169  bool is_stale_rewrite;
170  InputInfoStarVector revalidate;
171  scoped_ptr<OutputPartitions> partitions;
172  };
173 
176  public:
178  virtual ~CacheLookupResultCallback();
179  virtual void Done(const GoogleString& cache_key,
180  CacheLookupResult* result) = 0;
181  private:
182  DISALLOW_COPY_AND_ASSIGN(CacheLookupResultCallback);
183  };
184 
187  RewriteContext(RewriteDriver* driver,
189  ResourceContext* resource_context);
190  virtual ~RewriteContext();
191 
196  int num_slots() const { return slots_.size(); }
197  ResourceSlotPtr slot(int index) const { return slots_[index]; }
198 
201  int num_outputs() const { return outputs_.size(); }
202  OutputResourcePtr output(int i) const { return outputs_[i]; }
203 
206  int num_output_partitions() const;
207  const CachedResult* output_partition(int i) const;
208  CachedResult* output_partition(int i);
209 
212  bool chained() const { return chained_; }
213 
217  void AddSlot(const ResourceSlotPtr& slot);
218 
221  void RemoveLastSlot();
222 
228  void AddNestedContext(RewriteContext* context);
229 
240  void Initiate();
241 
247  bool Fetch(const OutputResourcePtr& output_resource,
248  AsyncFetch* fetch,
249  MessageHandler* message_handler);
250 
253  bool slow() const { return slow_; }
254 
256  bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }
257 
259  bool has_parent() const { return parent_ != NULL; }
260 
263  bool IsNestedIn(StringPiece id) const;
264 
266  RewriteContext* parent() { return parent_; }
267  const RewriteContext* parent() const { return parent_; }
268 
270  void set_force_rewrite(bool x) { force_rewrite_ = x; }
271 
272  bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
273  void set_rewrite_uncacheable(bool rewrite_uncacheable) {
274  rewrite_uncacheable_ = rewrite_uncacheable;
275  }
276 
277  const ResourceContext* resource_context() const {
278  return resource_context_.get();
279  }
280 
282  GoogleString ToString(StringPiece prefix) const;
283 
285  static void InitStats(Statistics* stats);
286 
287  protected:
288  typedef std::vector<GoogleUrl*> GoogleUrlStarVector;
289 
294 
298  ServerContext* FindServerContext() const;
299  const RewriteOptions* Options() const;
300  RewriteDriver* Driver() const {
301  return driver_;
302  }
303 
305  int num_nested() const { return nested_.size(); }
306  RewriteContext* nested(int i) const { return nested_[i]; }
307 
308  OutputPartitions* partitions() { return partitions_.get(); }
309 
312  void AddRecheckDependency();
313 
317  virtual bool OptimizationOnly() const { return true; }
318 
336  virtual bool Partition(OutputPartitions* partitions,
337  OutputResourceVector* outputs);
338 
344  virtual void PartitionAsync(OutputPartitions* partitions,
345  OutputResourceVector* outputs);
346 
350  void PartitionDone(RewriteResult result);
351 
355 
374  virtual void Rewrite(int partition_index,
375  CachedResult* partition,
376  const OutputResourcePtr& output) = 0;
377 
382  void RewriteDone(RewriteResult result, int partition_index);
383 
393  virtual bool SendFallbackResponse(StringPiece output_url_base,
394  StringPiece contents,
396  MessageHandler* handler);
397 
401  void StartNestedTasks();
402 
407  virtual void Harvest();
408 
420  virtual void Render();
421 
432  virtual void WillNotRender();
433 
437  virtual void Cancel();
438 
441 
448  virtual const UrlSegmentEncoder* encoder() const;
449 
452  virtual GoogleString CacheKeySuffix() const;
453 
460  virtual GoogleString UserAgentCacheKey(
461  const ResourceContext* context) const {
462  return "";
463  }
464 
469  virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}
470 
472  virtual const char* id() const = 0;
473 
482  virtual OutputResourceKind kind() const = 0;
483 
487 
490  void AttachDependentRequestTrace(const StringPiece& label);
491 
495  RequestTrace* dependent_request_trace() { return dependent_request_trace_; }
496 
499  void TracePrintf(const char* fmt, ...);
500 
508 
518  virtual void StartFetchReconstruction();
519 
525  bool ShouldDistributeRewrite() const;
526 
529  bool IsDistributedRewriteForHtml() const;
530 
534  void DistributeRewrite();
535 
541  void DetachFetch();
542 
547  virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
548  MessageHandler* message_handler,
549  GoogleUrlStarVector* url_vector);
550 
555  virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
556  ResponseHeaders* headers);
557 
560  virtual void FetchCallbackDone(bool success);
561 
566  virtual void FetchTryFallback(const GoogleString& url,
567  const StringPiece& hash);
568 
570  void Freshen();
571 
572  bool notify_driver_on_fetch_done() const {
573  return notify_driver_on_fetch_done_;
574  }
575  void set_notify_driver_on_fetch_done(bool value) {
576  notify_driver_on_fetch_done_ = value;
577  }
578 
583  bool block_distribute_rewrite() const { return block_distribute_rewrite_; }
584  void set_block_distribute_rewrite(const bool x) {
585  block_distribute_rewrite_ = x;
586  }
587 
589  AsyncFetch* async_fetch();
590 
592  bool FetchContextDetached();
593 
595  MessageHandler* fetch_message_handler();
596 
598  bool stale_rewrite() const { return stale_rewrite_; }
599 
603  virtual int64 GetRewriteDeadlineAlarmMs() const;
604 
606  virtual bool CreationLockBeforeStartFetch() { return true; }
607 
612  OutputResourcePtr output_resource,
613  const GoogleUrl& gurl,
614  RewriteContext* rewrite_context,
615  RewriteDriver* driver,
616  GoogleString* error_out,
617  CacheLookupResultCallback* callback);
618 
619  private:
620  class DistributedRewriteCallback;
621  class DistributedRewriteFetch;
622  class OutputCacheCallback;
623  class LookupMetadataForOutputResourceCallback;
624  class HTTPCacheCallback;
625  class ResourceCallbackUtils;
626  class ResourceFetchCallback;
627  class ResourceReconstructCallback;
628  class ResourceRevalidateCallback;
629  class InvokeRewriteFunction;
630  class RewriteFreshenCallback;
631  friend class RewriteDriver;
632 
633  typedef std::set<RewriteContext*> ContextSet;
634 
638  enum FallbackCondition {
639  kFallbackDiscretional,
640  kFallbackEmergency
642  };
644 
646  void Start();
647  void SetPartitionKey();
648  void StartFetch();
649  void StartFetchImpl();
650  void CancelFetch();
651  void OutputCacheDone(CacheLookupResult* cache_result);
652  void OutputCacheHit(bool write_partitions);
653  void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
654  void OutputCacheMiss();
655  void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
656  void ResourceRevalidateDone(InputInfo* input_info, bool success);
657  void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);
658 
663  void RepeatedSuccess(const RewriteContext* primary);
664  void RepeatedFailure();
665 
674  void Finalize();
675 
677  NamedLock* Lock();
678 
688  void FetchInputs();
689 
694  void DistributeRewriteDone(bool success);
695 
699  bool ParseAndRemoveMetadataFromResponseHeaders(
700  ResponseHeaders* response_headers, CacheLookupResult* cache_result);
701 
704  bool CreateOutputResourceFromContent(const CachedResult& cached_result,
705  const ResponseHeaders& response_headers,
706  StringPiece content,
707  OutputResourcePtr* output_resource);
708 
718  GoogleString DistributedFetchUrl(StringPiece url);
719 
722  bool IsFetchRewrite() const { return fetch_.get() != NULL; }
723 
728  void NestedRewriteDone(const RewriteContext* context);
729 
733  void Activate();
734 
758  void Propagate(bool render_slots);
759 
766  void StartRewriteForHtml();
767  void StartRewriteForFetch();
768 
775  bool ReadyToRewrite() const;
776 
781  void DetachSlots();
782 
785  void RunSuccessors();
786 
789  void WritePartition();
790 
800  void FinalizeRewriteForHtml();
801 
807  void RetireRewriteForHtml(bool permit_render);
808 
811  void MarkSlow();
812 
815  void MarkTooBusy();
816 
820  void CollectDependentTopLevel(ContextSet* contexts);
821 
824  void RewriteDoneImpl(RewriteResult result, int partition_index);
825 
828  void StartNestedTasksImpl();
829 
833  void RenderPartitionOnDetach(int partition_index);
834 
837  bool PrepareFetch(
838  const OutputResourcePtr& output_resource,
839  AsyncFetch* fetch,
840  MessageHandler* message_handler);
841 
844  bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
845  OutputResourcePtr* output_resource);
846 
848  void FetchCacheDone(CacheLookupResult* cache_result);
849 
852  void FetchFallbackCacheDone(HTTPCache::FindResult result,
853  HTTPCache::Callback* data);
854 
859  bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;
860 
863  bool HasDuplicateOtherDependency(const InputInfo& input);
864 
868  void CheckAndAddOtherDependency(const InputInfo& input);
869 
872  void CheckAndFreshenResource(const InputInfo& input_info,
873  ResourcePtr resource, int partition_index,
874  int input_index,
875  FreshenMetadataUpdateManager* freshen_manager);
876  ResourcePtr CreateUrlResource(const StringPiece& input_url);
877 
879  ResourceSlotVector slots_;
880 
884  std::vector<bool> render_slots_;
885 
896 
897  bool started_;
898  scoped_ptr<OutputPartitions> partitions_;
899  OutputResourceVector outputs_;
900  int outstanding_fetches_;
901  int outstanding_rewrites_;
902  scoped_ptr<ResourceContext> resource_context_;
903  GoogleString partition_key_;
904 
905  UrlSegmentEncoder default_encoder_;
906 
909  scoped_ptr<NamedLock> lock_;
910 
914  class FetchContext;
915  scoped_ptr<FetchContext> fetch_;
916 
919  std::vector<RewriteContext*> successors_;
920 
924  std::vector<RewriteContext*> repeated_;
925 
929  int num_pending_nested_;
930  std::vector<RewriteContext*> nested_;
931 
933  RewriteContext* parent_;
934 
943  RewriteDriver* driver_;
944 
946  int num_predecessors_;
947 
950  bool chained_;
951 
969 
971  bool rewrite_done_;
972 
984  bool ok_to_write_output_partitions_;
985 
989  bool was_too_busy_;
990 
994  bool slow_;
995 
997  bool revalidate_ok_;
998 
1001  bool notify_driver_on_fetch_done_;
1002 
1005  bool force_rewrite_;
1006 
1009  bool stale_rewrite_;
1010 
1013  bool is_metadata_cache_miss_;
1014 
1017  bool rewrite_uncacheable_;
1018 
1021  RequestTrace* dependent_request_trace_;
1022 
1025  bool block_distribute_rewrite_;
1026 
1028  scoped_ptr<DistributedRewriteFetch> distributed_fetch_;
1029 
1031  StringIntMap other_dependency_map_;
1032 
1033  Variable* const num_rewrites_abandoned_for_lock_contention_;
1034  Variable* const num_distributed_rewrite_failures_;
1035  Variable* const num_distributed_rewrite_successes_;
1036  Variable* const num_distributed_metadata_failures_;
1037  DISALLOW_COPY_AND_ASSIGN(RewriteContext);
1038 };
1039 
1040 }
1041 
1042 #endif
RewriteContext * parent()
Allows a nested rewriter to walk up its parent hierarchy.
Definition: rewrite_context.h:266
static const char kDistributedExt[]
The extension used for all distributed fetch URLs.
Definition: rewrite_context.h:153
virtual bool Partition(OutputPartitions *partitions, OutputResourceVector *outputs)
bool stale_rewrite() const
Indicates whether we are serving a stale rewrite.
Definition: rewrite_context.h:598
bool IsDistributedRewriteForHtml() const
bool block_distribute_rewrite() const
Definition: rewrite_context.h:583
int num_outputs() const
Definition: rewrite_context.h:201
static bool LookupMetadataForOutputResourceImpl(OutputResourcePtr output_resource, const GoogleUrl &gurl, RewriteContext *rewrite_context, RewriteDriver *driver, GoogleString *error_out, CacheLookupResultCallback *callback)
void AddNestedContext(RewriteContext *context)
virtual OutputResourceKind kind() const =0
static void InitStats(Statistics *stats)
Initializes statistics.
virtual const UrlSegmentEncoder * encoder() const
FindResult
When a lookup is done in the HTTP Cache, it returns one of these values.
Definition: http_cache.h:71
Definition: rewrite_context.h:158
virtual GoogleString UserAgentCacheKey(const ResourceContext *context) const
Definition: rewrite_context.h:460
void Freshen()
Freshens resources proactively to avoid expiration in the near future.
int num_output_partitions() const
void TracePrintf(const char *fmt,...)
virtual bool DecodeFetchUrls(const OutputResourcePtr &output_resource, MessageHandler *message_handler, GoogleUrlStarVector *url_vector)
bool ShouldDistributeRewrite() const
bool IsNestedIn(StringPiece id) const
void RewriteDone(RewriteResult result, int partition_index)
GoogleString ToString(StringPiece prefix) const
Returns debug information about this RewriteContext.
void PartitionDone(RewriteResult result)
bool Fetch(const OutputResourcePtr &output_resource, AsyncFetch *fetch, MessageHandler *message_handler)
ServerContext * FindServerContext() const
void CrossThreadPartitionDone(RewriteResult result)
Used for LookupMetadataForOutputResource.
Definition: rewrite_context.h:175
virtual const char * id() const =0
Returns the filter ID.
virtual bool OptimizationOnly() const
Definition: rewrite_context.h:317
Definition: async_fetch.h:53
virtual void StartFetchReconstruction()
int num_nested() const
Accessors for the nested rewrites.
Definition: rewrite_context.h:305
virtual void FixFetchFallbackHeaders(const CachedResult &cached_result, ResponseHeaders *headers)
RequestTrace * dependent_request_trace()
Definition: rewrite_context.h:495
Definition: rewrite_driver.h:98
virtual bool SendFallbackResponse(StringPiece output_url_base, StringPiece contents, AsyncFetch *async_fetch, MessageHandler *handler)
RewriteResult
Definition: rewrite_result.h:27
virtual GoogleString CacheKeySuffix() const
bool chained() const
Definition: rewrite_context.h:212
Definition: rewrite_context.h:144
int num_slots() const
Definition: rewrite_context.h:196
virtual void Rewrite(int partition_index, CachedResult *partition, const OutputResourcePtr &output)=0
virtual void FetchCallbackDone(bool success)
virtual void EncodeUserAgentIntoResourceContext(ResourceContext *context)
Definition: rewrite_context.h:469
static const char kDistributedHash[]
The hash value used for all distributed fetch URLs.
Definition: rewrite_context.h:155
virtual int64 GetRewriteDeadlineAlarmMs() const
bool slow() const
Definition: rewrite_context.h:253
bool is_metadata_cache_miss() const
This particular rewrite was a metadata cache miss.
Definition: rewrite_context.h:256
MessageHandler * fetch_message_handler()
The message handler for the fetch.
virtual void PartitionAsync(OutputPartitions *partitions, OutputResourceVector *outputs)
bool FetchContextDetached()
Is fetch_ detached? Only call this in the fetch flow.
RewriteContext(RewriteDriver *driver, RewriteContext *parent, ResourceContext *resource_context)
void AddSlot(const ResourceSlotPtr &slot)
virtual void FetchTryFallback(const GoogleString &url, const StringPiece &hash)
bool has_parent() const
Returns true if this is a nested rewriter.
Definition: rewrite_context.h:259
void set_force_rewrite(bool x)
If called with true, forces a rewrite and re-generates the output.
Definition: rewrite_context.h:270
void AttachDependentRequestTrace(const StringPiece &label)
virtual bool CreationLockBeforeStartFetch()
Should the context call LockForCreation before checking the cache?
Definition: rewrite_context.h:606
OutputResourceKind
Definition: output_resource_kind.h:26
AsyncFetch * async_fetch()
Note that the following must only be called in the fetch flow.