rewrite_context.h

Go to the documentation of this file.

/*
 * Copyright 2011 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_
#define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_CONTEXT_H_

#include <set>
#include <vector>

#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/rewriter/cached_result.pb.h"
#include "net/instaweb/rewriter/input_info.pb.h"
#include "net/instaweb/rewriter/public/csp_directive.h"
#include "net/instaweb/rewriter/public/output_resource_kind.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/rewriter/public/resource_slot.h"
#include "net/instaweb/rewriter/public/rewrite_result.h"
#include "net/instaweb/rewriter/public/server_context.h"
#include "pagespeed/controller/schedule_rewrite_callback.h"
#include "pagespeed/kernel/base/atomic_bool.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/function.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/http/google_url.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/util/url_segment_encoder.h"

namespace net_instaweb {

class AsyncFetch;
class MessageHandler;
class NamedLock;
class RequestTrace;
class RewriteDriver;
class RewriteOptions;
class Statistics;
class Variable;
class FreshenMetadataUpdateManager;

enum class RenderOp {
  kDontRender,
  kRenderOnlyCspWarning,
  kRender
};

class RewriteContext {
 public:
  typedef std::vector<InputInfo*> InputInfoStarVector;
  static const char kNumRewritesAbandonedForLockContention[];
  static const char kNumDeadlineAlarmInvocations[];
  static const char kHashMismatchMessage[];

  struct CacheLookupResult {
    CacheLookupResult()
        : cache_ok(false),
          can_revalidate(false),
          useable_cache_content(false),
          is_stale_rewrite(false),
          partitions(new OutputPartitions) {}

    bool cache_ok;
    bool can_revalidate;
    bool useable_cache_content;
    bool is_stale_rewrite;
    InputInfoStarVector revalidate;
    scoped_ptr<OutputPartitions> partitions;
  };

  class CacheLookupResultCallback {
   public:
    CacheLookupResultCallback() {}
    virtual ~CacheLookupResultCallback();
    virtual void Done(const GoogleString& cache_key,
                      CacheLookupResult* result) = 0;
   private:
    
  };

  RewriteContext(RewriteDriver* driver,   
                 RewriteContext* parent,  
                 ResourceContext* resource_context);
  virtual ~RewriteContext();

  int num_slots() const { return slots_.size(); }
  ResourceSlotPtr slot(int index) const { return slots_[index]; }

  int num_outputs() const { return outputs_.size(); }
  OutputResourcePtr output(int i) const { return outputs_[i]; }

  int num_output_partitions() const;
  const CachedResult* output_partition(int i) const;
  CachedResult* mutable_output_partition(int i);

  bool chained() const { return chained_; }

  void AddSlot(const ResourceSlotPtr& slot);

  void RemoveLastSlot();

  void AddNestedContext(RewriteContext* context);

  void CallFetchInputs();
  void CallLockFailed();
  void CallStartFetchImpl();

  void Initiate();

  bool Fetch(const OutputResourcePtr& output_resource,
             AsyncFetch* fetch,
             MessageHandler* message_handler);

  bool slow() const { return slow_; }

  bool is_metadata_cache_miss() const { return is_metadata_cache_miss_; }

  bool has_parent() const { return parent_ != NULL; }

  bool IsNestedIn(StringPiece id) const;

  void CheckNotFrozen();

  RewriteContext* parent() { return parent_; }
  const RewriteContext* parent() const { return parent_; }

  int num_nested() const { return nested_.size(); }
  RewriteContext* nested(int i) const { return nested_[i]; }

  RewriteDriver* Driver() const {
    return driver_;
  }

  void set_force_rewrite(bool x) { force_rewrite_ = x; }

  bool rewrite_uncacheable() const { return rewrite_uncacheable_; }
  void set_rewrite_uncacheable(bool rewrite_uncacheable) {
    rewrite_uncacheable_ = rewrite_uncacheable;
  }

  const ResourceContext* resource_context() const {
    return resource_context_.get();
  }

  GoogleString ToString() const;
  GoogleString ToStringWithPrefix(StringPiece prefix) const;

  static void InitStats(Statistics* stats);

 protected:
  typedef std::vector<GoogleUrl*> GoogleUrlStarVector;


  ServerContext* FindServerContext() const;
  const RewriteOptions* Options() const;

  OutputPartitions* partitions() { return partitions_.get(); }

  void AddRecheckDependency();

  virtual bool OptimizationOnly() const { return true; }

  virtual bool Partition(OutputPartitions* partitions,
                         OutputResourceVector* outputs);

  virtual void PartitionAsync(OutputPartitions* partitions,
                              OutputResourceVector* outputs);

  void PartitionDone(RewriteResult result);

  void CrossThreadPartitionDone(RewriteResult result);

  virtual void Rewrite(int partition_index,
                       CachedResult* partition,
                       const OutputResourcePtr& output) = 0;

  void RewriteDone(RewriteResult result, int partition_index);

  virtual bool SendFallbackResponse(StringPiece output_url_base,
                                    StringPiece contents,
                                    AsyncFetch* async_fetch,
                                    MessageHandler* handler);

  void StartNestedTasks();

  virtual void Harvest();

  virtual bool PolicyPermitsRendering() const = 0;

  bool AreOutputsAllowedByCsp(CspDirective role) const;

  virtual void Render();

  virtual void WillNotRender();

  virtual void Cancel();


  virtual const UrlSegmentEncoder* encoder() const;

  virtual GoogleString CacheKeySuffix() const;

  virtual GoogleString UserAgentCacheKey(
      const ResourceContext* context) const {
    return "";
  }

  virtual void EncodeUserAgentIntoResourceContext(ResourceContext* context) {}

  virtual const char* id() const = 0;

  virtual OutputResourceKind kind() const = 0;


  void AttachDependentRequestTrace(const StringPiece& label);

  RequestTrace* dependent_request_trace() { return dependent_request_trace_; }

  void TracePrintf(const char* fmt, ...);


  virtual void StartFetchReconstruction();

  void DetachFetch();

  virtual bool DecodeFetchUrls(const OutputResourcePtr& output_resource,
                               MessageHandler* message_handler,
                               GoogleUrlStarVector* url_vector);

  virtual void FixFetchFallbackHeaders(const CachedResult& cached_result,
                                       ResponseHeaders* headers);

  virtual void FetchCallbackDone(bool success);

  virtual void FetchTryFallback(const GoogleString& url,
                                const StringPiece& hash);

  void Freshen();

  bool notify_driver_on_fetch_done() const {
    return notify_driver_on_fetch_done_;
  }
  void set_notify_driver_on_fetch_done(bool value) {
    notify_driver_on_fetch_done_ = value;
  }

  AsyncFetch* async_fetch();

  bool FetchContextDetached();

  MessageHandler* fetch_message_handler();

  bool stale_rewrite() const { return stale_rewrite_; }

  virtual int64 GetRewriteDeadlineAlarmMs() const;

  virtual bool CreationLockBeforeStartFetch() const;

  virtual bool FailOnHashMismatch() const { return false; }

  virtual bool ScheduleViaCentralController() { return false; }

  virtual bool ScheduleNestedContextViaCentalController() const {
    return false;
  }

  void ObtainLockForCreation(ServerContext* server_context, Function* callback);

  void ReleaseCreationLock(bool succeeded);

  static bool LookupMetadataForOutputResourceImpl(
      OutputResourcePtr output_resource,
      const GoogleUrl& gurl,
      RewriteContext* rewrite_context,
      RewriteDriver* driver,
      GoogleString* error_out,
      CacheLookupResultCallback* callback);

 private:
  class OutputCacheCallback;
  class WriteIfChanged;
  class LookupMetadataForOutputResourceCallback;
  class HTTPCacheCallback;
  class ResourceCallbackUtils;
  class ResourceFetchCallback;
  class ResourceReconstructCallback;
  class ResourceRevalidateCallback;
  class InvokeRewriteFunction;
  class RewriteFreshenCallback;
  class TryLockFunction;
  friend class RewriteDriver;

  typedef std::set<RewriteContext*> ContextSet;

  enum FallbackCondition {
    kFallbackDiscretional,   
    kFallbackEmergency    
  };

  void Start();
  void SetPartitionKey();
  void StartFetch();
  void StartFetchImpl();
  void CancelFetch();
  void OutputCacheDone(CacheLookupResult* cache_result);
  void OutputCacheHit(bool write_partitions);
  void OutputCacheRevalidate(const InputInfoStarVector& to_revalidate);
  void OutputCacheMiss();
  void ResourceFetchDone(bool success, ResourcePtr resource, int slot_index);
  void ResourceRevalidateDone(InputInfo* input_info, bool success);
  void LogMetadataCacheInfo(bool cache_ok, bool can_revalidate);

  void RepeatedSuccess(const RewriteContext* primary);
  void RepeatedFailure();

  void Finalize();

  NamedLock* Lock();

  GoogleString LockName() const;

  void FetchInputs();

  void LockFailed();

  bool CreateOutputResourceFromContent(const CachedResult& cached_result,
                                       const ResponseHeaders& response_headers,
                                       StringPiece content,
                                       OutputResourcePtr* output_resource);

  bool IsFetchRewrite() const { return fetch_.get() != NULL; }

  void NestedRewriteDone(const RewriteContext* context);

  void Activate();

  void Propagate(RenderOp render_op);

  void StartRewriteForHtml();
  void StartRewriteForFetch();

  bool ReadyToRewrite() const;

  void DetachSlots();

  void RunSuccessors();

  void WritePartition();

  void FinalizeRewriteForHtml();

  void RetireRewriteForHtml(RenderOp permit_render);

  void MarkSlow();

  void MarkTooBusy();

  void CollectDependentTopLevel(ContextSet* contexts);

  void RewriteDoneImpl(RewriteResult result, int partition_index);

  void StartNestedTasksImpl();

  void RenderPartitionOnDetach(int partition_index);

  bool PrepareFetch(
      const OutputResourcePtr& output_resource,
      AsyncFetch* fetch,
      MessageHandler* message_handler);

  bool CreateOutputResourceForCachedOutput(const CachedResult* cached_result,
                                           OutputResourcePtr* output_resource);

  void FetchCacheDone(CacheLookupResult* cache_result);

  void FetchFallbackCacheDone(HTTPCache::FindResult result,
                              HTTPCache::Callback* data);

  bool CanFetchFallbackToOriginal(FallbackCondition circumstance) const;

  bool HasDuplicateOtherDependency(const InputInfo& input);

  void CheckAndAddOtherDependency(const InputInfo& input);

  void CheckAndFreshenResource(const InputInfo& input_info,
                               ResourcePtr resource, int partition_index,
                               int input_index,
                               FreshenMetadataUpdateManager* freshen_manager);
  ResourcePtr CreateUrlResource(const StringPiece& input_url);

  ResourceSlotVector slots_;

  std::vector<bool> render_slots_;


  bool started_;

  AtomicBool frozen_;

  scoped_ptr<OutputPartitions> partitions_;
  OutputResourceVector outputs_;
  int outstanding_fetches_;
  int outstanding_rewrites_;
  scoped_ptr<ResourceContext> resource_context_;
  GoogleString partition_key_;

  UrlSegmentEncoder default_encoder_;

  scoped_ptr<NamedLock> lock_;

  class FetchContext;
  scoped_ptr<FetchContext> fetch_;

  std::vector<RewriteContext*> successors_;

  std::vector<RewriteContext*> repeated_;

  int num_pending_nested_;
  std::vector<RewriteContext*> nested_;

  RewriteContext* parent_;

  RewriteDriver* driver_;

  int num_predecessors_;

  bool chained_;


  bool rewrite_done_;

  bool ok_to_write_output_partitions_;

  bool was_too_busy_;

  bool slow_;

  bool revalidate_ok_;

  bool notify_driver_on_fetch_done_;

  bool force_rewrite_;

  bool stale_rewrite_;

  bool is_metadata_cache_miss_;

  bool rewrite_uncacheable_;

  RequestTrace* dependent_request_trace_;

  StringIntMap other_dependency_map_;

  scoped_ptr<ScheduleRewriteContext> schedule_rewrite_context_;

  Variable* const num_rewrites_abandoned_for_lock_contention_;
  
};

}  

#endif