server_context.h

Go to the documentation of this file.

/*
 * Copyright 2010 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#ifndef NET_INSTAWEB_REWRITER_PUBLIC_SERVER_CONTEXT_H_
#define NET_INSTAWEB_REWRITER_PUBLIC_SERVER_CONTEXT_H_

#include <cstddef>                     
#include <set>
#include <utility>
#include <vector>

#include "net/instaweb/http/public/cache_url_async_fetcher.h"
#include "net/instaweb/http/public/http_cache.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/http/public/url_async_fetcher.h"
#include "net/instaweb/rewriter/public/output_resource.h"
#include "net/instaweb/rewriter/public/resource.h"
#include "net/instaweb/util/public/property_cache.h"
#include "pagespeed/controller/central_controller.h"
#include "pagespeed/kernel/base/abstract_mutex.h"
#include "pagespeed/kernel/base/atomic_bool.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/cache_interface.h"
#include "pagespeed/kernel/base/function.h"
#include "pagespeed/kernel/base/hasher.h"
#include "pagespeed/kernel/base/md5_hasher.h"
#include "pagespeed/kernel/base/ref_counted_ptr.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/thread_system.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/request_headers.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/thread/queued_worker_pool.h"
#include "pagespeed/kernel/thread/sequence.h"
#include "pagespeed/kernel/util/simple_random.h"

namespace pagespeed { namespace js { struct JsTokenizerPatterns; } }

namespace net_instaweb {

class AsyncFetch;
class CachePropertyStore;
class CriticalImagesFinder;
class CriticalSelectorFinder;
class RequestProperties;
class ExperimentMatcher;
class FileSystem;
class GoogleUrl;
class MessageHandler;
class NamedLock;
class NamedLockManager;
class PropertyStore;
class RewriteDriver;
class RewriteDriverFactory;
class RewriteDriverPool;
class RewriteFilter;
class RewriteOptions;
class RewriteOptionsManager;
class RewriteQuery;
class RewriteStats;
class SHA1Signature;
class Scheduler;
class StaticAssetManager;
class Statistics;
class ThreadSynchronizer;
class Timer;
class UrlNamer;
class UsageDataReporter;
class UserAgentMatcher;

typedef RefCountedPtr<OutputResource> OutputResourcePtr;
typedef std::vector<OutputResourcePtr> OutputResourceVector;

class ServerContext {
 public:
  typedef std::pair<RewriteOptions*, bool> OptionsBoolPair;

  static const int64 kGeneratedMaxAgeMs;
  static const int64 kCacheTtlForMismatchedContentMs;

  static const char kResourceEtagValue[];
  static const char kCacheKeyResourceNamePrefix[];

  explicit ServerContext(RewriteDriverFactory* factory);
  virtual ~ServerContext();

  void SetDefaultLongCacheHeaders(
      const ContentType* content_type, StringPiece charset,
      StringPiece cache_control_suffix, ResponseHeaders* header) const;

  void set_filename_prefix(const StringPiece& file_prefix);
  void set_statistics(Statistics* x) { statistics_ = x; }
  void set_rewrite_stats(RewriteStats* x) { rewrite_stats_ = x; }
  void set_lock_manager(NamedLockManager* x) { lock_manager_ = x; }
  void set_enable_property_cache(bool enabled);
  void set_message_handler(MessageHandler* x) { message_handler_ = x; }

  StringPiece filename_prefix() const { return file_prefix_; }
  Statistics* statistics() const { return statistics_; }
  NamedLockManager* lock_manager() const { return lock_manager_; }
  RewriteDriverFactory* factory() const { return factory_; }
  ThreadSynchronizer* thread_synchronizer() {
    return thread_synchronizer_.get();
  }
  ExperimentMatcher* experiment_matcher() { return experiment_matcher_.get(); }

  void ApplyInputCacheControl(const ResourceVector& inputs,
                              ResponseHeaders* headers);

  bool IsPagespeedResource(const GoogleUrl& url) const;

  const RewriteFilter* FindFilterForDecoding(const StringPiece& id) const;

  bool DecodeUrlGivenOptions(const GoogleUrl& url,
                             const RewriteOptions* options,
                             const UrlNamer* url_namer,
                             StringVector* decoded_urls) const;

  void ComputeSignature(RewriteOptions* rewrite_options) const;

  Hasher* hasher() const { return hasher_; }
  const Hasher* lock_hasher() const { return &lock_hasher_; }
  const Hasher* contents_hasher() const { return &contents_hasher_; }
  FileSystem* file_system() { return file_system_; }
  void set_file_system(FileSystem* fs ) { file_system_ = fs; }
  UrlNamer* url_namer() const { return url_namer_; }
  void set_url_namer(UrlNamer* n) { url_namer_ = n; }
  RewriteOptionsManager* rewrite_options_manager() const {
    return rewrite_options_manager_.get();
  }
  SHA1Signature* signature() const { return signature_; }
  void SetRewriteOptionsManager(RewriteOptionsManager* rom);
  StaticAssetManager* static_asset_manager() const {
    return static_asset_manager_;
  }
  void set_static_asset_manager(StaticAssetManager* manager) {
    static_asset_manager_ = manager;
  }
  Scheduler* scheduler() const { return scheduler_; }
  void set_scheduler(Scheduler* s) { scheduler_ = s; }
  bool has_default_system_fetcher() const {
    return default_system_fetcher_ != NULL;
  }
  UrlAsyncFetcher* DefaultSystemFetcher() { return default_system_fetcher_; }

  virtual CacheUrlAsyncFetcher* CreateCustomCacheFetcher(
      const RewriteOptions* options, const GoogleString& fragment,
      CacheUrlAsyncFetcher::AsyncOpHooks* hooks, UrlAsyncFetcher* fetcher);

  Timer* timer() const { return timer_; }

  void set_timer(Timer* timer) { timer_ = timer; }

  HTTPCache* http_cache() const { return http_cache_.get(); }
  void set_http_cache(HTTPCache* x) { http_cache_.reset(x); }

  void MakePagePropertyCache(PropertyStore* property_store);

  PropertyCache* page_property_cache() const {
    return page_property_cache_.get();
  }

  const PropertyCache::Cohort* dom_cohort() const { return dom_cohort_; }
  void set_dom_cohort(const PropertyCache::Cohort* c) { dom_cohort_ = c; }

  const PropertyCache::Cohort* beacon_cohort() const { return beacon_cohort_; }
  void set_beacon_cohort(const PropertyCache::Cohort* c) { beacon_cohort_ = c; }

  const PropertyCache::Cohort* dependencies_cohort() const {
    return dependencies_cohort_;
  }
  void set_dependencies_cohort(const PropertyCache::Cohort* c) {
    dependencies_cohort_ = c;
  }

  const PropertyCache::Cohort* fix_reflow_cohort() const {
    return fix_reflow_cohort_;
  }
  void set_fix_reflow_cohort(const PropertyCache::Cohort* c) {
    fix_reflow_cohort_ = c;
  }

  CacheInterface* filesystem_metadata_cache() const {
    return filesystem_metadata_cache_;
  }
  void set_filesystem_metadata_cache(CacheInterface* x) {
    filesystem_metadata_cache_ = x;
  }

  CacheInterface* metadata_cache() const { return metadata_cache_; }
  void set_metadata_cache(CacheInterface* x) { metadata_cache_ = x; }

  CriticalImagesFinder* critical_images_finder() const {
    return critical_images_finder_.get();
  }
  void set_critical_images_finder(CriticalImagesFinder* finder);

  CriticalSelectorFinder* critical_selector_finder() const {
    return critical_selector_finder_.get();
  }
  void set_critical_selector_finder(CriticalSelectorFinder* finder);

  UserAgentMatcher* user_agent_matcher() const {
    return user_agent_matcher_;
  }
  void set_user_agent_matcher(UserAgentMatcher* n) { user_agent_matcher_ = n; }

  SimpleRandom* simple_random() {
    return &simple_random_;
  }

  bool store_outputs_in_file_system() { return store_outputs_in_file_system_; }
  void set_store_outputs_in_file_system(bool store) {
    store_outputs_in_file_system_ = store;
  }

  RewriteStats* rewrite_stats() const { return rewrite_stats_; }
  MessageHandler* message_handler() const { return message_handler_; }

  NamedLock* MakeCreationLock(const GoogleString& name);

  void TryLockForCreation(NamedLock* creation_lock, Function* callback);

  void LockForCreation(NamedLock* creation_lock,
                       Sequence* worker, Function* callback);

  void set_hasher(Hasher* hasher) { hasher_ = hasher; }
  void set_signature(SHA1Signature* signature) { signature_ = signature; }
  void set_default_system_fetcher(UrlAsyncFetcher* fetcher) {
    default_system_fetcher_ = fetcher;
  }

  bool HandleBeacon(StringPiece body,
                    StringPiece user_agent,
                    const RequestContextPtr& request_context);

  RewriteOptions* global_options();

  const RewriteOptions* global_options() const;

  void reset_global_options(RewriteOptions* options);

  RewriteOptions* NewOptions();

  bool GetQueryOptions(const RequestContextPtr& request_context,
                       const RewriteOptions* domain_options,
                       GoogleUrl* request_url,
                       RequestHeaders* request_headers,
                       ResponseHeaders* response_headers,
                       RewriteQuery* rewrite_query);

  void GetRemoteOptions(RewriteOptions* remote_options, bool on_startup);

  RewriteOptions* GetCustomOptions(RequestHeaders* request_headers,
                                   RewriteOptions* domain_options,
                                   RewriteOptions* query_options);

  GoogleString GetRewriteOptionsSignatureHash(const RewriteOptions* options);

  RewriteDriver* NewRewriteDriver(const RequestContextPtr& request_ctx);

  RewriteDriver* NewRewriteDriverFromPool(
      RewriteDriverPool* pool, const RequestContextPtr& request_ctx);

  RewriteDriver* NewUnmanagedRewriteDriver(
      RewriteDriverPool* pool, RewriteOptions* options,
      const RequestContextPtr& request_ctx);

  RewriteDriver* NewCustomRewriteDriver(
      RewriteOptions* custom_options, const RequestContextPtr& request_ctx);

  void ReleaseRewriteDriver(RewriteDriver* rewrite_driver);

  ThreadSystem* thread_system() { return thread_system_; }
  UsageDataReporter* usage_data_reporter() { return usage_data_reporter_; }

  void set_shutting_down() {
    shutting_down_.set_value(true);
  }

  bool shutting_down() const {
    return shutting_down_.value();
  }

  void ShutDownDrivers(int64 cutoff_time_ms);

  void MergeNonCachingResponseHeaders(const ResourcePtr& input,
                                      const OutputResourcePtr& output) {
    MergeNonCachingResponseHeaders(*input->response_headers(),
                                   output->response_headers());
  }

  void MergeNonCachingResponseHeaders(const ResponseHeaders& input_headers,
                                      ResponseHeaders* output_headers);

  QueuedWorkerPool* html_workers() { return html_workers_; }

  QueuedWorkerPool* rewrite_workers() { return rewrite_workers_; }

  QueuedWorkerPool* low_priority_rewrite_workers() {
    return low_priority_rewrite_workers_;
  }

  size_t num_active_rewrite_drivers();

  void InitWorkers();

  virtual void PostInitHook();

  static bool IsExcludedAttribute(const char* attribute);

  bool response_headers_finalized() const {
    return response_headers_finalized_;
  }
  void set_response_headers_finalized(bool x) {
    response_headers_finalized_ = x;
  }

  RewriteDriverPool* standard_rewrite_driver_pool() {
    return available_rewrite_drivers_.get();
  }

  const GoogleString& hostname() const {
    return hostname_;
  }
  void set_hostname(const GoogleString& x) {
    hostname_ = x;
  }

  void set_central_controller(std::shared_ptr<CentralController> controller) {
    central_controller_ = controller;
  }

  CentralController* central_controller() {
    return central_controller_.get();
  }

  void AddOriginalContentLengthHeader(const ResourceVector& inputs,
                                      ResponseHeaders* headers);

  virtual void ApplySessionFetchers(const RequestContextPtr& req,
                                    RewriteDriver* driver);

  virtual bool ProxiesHtml() const = 0;

  RequestProperties* NewRequestProperties();

  void DeleteCacheOnDestruction(CacheInterface* cache);

  void set_cache_property_store(CachePropertyStore* p);

  void set_decoding_driver(RewriteDriver* rd) { decoding_driver_ = rd; }

  virtual PropertyStore* CreatePropertyStore(CacheInterface* cache_backend);

  const PropertyCache::Cohort* AddCohort(
      const GoogleString& cohort_name,
      PropertyCache* pcache);

  const PropertyCache::Cohort* AddCohortWithCache(
      const GoogleString& cohort_name,
      CacheInterface* cache,
      PropertyCache* pcache);

  const CacheInterface* pcache_cache_backend();

  const pagespeed::js::JsTokenizerPatterns* js_tokenizer_patterns() const {
    return js_tokenizer_patterns_;
  }

  enum Format {
    kFormatAsHtml,
    kFormatAsJson
  };

  void ShowCacheHandler(Format format, StringPiece url, StringPiece ua,
                        bool should_delete, AsyncFetch* fetch,
                        RewriteOptions* options);

  static GoogleString ShowCacheForm(StringPiece user_agent);

  virtual GoogleString FormatOption(StringPiece option_name, StringPiece args);

 protected:
  void ManageRewriteDriverPool(RewriteDriverPool* pool) {
    additional_driver_pools_.push_back(pool);
  }

 private:
  friend class ServerContextTest;
  typedef std::set<RewriteDriver*> RewriteDriverSet;

  void ReleaseRewriteDriverImpl(RewriteDriver* rewrite_driver);

  void ApplyRemoteConfig(const GoogleString& config, RewriteOptions* options);
  void ApplyConfigLine(StringPiece linesp, RewriteOptions* options);
  GoogleString FetchRemoteConfig(const GoogleString& url, int64 timeout_ms,
                                 bool on_startup,
                                 RequestContextPtr request_ctx);

  ThreadSystem* thread_system_;
  RewriteStats* rewrite_stats_;
  GoogleString file_prefix_;
  FileSystem* file_system_;
  UrlNamer* url_namer_;
  scoped_ptr<RewriteOptionsManager> rewrite_options_manager_;
  UserAgentMatcher* user_agent_matcher_;
  Scheduler* scheduler_;
  UrlAsyncFetcher* default_system_fetcher_;
  Hasher* hasher_;
  SHA1Signature* signature_;
  scoped_ptr<CriticalImagesFinder> critical_images_finder_;
  scoped_ptr<CriticalSelectorFinder> critical_selector_finder_;

  MD5Hasher lock_hasher_; 

  MD5Hasher contents_hasher_;

  Statistics* statistics_;

  Timer* timer_;
  scoped_ptr<HTTPCache> http_cache_;
  scoped_ptr<PropertyCache> page_property_cache_;
  CacheInterface* filesystem_metadata_cache_;
  CacheInterface* metadata_cache_;

  bool store_outputs_in_file_system_;
  bool response_headers_finalized_;
  bool enable_property_cache_;

  NamedLockManager* lock_manager_;
  MessageHandler* message_handler_;

  const PropertyCache::Cohort* dom_cohort_;
  const PropertyCache::Cohort* beacon_cohort_;
  const PropertyCache::Cohort* dependencies_cohort_;
  const PropertyCache::Cohort* fix_reflow_cohort_;

  scoped_ptr<RewriteDriverPool> available_rewrite_drivers_;

  std::vector<RewriteDriverPool*> additional_driver_pools_;

  RewriteDriverSet active_rewrite_drivers_;

  bool trying_to_cleanup_rewrite_drivers_;
  RewriteDriverSet deferred_release_rewrite_drivers_;
  bool shutdown_drivers_called_;

  RewriteDriverFactory* factory_;

  scoped_ptr<AbstractMutex> rewrite_drivers_mutex_;

  scoped_ptr<RewriteOptions> base_class_options_;

  RewriteDriver* decoding_driver_;

  QueuedWorkerPool* html_workers_; 
  QueuedWorkerPool* rewrite_workers_; 
  QueuedWorkerPool* low_priority_rewrite_workers_; 

  AtomicBool shutting_down_;

  StaticAssetManager* static_asset_manager_;

  scoped_ptr<ThreadSynchronizer> thread_synchronizer_;

  scoped_ptr<ExperimentMatcher> experiment_matcher_;

  UsageDataReporter* usage_data_reporter_;

  GoogleString hostname_;

  SimpleRandom simple_random_;
  const pagespeed::js::JsTokenizerPatterns* js_tokenizer_patterns_;

  scoped_ptr<CachePropertyStore> cache_property_store_;

  std::shared_ptr<CentralController> central_controller_;

  
};

}  

#endif