proxy_fetch.h

Go to the documentation of this file.

/*
 * Copyright 2011 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#ifndef PAGESPEED_AUTOMATIC_PROXY_FETCH_H_
#define PAGESPEED_AUTOMATIC_PROXY_FETCH_H_

#include <map>
#include <set>
#include <vector>

#include "net/instaweb/http/public/async_fetch.h"
#include "net/instaweb/http/public/request_context.h"
#include "net/instaweb/util/public/fallback_property_page.h"
#include "net/instaweb/util/public/property_cache.h"
#include "pagespeed/automatic/html_detector.h"
#include "pagespeed/kernel/base/abstract_mutex.h"
#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/function.h"
#include "pagespeed/kernel/base/gtest_prod.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/http/http_names.h"
#include "pagespeed/kernel/http/response_headers.h"
#include "pagespeed/kernel/http/user_agent_matcher.h"
#include "pagespeed/kernel/thread/queued_worker_pool.h"

namespace net_instaweb {

class CacheUrlAsyncFetcher;
class GoogleUrl;
class MessageHandler;
class ProxyFetch;
class ProxyFetchPropertyCallbackCollector;
class QueuedAlarm;
class ServerContext;
class RewriteDriver;
class RewriteOptions;
class Timer;

class ProxyFetchFactory {
 public:
  explicit ProxyFetchFactory(ServerContext* server_context);
  ~ProxyFetchFactory();

  void StartNewProxyFetch(
      const GoogleString& url,
      AsyncFetch* async_fetch,
      RewriteDriver* driver,
      ProxyFetchPropertyCallbackCollector* property_callback,
      AsyncFetch* original_content_fetch);

  ProxyFetch* CreateNewProxyFetch(
      const GoogleString& url,
      AsyncFetch* async_fetch,
      RewriteDriver* driver,
      ProxyFetchPropertyCallbackCollector* property_callback,
      AsyncFetch* original_content_fetch);

  static ProxyFetchPropertyCallbackCollector* InitiatePropertyCacheLookup(
      bool is_resource_fetch,
      const GoogleUrl& request_url,
      ServerContext* server_context,
      RewriteOptions* options,
      AsyncFetch* async_fetch);

  MessageHandler* message_handler() const { return handler_; }

 private:
  friend class ProxyFetch;

  void RegisterNewFetch(ProxyFetch* proxy_fetch);
  void RegisterFinishedFetch(ProxyFetch* proxy_fetch);

  ServerContext* server_context_;
  Timer* timer_;
  MessageHandler* handler_;

  scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
  std::set<ProxyFetch*> outstanding_proxy_fetches_;

  
};

class ProxyFetchPropertyCallback : public PropertyPage {
 public:
  ProxyFetchPropertyCallback(PageType page_type,
                             PropertyCache* property_cache,
                             const StringPiece& url,
                             const StringPiece& options_signature_hash,
                             UserAgentMatcher::DeviceType device_type,
                             ProxyFetchPropertyCallbackCollector* collector,
                             AbstractMutex* mutex);

  PageType page_type() const { return page_type_; }

  virtual bool IsCacheValid(int64 write_timestamp_ms) const;

  virtual void Done(bool success);

 private:
  PageType page_type_;
  UserAgentMatcher::DeviceType device_type_;
  ProxyFetchPropertyCallbackCollector* collector_;
  GoogleString url_;
  
};

class ProxyFetchPropertyCallbackCollector {
 public:
  ProxyFetchPropertyCallbackCollector(ServerContext* server_context,
                                      const StringPiece& url,
                                      const RequestContextPtr& req_ctx,
                                      const RewriteOptions* options,
                                      UserAgentMatcher::DeviceType device_type);
  virtual ~ProxyFetchPropertyCallbackCollector();

  void AddCallback(ProxyFetchPropertyCallback* callback);

  void RequestHeadersComplete();

  void ConnectProxyFetch(ProxyFetch* proxy_fetch);

  void Detach(HttpStatus::Code status_code);

  PropertyPage* property_page() {
    return fallback_property_page_ == NULL ?
        NULL : fallback_property_page_->actual_property_page();
  }

  FallbackPropertyPage* fallback_property_page() {
    return fallback_property_page_.get();
  }

  PropertyPage* ReleasePropertyPage(
      ProxyFetchPropertyCallback::PageType page_type);

  FallbackPropertyPage* ReleaseFallbackPropertyPage() {
    return fallback_property_page_.release();
  }

  PropertyPage* ReleaseOriginPropertyPage() {
    return origin_property_page_.release();
  }

  void AddPostLookupTask(Function* func);

  bool IsCacheValid(int64 write_timestamp_ms) const;

  void Done(ProxyFetchPropertyCallback* callback);

  const RequestContextPtr& request_context() { return request_context_; }

  UserAgentMatcher::DeviceType device_type() { return device_type_; }

 private:
  friend class ProxyFetchPropertyCallbackCollectorTest;
  void ExecuteDone(ProxyFetchPropertyCallback* callback);
  void ExecuteAddPostLookupTask(Function* func);
  void ExecuteConnectProxyFetch(ProxyFetch* proxy_fetch);
  void ExecuteDetach(HttpStatus::Code status_code);
  void ExecuteRequestHeadersComplete();

  void RunPostLookupsAndCleanupIfSafe();

  void UpdateStatusCodeInPropertyCache();

  std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
  std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*>
  property_pages_;
  scoped_ptr<AbstractMutex> mutex_;
  ServerContext* const server_context_;
  QueuedWorkerPool::Sequence* const sequence_;
  const GoogleString url_;
  const RequestContextPtr request_context_;
  const UserAgentMatcher::DeviceType device_type_;
  bool is_options_valid_; 
  bool detached_;
  bool done_;
  bool request_headers_ok_;
  ProxyFetch* proxy_fetch_;
  std::vector<Function*> post_lookup_task_vector_;
  const RewriteOptions* options_; 
  HttpStatus::Code status_code_; 
  scoped_ptr<FallbackPropertyPage> fallback_property_page_;
  scoped_ptr<PropertyPage> origin_property_page_;

  
};

class ProxyFetch : public SharedAsyncFetch {
 public:
  static const char kCollectorConnectProxyFetchFinish[];
  static const char kCollectorDetachFinish[];
  static const char kCollectorDoneFinish[];
  static const char kCollectorFinish[];
  static const char kCollectorDetachStart[];
  static const char kCollectorRequestHeadersCompleteFinish[];

  static const char kHeadersSetupRaceAlarmQueued[];
  static const char kHeadersSetupRaceDone[];
  static const char kHeadersSetupRaceFlush[];
  static const char kHeadersSetupRacePrefix[];
  static const char kHeadersSetupRaceWait[];

  static const int kTestSignalTimeoutMs = 200;

  void set_trusted_input(bool trusted_input) { trusted_input_ = trusted_input; }

 protected:
  virtual void HandleHeadersComplete();
  virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
  virtual bool HandleFlush(MessageHandler* handler);
  virtual void HandleDone(bool success);
  virtual bool IsCachedResultValid(const ResponseHeaders& headers);

 private:
  friend class ProxyFetchFactory;
  friend class ProxyFetchPropertyCallbackCollector;
  friend class MockProxyFetch;
  FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
  FRIEND_TEST(ProxyFetchTest, TestFollowFlushes);

  virtual void PropertyCacheComplete(
      ProxyFetchPropertyCallbackCollector* collector);

  ProxyFetch(const GoogleString& url,
             bool cross_domain,
             ProxyFetchPropertyCallbackCollector* property_cache_callback,
             AsyncFetch* async_fetch,
             AsyncFetch* original_content_fetch,
             RewriteDriver* driver,
             ServerContext* server_context,
             Timer* timer,
             ProxyFetchFactory* factory);
  virtual ~ProxyFetch();

  const RewriteOptions* Options();

  void SetupForHtml();

  void AddPagespeedHeader();

  bool StartParse();

  void StartFetch();

  void DoFetch(bool prepare_success);

  void ExecuteQueued();

  void ScheduleQueueExecutionIfNeeded();

  void Finish(bool success);

  void CompleteFinishParse(bool success);

  void FlushDone();


  void CancelIdleAlarm();

  void QueueIdleAlarm();

  void HandleIdleAlarm();

  GoogleString url_;
  ServerContext* server_context_;
  Timer* timer_;

  scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;

  bool cross_domain_;

  bool claims_html_;

  bool started_parse_;

  bool parse_text_called_;

  bool done_called_;

  HtmlDetector html_detector_;

  ProxyFetchPropertyCallbackCollector* property_cache_callback_;

  AsyncFetch* original_content_fetch_;

  RewriteDriver* driver_;

  bool queue_run_job_created_;

  scoped_ptr<AbstractMutex> mutex_;
  StringStarVector text_queue_;
  bool network_flush_outstanding_;
  QueuedWorkerPool::Sequence* sequence_;

  bool done_outstanding_;

  bool finishing_;

  bool done_result_;

  bool waiting_for_flush_to_finish_;

  QueuedAlarm* idle_alarm_;

  ProxyFetchFactory* factory_;

  bool trusted_input_;

  
};

}  

#endif