Page Speed Optimization Libraries  1.9.32.2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
proxy_fetch.h
Go to the documentation of this file.
1 /*
2  * Copyright 2011 Google Inc.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  * http:///www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
22 
23 #ifndef NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
24 #define NET_INSTAWEB_AUTOMATIC_PUBLIC_PROXY_FETCH_H_
25 
26 #include <map>
27 #include <set>
28 #include <vector>
29 
32 #include "net/instaweb/http/public/request_context.h"
33 #include "net/instaweb/util/public/fallback_property_page.h"
34 #include "net/instaweb/util/public/property_cache.h"
35 #include "pagespeed/kernel/base/basictypes.h"
36 #include "pagespeed/kernel/base/gtest_prod.h"
37 #include "pagespeed/kernel/base/scoped_ptr.h"
38 #include "pagespeed/kernel/base/string.h"
39 #include "pagespeed/kernel/base/string_util.h"
40 #include "pagespeed/kernel/http/http_names.h"
41 #include "pagespeed/kernel/http/user_agent_matcher.h"
42 #include "pagespeed/kernel/thread/queued_worker_pool.h"
43 
44 namespace net_instaweb {
45 
46 class AbstractMutex;
47 class CacheUrlAsyncFetcher;
48 class Function;
49 class GoogleUrl;
50 class MessageHandler;
51 class ProxyFetch;
52 class ProxyFetchPropertyCallbackCollector;
53 class QueuedAlarm;
54 class ServerContext;
55 class ResponseHeaders;
56 class RewriteDriver;
57 class RewriteOptions;
58 class Timer;
59 
63  public:
64  explicit ProxyFetchFactory(ServerContext* server_context);
66 
69  void StartNewProxyFetch(
70  const GoogleString& url,
71  AsyncFetch* async_fetch,
72  RewriteDriver* driver,
73  ProxyFetchPropertyCallbackCollector* property_callback,
74  AsyncFetch* original_content_fetch);
75 
86  const GoogleString& url,
87  AsyncFetch* async_fetch,
88  RewriteDriver* driver,
89  ProxyFetchPropertyCallbackCollector* property_callback,
90  AsyncFetch* original_content_fetch);
91 
95  bool is_resource_fetch,
96  const GoogleUrl& request_url,
97  ServerContext* server_context,
98  RewriteOptions* options,
99  AsyncFetch* async_fetch,
100  const bool requires_blink_cohort,
101  bool* added_page_property_callback);
102 
103  MessageHandler* message_handler() const { return handler_; }
104 
105  private:
106  friend class ProxyFetch;
107 
112  void RegisterNewFetch(ProxyFetch* proxy_fetch);
113  void RegisterFinishedFetch(ProxyFetch* proxy_fetch);
114 
115  ServerContext* server_context_;
116  Timer* timer_;
117  MessageHandler* handler_;
118 
119  scoped_ptr<AbstractMutex> outstanding_proxy_fetches_mutex_;
120  std::set<ProxyFetch*> outstanding_proxy_fetches_;
121 
122  DISALLOW_COPY_AND_ASSIGN(ProxyFetchFactory);
123 };
124 
133 class ProxyFetchPropertyCallback : public PropertyPage {
134  public:
135  ProxyFetchPropertyCallback(PageType page_type,
136  PropertyCache* property_cache,
137  const StringPiece& url,
138  const StringPiece& options_signature_hash,
139  UserAgentMatcher::DeviceType device_type,
141  AbstractMutex* mutex);
142 
143  PageType page_type() const { return page_type_; }
144 
146  virtual bool IsCacheValid(int64 write_timestamp_ms) const;
147 
148  virtual void Done(bool success);
149 
150  private:
151  PageType page_type_;
152  UserAgentMatcher::DeviceType device_type_;
154  GoogleString url_;
155  DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallback);
156 };
157 
179  public:
181  const StringPiece& url,
182  const RequestContextPtr& req_ctx,
183  const RewriteOptions* options,
184  UserAgentMatcher::DeviceType device_type);
186 
189  void AddCallback(ProxyFetchPropertyCallback* callback);
190 
193  void RequestHeadersComplete();
194 
201  void ConnectProxyFetch(ProxyFetch* proxy_fetch);
202 
209  void Detach(HttpStatus::Code status_code);
210 
212  PropertyPage* property_page() {
213  return fallback_property_page_ == NULL ?
214  NULL : fallback_property_page_->actual_property_page();
215  }
216 
218  FallbackPropertyPage* fallback_property_page() {
219  return fallback_property_page_.get();
220  }
221 
224  PropertyPage* ReleasePropertyPage(
225  ProxyFetchPropertyCallback::PageType page_type);
226 
228  FallbackPropertyPage* ReleaseFallbackPropertyPage() {
229  return fallback_property_page_.release();
230  }
231 
242  void AddPostLookupTask(Function* func);
243 
247  bool IsCacheValid(int64 write_timestamp_ms) const;
248 
250  void Done(ProxyFetchPropertyCallback* callback);
251 
252  const RequestContextPtr& request_context() { return request_context_; }
253 
255  UserAgentMatcher::DeviceType device_type() { return device_type_; }
256 
257  private:
258  friend class ProxyFetchPropertyCallbackCollectorTest;
259  void ExecuteDone(ProxyFetchPropertyCallback* callback);
260  void ExecuteAddPostLookupTask(Function* func);
261  void ExecuteConnectProxyFetch(ProxyFetch* proxy_fetch);
262  void ExecuteDetach(HttpStatus::Code status_code);
263  void ExecuteRequestHeadersComplete();
264 
265  void RunPostLookupsAndCleanupIfSafe();
266 
268  void UpdateStatusCodeInPropertyCache();
269 
270  std::set<ProxyFetchPropertyCallback*> pending_callbacks_;
271  std::map<ProxyFetchPropertyCallback::PageType, PropertyPage*>
272  property_pages_;
273  scoped_ptr<AbstractMutex> mutex_;
274  ServerContext* const server_context_;
275  QueuedWorkerPool::Sequence* const sequence_;
276  const GoogleString url_;
277  const RequestContextPtr request_context_;
278  const UserAgentMatcher::DeviceType device_type_;
279  bool is_options_valid_;
280  bool detached_;
283  bool done_;
284  bool request_headers_ok_;
285  ProxyFetch* proxy_fetch_;
286  std::vector<Function*> post_lookup_task_vector_;
287  const RewriteOptions* options_;
288  HttpStatus::Code status_code_;
289  scoped_ptr<FallbackPropertyPage> fallback_property_page_;
290 
291  DISALLOW_COPY_AND_ASSIGN(ProxyFetchPropertyCallbackCollector);
292 };
293 
314 class ProxyFetch : public SharedAsyncFetch {
315  public:
319  static const char kCollectorDetachFinish[];
320  static const char kCollectorDoneFinish[];
321  static const char kCollectorFinish[];
322  static const char kCollectorDetachStart[];
323  static const char kCollectorRequestHeadersCompleteFinish[];
324 
327  static const char kHeadersSetupRaceAlarmQueued[];
328  static const char kHeadersSetupRaceDone[];
329  static const char kHeadersSetupRaceFlush[];
330  static const char kHeadersSetupRacePrefix[];
331  static const char kHeadersSetupRaceWait[];
332 
339  static const int kTestSignalTimeoutMs = 200;
340 
341  protected:
343  virtual void HandleHeadersComplete();
344  virtual bool HandleWrite(const StringPiece& content, MessageHandler* handler);
345  virtual bool HandleFlush(MessageHandler* handler);
346  virtual void HandleDone(bool success);
347  virtual bool IsCachedResultValid(const ResponseHeaders& headers);
348 
349  private:
350  friend class ProxyFetchFactory;
352  friend class MockProxyFetch;
353  FRIEND_TEST(ProxyFetchTest, TestInhibitParsing);
354 
357  virtual void PropertyCacheComplete(
359 
362  ProxyFetch(const GoogleString& url,
363  bool cross_domain,
364  ProxyFetchPropertyCallbackCollector* property_cache_callback,
365  AsyncFetch* async_fetch,
366  AsyncFetch* original_content_fetch,
367  RewriteDriver* driver,
368  ServerContext* server_context,
369  Timer* timer,
370  ProxyFetchFactory* factory);
371  virtual ~ProxyFetch();
372 
373  const RewriteOptions* Options();
374 
376  void SetupForHtml();
377 
379  void AddPagespeedHeader();
380 
383  bool StartParse();
384 
386  void StartFetch();
387 
391  void DoFetch(bool prepare_success);
392 
395  void ExecuteQueued();
396 
399  void ScheduleQueueExecutionIfNeeded();
400 
405  void Finish(bool success);
406 
408  void CompleteFinishParse(bool success);
409 
412  void FlushDone();
413 
416 
418  void CancelIdleAlarm();
419 
421  void QueueIdleAlarm();
422 
424  void HandleIdleAlarm();
425 
426  GoogleString url_;
427  ServerContext* server_context_;
428  Timer* timer_;
429 
430  scoped_ptr<CacheUrlAsyncFetcher> cache_fetcher_;
431 
434  bool cross_domain_;
435 
437  bool claims_html_;
438 
441  bool started_parse_;
442 
444  bool parse_text_called_;
445 
447  bool done_called_;
448 
449  HtmlDetector html_detector_;
450 
455  ProxyFetchPropertyCallbackCollector* property_cache_callback_;
456 
460  AsyncFetch* original_content_fetch_;
461 
464  RewriteDriver* driver_;
465 
468  bool queue_run_job_created_;
469 
481  scoped_ptr<AbstractMutex> mutex_;
482  StringStarVector text_queue_;
483  bool network_flush_outstanding_;
484  QueuedWorkerPool::Sequence* sequence_;
485 
488  bool done_outstanding_;
489 
492  bool finishing_;
493 
496  bool done_result_;
497 
501  bool waiting_for_flush_to_finish_;
502 
505  QueuedAlarm* idle_alarm_;
506 
507  ProxyFetchFactory* factory_;
508 
510  bool distributed_fetch_;
511 
512  DISALLOW_COPY_AND_ASSIGN(ProxyFetch);
513 };
514 
515 }
516 
517 #endif
static ProxyFetchPropertyCallbackCollector * InitiatePropertyCacheLookup(bool is_resource_fetch, const GoogleUrl &request_url, ServerContext *server_context, RewriteOptions *options, AsyncFetch *async_fetch, const bool requires_blink_cohort, bool *added_page_property_callback)
Definition: html_detector.h:54
Definition: proxy_fetch.h:314
static const int kTestSignalTimeoutMs
Definition: proxy_fetch.h:339
void AddCallback(ProxyFetchPropertyCallback *callback)
UserAgentMatcher::DeviceType device_type()
Returns DeviceType from device property page.
Definition: proxy_fetch.h:255
FallbackPropertyPage * fallback_property_page()
Returns the fallback property page.
Definition: proxy_fetch.h:218
static const char kCollectorConnectProxyFetchFinish[]
Definition: proxy_fetch.h:318
virtual bool IsCacheValid(int64 write_timestamp_ms) const
Delegates to collector_'s IsCacheValid.
void Detach(HttpStatus::Code status_code)
static const char kHeadersSetupRaceAlarmQueued[]
Definition: proxy_fetch.h:327
virtual void HandleHeadersComplete()
protected interface from AsyncFetch.
Definition: proxy_fetch.h:133
void ConnectProxyFetch(ProxyFetch *proxy_fetch)
Definition: async_fetch.h:267
FallbackPropertyPage * ReleaseFallbackPropertyPage()
Releases the ownership of fallback property page.
Definition: proxy_fetch.h:228
Definition: async_fetch.h:53
PropertyPage * ReleasePropertyPage(ProxyFetchPropertyCallback::PageType page_type)
bool IsCacheValid(int64 write_timestamp_ms) const
Definition: rewrite_driver.h:98
Definition: server_context.h:101
PropertyPage * property_page()
Returns the actual property page.
Definition: proxy_fetch.h:212
ProxyFetch * CreateNewProxyFetch(const GoogleString &url, AsyncFetch *async_fetch, RewriteDriver *driver, ProxyFetchPropertyCallbackCollector *property_callback, AsyncFetch *original_content_fetch)
Definition: proxy_fetch.h:62
virtual bool IsCachedResultValid(const ResponseHeaders &headers)
Definition: rewrite_options.h:83
void Done(ProxyFetchPropertyCallback *callback)
Called by a ProxyFetchPropertyCallback when the former is complete.
void StartNewProxyFetch(const GoogleString &url, AsyncFetch *async_fetch, RewriteDriver *driver, ProxyFetchPropertyCallbackCollector *property_callback, AsyncFetch *original_content_fetch)