Page Speed Optimization Libraries
1.7.30.2
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00020 00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00023 00024 #include <utility> 00025 #include <vector> 00026 00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h" 00028 #include "net/instaweb/http/public/mock_url_fetcher.h" 00029 #include "net/instaweb/http/public/http_cache.h" 00030 #include "net/instaweb/http/public/logging_proto.h" 00031 #include "net/instaweb/http/public/logging_proto_impl.h" 00032 #include "net/instaweb/http/public/request_context.h" 00033 #include "net/instaweb/http/public/response_headers.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/rewrite_driver.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/server_context.h" 00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h" 00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" 00042 #include "net/instaweb/util/public/basictypes.h" 00043 #include "net/instaweb/util/public/md5_hasher.h" 00044 #include "net/instaweb/util/public/mem_file_system.h" 00045 #include "net/instaweb/util/public/mock_hasher.h" 00046 #include "net/instaweb/util/public/mock_message_handler.h" 00047 #include "net/instaweb/util/public/mock_property_page.h" 00049 #include "net/instaweb/util/public/mock_timer.h" 00050 #include "net/instaweb/util/public/property_cache.h" 00051 #include "net/instaweb/util/public/scoped_ptr.h" 00052 #include "net/instaweb/util/public/string.h" 00053 #include "net/instaweb/util/public/string_util.h" 00054 #include "net/instaweb/util/public/timer.h" 00055 #include "net/instaweb/util/public/url_segment_encoder.h" 00056 #include "pagespeed/kernel/http/content_type.h" 00057 00058 00059 namespace net_instaweb { 00060 00061 class AbstractLogRecord; 00062 class CountingUrlAsyncFetcher; 00063 class DelayCache; 00064 class HTTPValue; 00065 class Hasher; 00066 class HtmlWriterFilter; 00067 class LRUCache; 00068 class MessageHandler; 00069 class MockLogRecord; 00070 class MockScheduler; 00071 class RequestHeaders; 00072 class ResourceNamer; 00073 class RewriteFilter; 00074 class Statistics; 00075 class WaitUrlAsyncFetcher; 00076 00077 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc { 00078 protected: 00079 RewriteOptionsTestBase() { 00080 RewriteOptions::Initialize(); 00081 } 00082 ~RewriteOptionsTestBase() { 00083 RewriteOptions::Terminate(); 00084 } 00085 }; 00086 00087 class RewriteTestBase : public RewriteOptionsTestBase { 00088 public: 00089 static const char kTestData[]; 00090 00091 00092 00093 00094 00095 00096 enum ActiveServerFlag { 00097 kPrimary, 00098 kSecondary, 00099 }; 00100 00101 RewriteTestBase(); 00102 explicit RewriteTestBase(Statistics* statistics); 00103 00109 explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*, 00110 TestRewriteDriverFactory*> factories); 00111 virtual ~RewriteTestBase(); 00112 00113 virtual void SetUp(); 00114 virtual void TearDown(); 00115 00119 virtual bool AddBody() const { return false; } 00120 00130 virtual TestRewriteDriverFactory* MakeTestFactory(); 00131 00134 void AddRecompressImageFilters(); 00135 00137 void AddFilter(RewriteOptions::Filter filter); 00138 00140 void AddOtherFilter(RewriteOptions::Filter filter); 00141 00144 void AddRewriteFilter(RewriteFilter* filter); 00145 00148 void AddFetchOnlyRewriteFilter(RewriteFilter* filter); 00149 00152 void AddOtherRewriteFilter(RewriteFilter* filter); 00153 00156 void SetBaseUrlForFetch(const StringPiece& url); 00157 00159 void SetDummyRequestHeaders(); 00160 00163 void SetDownstreamCacheDirectives( 00164 StringPiece downstream_cache_location, 00165 StringPiece rebeaconing_key); 00166 00167 ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url); 00168 00173 Timer* timer() { return factory()->mock_timer(); } 00174 00176 void AppendDefaultHeaders(const ContentType& content_type, 00177 GoogleString* text); 00178 00181 void AppendDefaultHeaders(const ContentType& content_type, 00182 int64 original_content_length, 00183 GoogleString* text); 00184 00185 void ServeResourceFromManyContexts(const GoogleString& resource_url, 00186 const StringPiece& expected_content); 00187 00188 void ServeResourceFromManyContextsWithUA( 00189 const GoogleString& resource_url, 00190 const StringPiece& expected_content, 00191 const StringPiece& user_agent); 00192 00195 void ServeResourceFromNewContext( 00196 const GoogleString& resource_url, 00197 const StringPiece& expected_content); 00198 00202 virtual RewriteDriver* html_parse() { return rewrite_driver_; } 00203 00205 void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec, 00206 ResponseHeaders* response_headers); 00207 00211 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00212 const StringPiece& name, const StringPiece& ext, 00213 GoogleString* content); 00214 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00215 const StringPiece& name, const StringPiece& ext, 00216 GoogleString* content, ResponseHeaders* response); 00217 00218 bool FetchResourceUrl(const StringPiece& url, GoogleString* content, 00219 ResponseHeaders* response); 00220 bool FetchResourceUrl(const StringPiece& url, 00221 RequestHeaders* request_headers, 00222 GoogleString* content, 00223 ResponseHeaders* response_headers); 00224 bool FetchResourceUrl(const StringPiece& url, GoogleString* content); 00225 00227 bool TryFetchResource(const StringPiece& url); 00228 00233 void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers); 00234 00235 GoogleString CssLinkHref(const StringPiece& url) { 00236 return StrCat("<link rel=stylesheet href=", url, ">"); 00237 } 00238 00240 class CssLink { 00241 public: 00242 CssLink(const StringPiece& url, const StringPiece& content, 00243 const StringPiece& media, bool supply_mock); 00244 00246 class Vector : public std::vector<CssLink*> { 00247 public: 00248 ~Vector(); 00249 void Add(const StringPiece& url, const StringPiece& content, 00250 const StringPiece& media, bool supply_mock); 00251 }; 00252 00255 bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base, 00256 StringVector* segments, MessageHandler* handler); 00257 00258 GoogleString url_; 00259 GoogleString content_; 00260 GoogleString media_; 00261 bool supply_mock_; 00262 }; 00263 00265 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00266 StringVector* css_links); 00267 00269 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00270 CssLink::Vector* css_links); 00271 00273 void EncodePathAndLeaf(const StringPiece& filter_id, 00274 const StringPiece& hash, 00275 const StringVector& name_vector, 00276 const StringPiece& ext, 00277 ResourceNamer* namer); 00278 00279 StringVector MultiUrl(const StringPiece& url1) { 00280 StringVector v; 00281 v.push_back(url1.as_string()); 00282 return v; 00283 } 00284 00285 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) { 00286 StringVector v; 00287 v.push_back(url1.as_string()); 00288 v.push_back(url2.as_string()); 00289 return v; 00290 } 00291 00292 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00293 const StringPiece& url3) { 00294 StringVector v; 00295 v.push_back(url1.as_string()); 00296 v.push_back(url2.as_string()); 00297 v.push_back(url3.as_string()); 00298 return v; 00299 } 00300 00301 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00302 const StringPiece& url3, const StringPiece& url4) { 00303 StringVector v; 00304 v.push_back(url1.as_string()); 00305 v.push_back(url2.as_string()); 00306 v.push_back(url3.as_string()); 00307 v.push_back(url4.as_string()); 00308 return v; 00309 } 00310 00313 GoogleString Encode(const StringPiece& path, 00314 const StringPiece& filter_id, 00315 const StringPiece& hash, 00316 const StringPiece& name, 00317 const StringPiece& ext) { 00318 return Encode(path, filter_id, hash, MultiUrl(name), ext); 00319 } 00320 GoogleString Encode(const StringPiece& path, 00321 const StringPiece& filter_id, 00322 const StringPiece& hash, 00323 const StringVector& name_vector, 00324 const StringPiece& ext); 00325 00327 GoogleString EncodeNormal(const StringPiece& path, 00328 const StringPiece& filter_id, 00329 const StringPiece& hash, 00330 const StringPiece& name, 00331 const StringPiece& ext) { 00332 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext); 00333 } 00334 GoogleString EncodeNormal(const StringPiece& path, 00335 const StringPiece& filter_id, 00336 const StringPiece& hash, 00337 const StringVector& name_vector, 00338 const StringPiece& ext); 00339 00342 GoogleString EncodeWithBase(const StringPiece& base, 00343 const StringPiece& path, 00344 const StringPiece& filter_id, 00345 const StringPiece& hash, 00346 const StringPiece& name, 00347 const StringPiece& ext) { 00348 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext); 00349 } 00350 GoogleString EncodeWithBase(const StringPiece& base, 00351 const StringPiece& path, 00352 const StringPiece& filter_id, 00353 const StringPiece& hash, 00354 const StringVector& name_vector, 00355 const StringPiece& ext); 00356 00358 GoogleString AddOptionsToEncodedUrl(const StringPiece& url, 00359 const StringPiece& options); 00360 00365 static GoogleString ChangeSuffix( 00366 StringPiece old_url, bool append_new_suffix, 00367 StringPiece old_suffix, StringPiece new_suffix); 00368 00373 void SetupWaitFetcher(); 00374 void CallFetcherCallbacks(); 00375 void OtherCallFetcherCallbacks(); 00376 RewriteOptions* options() { return options_; } 00377 RewriteOptions* other_options() { return other_options_; } 00378 00380 void SetRewriteOptions(RewriteOptions* opts); 00381 00384 bool AddDomain(StringPiece domain); 00385 00388 bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain); 00389 00392 bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain); 00393 00396 bool AddShard(StringPiece domain, StringPiece shards); 00397 00399 void TestServeFiles(const ContentType* content_type, 00400 const StringPiece& filter_id, 00401 const StringPiece& rewritten_ext, 00402 const StringPiece& orig_name, 00403 const StringPiece& orig_content, 00404 const StringPiece& rewritten_name, 00405 const StringPiece& rewritten_content); 00406 00407 TestRewriteDriverFactory* factory() { return factory_.get(); } 00408 TestRewriteDriverFactory* other_factory() { return other_factory_.get(); } 00409 00410 void UseMd5Hasher() { 00411 server_context_->set_hasher(&md5_hasher_); 00412 server_context_->http_cache()->set_hasher(&md5_hasher_); 00413 other_server_context_->set_hasher(&md5_hasher_); 00414 other_server_context_->http_cache()->set_hasher(&md5_hasher_); 00415 } 00416 00417 00418 void SetDefaultLongCacheHeaders(const ContentType* content_type, 00419 ResponseHeaders* header) { 00420 server_context_->SetDefaultLongCacheHeaders(content_type, header); 00421 } 00422 00423 void SetFetchResponse(const StringPiece& url, 00424 const ResponseHeaders& response_header, 00425 const StringPiece& response_body) { 00426 mock_url_fetcher()->SetResponse(url, response_header, response_body); 00427 } 00428 00430 void SetResponseWithDefaultHeaders(const StringPiece& relative_url, 00431 const ContentType& content_type, 00432 const StringPiece& content, 00433 int64 ttl_sec); 00434 00437 bool LoadFile(const StringPiece& filename, GoogleString* contents); 00438 00440 void AddFileToMockFetcher(const StringPiece& url, 00441 const StringPiece& filename, 00442 const ContentType& content_type, int64 ttl_sec); 00443 00444 void AddToResponse(const StringPiece& url, 00445 const StringPiece& name, 00446 const StringPiece& value) { 00447 mock_url_fetcher()->AddToResponse(url, name, value); 00448 } 00449 00450 void SetFetchResponse404(const StringPiece& url); 00451 00452 void SetFetchFailOnUnexpected(bool fail) { 00453 mock_url_fetcher()->set_fail_on_unexpected(fail); 00454 } 00455 void FetcherUpdateDateHeaders() { 00456 mock_url_fetcher()->set_timer(timer()); 00457 mock_url_fetcher()->set_update_date_headers(true); 00458 } 00459 void ClearFetcherResponses() { mock_url_fetcher()->Clear(); } 00460 00461 virtual void ClearStats(); 00462 00465 void ClearRewriteDriver(); 00466 00467 MockUrlFetcher* mock_url_fetcher() { 00468 return &mock_url_fetcher_; 00469 } 00470 TestDistributedFetcher* test_distributed_fetcher() { 00471 return &test_distributed_fetcher_; 00472 } 00473 Hasher* hasher() { return server_context_->hasher(); } 00474 DelayCache* delay_cache() { return factory_->delay_cache(); } 00475 LRUCache* lru_cache() { return factory_->lru_cache(); } 00476 Statistics* statistics() { return factory_->statistics(); } 00477 MemFileSystem* file_system() { return factory_->mem_file_system(); } 00478 HTTPCache* http_cache() { return server_context_->http_cache(); } 00479 PropertyCache* page_property_cache() { 00480 return server_context_->page_property_cache(); 00481 } 00482 MockMessageHandler* message_handler() { 00483 return factory_->mock_message_handler(); 00484 } 00485 00493 RewriteDriver* rewrite_driver() { return rewrite_driver_; } 00494 RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; } 00495 00497 MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); } 00498 00499 int64 start_time_ms() const { return factory_->kStartTimeMs; } 00500 00501 bool ReadFile(const char* filename, GoogleString* contents) { 00502 return file_system()->ReadFile(filename, contents, message_handler()); 00503 } 00504 bool WriteFile(const char* filename, const StringPiece& contents) { 00505 return file_system()->WriteFile(filename, contents, message_handler()); 00506 } 00507 00508 ServerContext* server_context() { return server_context_; } 00509 ServerContext* other_server_context() { return other_server_context_; } 00510 CountingUrlAsyncFetcher* counting_url_async_fetcher() { 00511 return factory_->counting_url_async_fetcher(); 00512 } 00513 CountingUrlAsyncFetcher* counting_distributed_fetcher() { 00514 return factory_->counting_distributed_async_fetcher(); 00515 } 00516 void SetMockHashValue(const GoogleString& value) { 00517 factory_->mock_hasher()->set_hash_value(value); 00518 } 00519 00520 void SetCacheDelayUs(int64 delay_us); 00521 00526 RewriteDriver* MakeDriver(ServerContext* server_context, 00527 RewriteOptions* options); 00528 00530 GoogleString AbsolutifyUrl(const StringPiece& in); 00531 00536 void TestRetainExtraHeaders(const StringPiece& name, 00537 const StringPiece& filter_id, 00538 const StringPiece& ext); 00539 00543 const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const; 00544 00546 void SetUseTestUrlNamer(bool use_test_url_namer); 00547 00550 GoogleString EncodeCssName(const StringPiece& name, 00551 bool supports_webp, 00552 bool can_inline); 00553 00557 bool ReadIfCached(const ResourcePtr& resource); 00558 00563 void InitiateResourceRead(const ResourcePtr& resource); 00564 00568 HTTPCache::FindResult HttpBlockingFind( 00569 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out, 00570 ResponseHeaders* headers); 00571 00574 HTTPCache::FindResult HttpBlockingFindStatus( 00575 const GoogleString& key, HTTPCache* http_cache); 00576 00578 void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); } 00579 00581 void SetHtmlMimetype() { SetMimetype("text/html"); } 00582 00584 void SetMimetype(const StringPiece& mimetype); 00585 00588 void CheckFetchFromHttpCache( 00589 StringPiece url, 00590 StringPiece expected_contents, 00591 int64 expected_expiration_ms); 00592 00594 const PropertyCache::Cohort* SetupCohort( 00595 PropertyCache* cache, const GoogleString& cohort) { 00596 return factory()->SetupCohort(cache, cohort); 00597 } 00598 00601 void SetupSharedCache(); 00602 00604 MockPropertyPage* NewMockPage(const StringPiece& url, 00605 const StringPiece& options_signature_hash, 00606 UserAgentMatcher::DeviceType device_type) { 00607 return new MockPropertyPage( 00608 server_context_->thread_system(), 00609 server_context_->page_property_cache(), 00610 url, 00611 options_signature_hash, 00612 UserAgentMatcher::DeviceTypeSuffix(device_type)); 00613 } 00614 00615 MockPropertyPage* NewMockPage(const StringPiece& url) { 00616 return NewMockPage(url, "hash", UserAgentMatcher::kDesktop); 00617 } 00618 00620 void SetMockLogRecord(); 00621 00623 MockLogRecord* mock_log_record(); 00624 00626 GoogleString GetLazyloadScriptHtml(); 00627 GoogleString GetLazyloadPostscriptHtml(); 00628 00634 void SetCacheInvalidationTimestamp(); 00635 00638 void SetCacheInvalidationTimestampForUrl( 00639 StringPiece url, bool ignores_metadata_and_pcache); 00640 00644 void EnableCachePurge(); 00645 00646 protected: 00647 void Init(); 00648 00651 virtual RequestContextPtr CreateRequestContext(); 00652 00655 void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher, 00656 RewriteDriver* driver); 00657 00660 void PopulateDefaultHeaders(const ContentType& content_type, 00661 int64 original_content_length, 00662 ResponseHeaders* headers); 00663 00666 void SetActiveServer(ActiveServerFlag server_to_use); 00667 00670 void AdvanceTimeUs(int64 delay_ms); 00671 void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); } 00672 void SetTimeUs(int64 time_us); 00673 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); } 00674 00676 void AdjustTimeUsWithoutWakingAlarms(int64 time_us); 00677 00679 const RequestContext::TimingInfo& timing_info(); 00680 RequestContext::TimingInfo* mutable_timing_info(); 00681 00686 LoggingInfo* logging_info(); 00687 00689 const MetadataCacheInfo& metadata_cache_info() { 00690 return logging_info()->metadata_cache_info(); 00691 } 00692 00695 GoogleString AppliedRewriterStringFromLog(); 00696 00699 void VerifyRewriterInfoEntry(AbstractLogRecord* log_record, 00700 const GoogleString& id, 00701 int url_index, 00702 int rewriter_info_index, 00703 int rewriter_info_size, 00704 int url_list_size, 00705 const GoogleString& url); 00706 00708 void SetCurrentUserAgent(const StringPiece& user_agent) { 00709 current_user_agent_ = user_agent; 00710 } 00711 00712 GoogleString ExpectedNonce(); 00713 00716 MockUrlFetcher mock_url_fetcher_; 00717 TestDistributedFetcher test_distributed_fetcher_; 00718 scoped_ptr<Statistics> statistics_; 00719 00725 scoped_ptr<TestRewriteDriverFactory> factory_; 00726 scoped_ptr<TestRewriteDriverFactory> other_factory_; 00727 ServerContext* server_context_; 00728 RewriteDriver* rewrite_driver_; 00729 ServerContext* other_server_context_; 00730 RewriteDriver* other_rewrite_driver_; 00731 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_; 00732 ActiveServerFlag active_server_; 00733 bool use_managed_rewrite_drivers_; 00734 StringPiece current_user_agent_; 00735 MD5Hasher md5_hasher_; 00736 00737 RewriteOptions* options_; 00738 RewriteOptions* other_options_; 00739 UrlSegmentEncoder default_encoder_; 00740 ResponseHeaders response_headers_; 00741 const GoogleString kEtag0; 00742 uint64 expected_nonce_; 00743 }; 00744 00745 } 00746 00747 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_