Page Speed Optimization Libraries
1.7.30.4
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00020 00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00023 00024 #include <utility> 00025 #include <vector> 00026 00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h" 00028 #include "net/instaweb/http/public/mock_url_fetcher.h" 00029 #include "net/instaweb/http/public/http_cache.h" 00030 #include "net/instaweb/http/public/logging_proto.h" 00031 #include "net/instaweb/http/public/logging_proto_impl.h" 00032 #include "net/instaweb/http/public/request_context.h" 00033 #include "net/instaweb/http/public/response_headers.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/rewrite_driver.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/server_context.h" 00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h" 00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" 00042 #include "net/instaweb/util/public/basictypes.h" 00043 #include "net/instaweb/util/public/md5_hasher.h" 00044 #include "net/instaweb/util/public/mem_file_system.h" 00045 #include "net/instaweb/util/public/mock_hasher.h" 00046 #include "net/instaweb/util/public/mock_message_handler.h" 00047 #include "net/instaweb/util/public/mock_property_page.h" 00049 #include "net/instaweb/util/public/mock_timer.h" 00050 #include "net/instaweb/util/public/property_cache.h" 00051 #include "net/instaweb/util/public/scoped_ptr.h" 00052 #include "net/instaweb/util/public/string.h" 00053 #include "net/instaweb/util/public/string_util.h" 00054 #include "net/instaweb/util/public/timer.h" 00055 #include "net/instaweb/util/public/url_segment_encoder.h" 00056 #include "pagespeed/kernel/http/content_type.h" 00057 00058 00059 namespace net_instaweb { 00060 00061 class AbstractLogRecord; 00062 class CountingUrlAsyncFetcher; 00063 class DelayCache; 00064 class HTTPValue; 00065 class Hasher; 00066 class HtmlWriterFilter; 00067 class LRUCache; 00068 class MessageHandler; 00069 class MockLogRecord; 00070 class MockScheduler; 00071 class ProcessContext; 00072 class RequestHeaders; 00073 class ResourceNamer; 00074 class RewriteFilter; 00075 class Statistics; 00076 class WaitUrlAsyncFetcher; 00077 00078 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc { 00079 protected: 00080 RewriteOptionsTestBase() { 00081 RewriteOptions::Initialize(); 00082 } 00083 ~RewriteOptionsTestBase() { 00084 RewriteOptions::Terminate(); 00085 } 00086 }; 00087 00088 class RewriteTestBase : public RewriteOptionsTestBase { 00089 public: 00090 static const char kTestData[]; 00091 00093 static const char kConfiguredBeaconingKey[]; 00094 static const char kWrongBeaconingKey[]; 00095 00102 enum ActiveServerFlag { 00103 kPrimary, 00104 kSecondary, 00105 }; 00106 00107 RewriteTestBase(); 00108 explicit RewriteTestBase(Statistics* statistics); 00109 00115 explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*, 00116 TestRewriteDriverFactory*> factories); 00117 virtual ~RewriteTestBase(); 00118 00119 virtual void SetUp(); 00120 virtual void TearDown(); 00121 00125 virtual bool AddBody() const { return false; } 00126 00136 virtual TestRewriteDriverFactory* MakeTestFactory(); 00137 00140 void AddRecompressImageFilters(); 00141 00143 void AddFilter(RewriteOptions::Filter filter); 00144 00146 void AddOtherFilter(RewriteOptions::Filter filter); 00147 00150 void AddRewriteFilter(RewriteFilter* filter); 00151 00154 void AddFetchOnlyRewriteFilter(RewriteFilter* filter); 00155 00158 void AddOtherRewriteFilter(RewriteFilter* filter); 00159 00162 void SetBaseUrlForFetch(const StringPiece& url); 00163 00165 void SetDummyRequestHeaders(); 00166 00169 void SetDownstreamCacheDirectives( 00170 StringPiece downstream_cache_location, 00171 StringPiece rebeaconing_key); 00172 00174 void SetShouldBeaconHeader(StringPiece rebeaconing_key); 00175 00176 ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url); 00177 00182 Timer* timer() { return factory()->mock_timer(); } 00183 00185 void AppendDefaultHeaders(const ContentType& content_type, 00186 GoogleString* text); 00187 00190 void AppendDefaultHeaders(const ContentType& content_type, 00191 int64 original_content_length, 00192 GoogleString* text); 00193 00194 void ServeResourceFromManyContexts(const GoogleString& resource_url, 00195 const StringPiece& expected_content); 00196 00197 void ServeResourceFromManyContextsWithUA( 00198 const GoogleString& resource_url, 00199 const StringPiece& expected_content, 00200 const StringPiece& user_agent); 00201 00204 void ServeResourceFromNewContext( 00205 const GoogleString& resource_url, 00206 const StringPiece& expected_content); 00207 00211 virtual RewriteDriver* html_parse() { return rewrite_driver_; } 00212 00214 void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec, 00215 ResponseHeaders* response_headers); 00216 00220 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00221 const StringPiece& name, const StringPiece& ext, 00222 GoogleString* content); 00223 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00224 const StringPiece& name, const StringPiece& ext, 00225 GoogleString* content, ResponseHeaders* response); 00226 00227 bool FetchResourceUrl(const StringPiece& url, GoogleString* content, 00228 ResponseHeaders* response); 00229 bool FetchResourceUrl(const StringPiece& url, 00230 RequestHeaders* request_headers, 00231 GoogleString* content, 00232 ResponseHeaders* response_headers); 00233 bool FetchResourceUrl(const StringPiece& url, GoogleString* content); 00234 00236 bool TryFetchResource(const StringPiece& url); 00237 00242 void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers); 00243 00244 GoogleString CssLinkHref(const StringPiece& url) { 00245 return StrCat("<link rel=stylesheet href=", url, ">"); 00246 } 00247 00249 class CssLink { 00250 public: 00251 CssLink(const StringPiece& url, const StringPiece& content, 00252 const StringPiece& media, bool supply_mock); 00253 00255 class Vector : public std::vector<CssLink*> { 00256 public: 00257 ~Vector(); 00258 void Add(const StringPiece& url, const StringPiece& content, 00259 const StringPiece& media, bool supply_mock); 00260 }; 00261 00264 bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base, 00265 StringVector* segments, MessageHandler* handler); 00266 00267 GoogleString url_; 00268 GoogleString content_; 00269 GoogleString media_; 00270 bool supply_mock_; 00271 }; 00272 00274 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00275 StringVector* css_links); 00276 00278 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00279 CssLink::Vector* css_links); 00280 00282 void EncodePathAndLeaf(const StringPiece& filter_id, 00283 const StringPiece& hash, 00284 const StringVector& name_vector, 00285 const StringPiece& ext, 00286 ResourceNamer* namer); 00287 00288 StringVector MultiUrl(const StringPiece& url1) { 00289 StringVector v; 00290 v.push_back(url1.as_string()); 00291 return v; 00292 } 00293 00294 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) { 00295 StringVector v; 00296 v.push_back(url1.as_string()); 00297 v.push_back(url2.as_string()); 00298 return v; 00299 } 00300 00301 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00302 const StringPiece& url3) { 00303 StringVector v; 00304 v.push_back(url1.as_string()); 00305 v.push_back(url2.as_string()); 00306 v.push_back(url3.as_string()); 00307 return v; 00308 } 00309 00310 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00311 const StringPiece& url3, const StringPiece& url4) { 00312 StringVector v; 00313 v.push_back(url1.as_string()); 00314 v.push_back(url2.as_string()); 00315 v.push_back(url3.as_string()); 00316 v.push_back(url4.as_string()); 00317 return v; 00318 } 00319 00322 GoogleString Encode(const StringPiece& path, 00323 const StringPiece& filter_id, 00324 const StringPiece& hash, 00325 const StringPiece& name, 00326 const StringPiece& ext) { 00327 return Encode(path, filter_id, hash, MultiUrl(name), ext); 00328 } 00329 GoogleString Encode(const StringPiece& path, 00330 const StringPiece& filter_id, 00331 const StringPiece& hash, 00332 const StringVector& name_vector, 00333 const StringPiece& ext); 00334 00336 GoogleString EncodeNormal(const StringPiece& path, 00337 const StringPiece& filter_id, 00338 const StringPiece& hash, 00339 const StringPiece& name, 00340 const StringPiece& ext) { 00341 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext); 00342 } 00343 GoogleString EncodeNormal(const StringPiece& path, 00344 const StringPiece& filter_id, 00345 const StringPiece& hash, 00346 const StringVector& name_vector, 00347 const StringPiece& ext); 00348 00351 GoogleString EncodeWithBase(const StringPiece& base, 00352 const StringPiece& path, 00353 const StringPiece& filter_id, 00354 const StringPiece& hash, 00355 const StringPiece& name, 00356 const StringPiece& ext) { 00357 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext); 00358 } 00359 GoogleString EncodeWithBase(const StringPiece& base, 00360 const StringPiece& path, 00361 const StringPiece& filter_id, 00362 const StringPiece& hash, 00363 const StringVector& name_vector, 00364 const StringPiece& ext); 00365 00367 GoogleString AddOptionsToEncodedUrl(const StringPiece& url, 00368 const StringPiece& options); 00369 00374 static GoogleString ChangeSuffix( 00375 StringPiece old_url, bool append_new_suffix, 00376 StringPiece old_suffix, StringPiece new_suffix); 00377 00382 void SetupWaitFetcher(); 00383 void CallFetcherCallbacks(); 00384 void OtherCallFetcherCallbacks(); 00385 RewriteOptions* options() { return options_; } 00386 RewriteOptions* other_options() { return other_options_; } 00387 00389 void SetRewriteOptions(RewriteOptions* opts); 00390 00393 bool AddDomain(StringPiece domain); 00394 00397 bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain); 00398 00401 bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain); 00402 00405 bool AddShard(StringPiece domain, StringPiece shards); 00406 00408 void TestServeFiles(const ContentType* content_type, 00409 const StringPiece& filter_id, 00410 const StringPiece& rewritten_ext, 00411 const StringPiece& orig_name, 00412 const StringPiece& orig_content, 00413 const StringPiece& rewritten_name, 00414 const StringPiece& rewritten_content); 00415 00416 TestRewriteDriverFactory* factory() { return factory_.get(); } 00417 TestRewriteDriverFactory* other_factory() { return other_factory_.get(); } 00418 00419 void UseMd5Hasher() { 00420 server_context_->set_hasher(&md5_hasher_); 00421 server_context_->http_cache()->set_hasher(&md5_hasher_); 00422 other_server_context_->set_hasher(&md5_hasher_); 00423 other_server_context_->http_cache()->set_hasher(&md5_hasher_); 00424 } 00425 00426 00427 void SetDefaultLongCacheHeaders(const ContentType* content_type, 00428 ResponseHeaders* header) { 00429 server_context_->SetDefaultLongCacheHeaders(content_type, header); 00430 } 00431 00432 void SetFetchResponse(const StringPiece& url, 00433 const ResponseHeaders& response_header, 00434 const StringPiece& response_body) { 00435 mock_url_fetcher()->SetResponse(url, response_header, response_body); 00436 } 00437 00439 void SetResponseWithDefaultHeaders(const StringPiece& relative_url, 00440 const ContentType& content_type, 00441 const StringPiece& content, 00442 int64 ttl_sec); 00443 00446 bool LoadFile(const StringPiece& filename, GoogleString* contents); 00447 00449 void AddFileToMockFetcher(const StringPiece& url, 00450 const StringPiece& filename, 00451 const ContentType& content_type, int64 ttl_sec); 00452 00453 void AddToResponse(const StringPiece& url, 00454 const StringPiece& name, 00455 const StringPiece& value) { 00456 mock_url_fetcher()->AddToResponse(url, name, value); 00457 } 00458 00459 void SetFetchResponse404(const StringPiece& url); 00460 00461 void SetFetchFailOnUnexpected(bool fail) { 00462 mock_url_fetcher()->set_fail_on_unexpected(fail); 00463 } 00464 void FetcherUpdateDateHeaders() { 00465 mock_url_fetcher()->set_timer(timer()); 00466 mock_url_fetcher()->set_update_date_headers(true); 00467 } 00468 void ClearFetcherResponses() { mock_url_fetcher()->Clear(); } 00469 00470 virtual void ClearStats(); 00471 00474 void ClearRewriteDriver(); 00475 00476 MockUrlFetcher* mock_url_fetcher() { 00477 return &mock_url_fetcher_; 00478 } 00479 TestDistributedFetcher* test_distributed_fetcher() { 00480 return &test_distributed_fetcher_; 00481 } 00482 Hasher* hasher() { return server_context_->hasher(); } 00483 DelayCache* delay_cache() { return factory_->delay_cache(); } 00484 LRUCache* lru_cache() { return factory_->lru_cache(); } 00485 Statistics* statistics() { return factory_->statistics(); } 00486 MemFileSystem* file_system() { return factory_->mem_file_system(); } 00487 HTTPCache* http_cache() { return server_context_->http_cache(); } 00488 PropertyCache* page_property_cache() { 00489 return server_context_->page_property_cache(); 00490 } 00491 MockMessageHandler* message_handler() { 00492 return factory_->mock_message_handler(); 00493 } 00494 00502 RewriteDriver* rewrite_driver() { return rewrite_driver_; } 00503 RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; } 00504 00506 MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); } 00507 00508 int64 start_time_ms() const { return factory_->kStartTimeMs; } 00509 00510 bool ReadFile(const char* filename, GoogleString* contents) { 00511 return file_system()->ReadFile(filename, contents, message_handler()); 00512 } 00513 bool WriteFile(const char* filename, const StringPiece& contents) { 00514 return file_system()->WriteFile(filename, contents, message_handler()); 00515 } 00516 00517 ServerContext* server_context() { return server_context_; } 00518 ServerContext* other_server_context() { return other_server_context_; } 00519 CountingUrlAsyncFetcher* counting_url_async_fetcher() { 00520 return factory_->counting_url_async_fetcher(); 00521 } 00522 CountingUrlAsyncFetcher* counting_distributed_fetcher() { 00523 return factory_->counting_distributed_async_fetcher(); 00524 } 00525 void SetMockHashValue(const GoogleString& value) { 00526 factory_->mock_hasher()->set_hash_value(value); 00527 } 00528 00529 void SetCacheDelayUs(int64 delay_us); 00530 00535 RewriteDriver* MakeDriver(ServerContext* server_context, 00536 RewriteOptions* options); 00537 00539 GoogleString AbsolutifyUrl(const StringPiece& in); 00540 00545 void TestRetainExtraHeaders(const StringPiece& name, 00546 const StringPiece& filter_id, 00547 const StringPiece& ext); 00548 00552 const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const; 00553 00555 void SetUseTestUrlNamer(bool use_test_url_namer); 00556 00559 GoogleString EncodeCssName(const StringPiece& name, 00560 bool supports_webp, 00561 bool can_inline); 00562 00566 bool ReadIfCached(const ResourcePtr& resource); 00567 00572 void InitiateResourceRead(const ResourcePtr& resource); 00573 00577 HTTPCache::FindResult HttpBlockingFind( 00578 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out, 00579 ResponseHeaders* headers); 00580 00583 HTTPCache::FindResult HttpBlockingFindStatus( 00584 const GoogleString& key, HTTPCache* http_cache); 00585 00587 void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); } 00588 00590 void SetHtmlMimetype() { SetMimetype("text/html"); } 00591 00593 void SetMimetype(const StringPiece& mimetype); 00594 00597 void CheckFetchFromHttpCache( 00598 StringPiece url, 00599 StringPiece expected_contents, 00600 int64 expected_expiration_ms); 00601 00603 const PropertyCache::Cohort* SetupCohort( 00604 PropertyCache* cache, const GoogleString& cohort) { 00605 return factory()->SetupCohort(cache, cohort); 00606 } 00607 00610 void SetupSharedCache(); 00611 00613 MockPropertyPage* NewMockPage(const StringPiece& url, 00614 const StringPiece& options_signature_hash, 00615 UserAgentMatcher::DeviceType device_type) { 00616 return new MockPropertyPage( 00617 server_context_->thread_system(), 00618 server_context_->page_property_cache(), 00619 url, 00620 options_signature_hash, 00621 UserAgentMatcher::DeviceTypeSuffix(device_type)); 00622 } 00623 00624 MockPropertyPage* NewMockPage(const StringPiece& url) { 00625 return NewMockPage(url, "hash", UserAgentMatcher::kDesktop); 00626 } 00627 00629 void SetMockLogRecord(); 00630 00632 MockLogRecord* mock_log_record(); 00633 00635 GoogleString GetLazyloadScriptHtml(); 00636 GoogleString GetLazyloadPostscriptHtml(); 00637 00643 void SetCacheInvalidationTimestamp(); 00644 00647 void SetCacheInvalidationTimestampForUrl( 00648 StringPiece url, bool ignores_metadata_and_pcache); 00649 00653 void EnableCachePurge(); 00654 00657 static const ProcessContext& process_context(); 00658 00659 protected: 00660 void Init(); 00661 00664 virtual RequestContextPtr CreateRequestContext(); 00665 00668 void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher, 00669 RewriteDriver* driver); 00670 00673 void PopulateDefaultHeaders(const ContentType& content_type, 00674 int64 original_content_length, 00675 ResponseHeaders* headers); 00676 00679 void SetActiveServer(ActiveServerFlag server_to_use); 00680 00683 void AdvanceTimeUs(int64 delay_ms); 00684 void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); } 00685 void SetTimeUs(int64 time_us); 00686 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); } 00687 00689 void AdjustTimeUsWithoutWakingAlarms(int64 time_us); 00690 00692 const RequestContext::TimingInfo& timing_info(); 00693 RequestContext::TimingInfo* mutable_timing_info(); 00694 00699 LoggingInfo* logging_info(); 00700 00702 const MetadataCacheInfo& metadata_cache_info() { 00703 return logging_info()->metadata_cache_info(); 00704 } 00705 00708 GoogleString AppliedRewriterStringFromLog(); 00709 00712 void VerifyRewriterInfoEntry(AbstractLogRecord* log_record, 00713 const GoogleString& id, 00714 int url_index, 00715 int rewriter_info_index, 00716 int rewriter_info_size, 00717 int url_list_size, 00718 const GoogleString& url); 00719 00721 void SetCurrentUserAgent(const StringPiece& user_agent) { 00722 current_user_agent_ = user_agent; 00723 } 00724 00725 GoogleString ExpectedNonce(); 00726 00729 GoogleString HttpCacheKey(StringPiece url) { 00730 return HTTPCache::CompositeKey(url, rewrite_driver_->CacheFragment()); 00731 } 00732 00735 MockUrlFetcher mock_url_fetcher_; 00736 TestDistributedFetcher test_distributed_fetcher_; 00737 scoped_ptr<Statistics> statistics_; 00738 00744 scoped_ptr<TestRewriteDriverFactory> factory_; 00745 scoped_ptr<TestRewriteDriverFactory> other_factory_; 00746 ServerContext* server_context_; 00747 RewriteDriver* rewrite_driver_; 00748 ServerContext* other_server_context_; 00749 RewriteDriver* other_rewrite_driver_; 00750 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_; 00751 ActiveServerFlag active_server_; 00752 bool use_managed_rewrite_drivers_; 00753 StringPiece current_user_agent_; 00754 MD5Hasher md5_hasher_; 00755 00756 RewriteOptions* options_; 00757 RewriteOptions* other_options_; 00758 UrlSegmentEncoder default_encoder_; 00759 ResponseHeaders response_headers_; 00760 const GoogleString kEtag0; 00761 uint64 expected_nonce_; 00762 }; 00763 00764 } 00765 00766 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_