Page Speed Optimization Libraries
1.8.31.2
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00020 00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00023 00024 #include <utility> 00025 #include <vector> 00026 00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h" 00028 #include "net/instaweb/http/public/mock_url_fetcher.h" 00029 #include "net/instaweb/http/public/http_cache.h" 00030 #include "net/instaweb/http/public/logging_proto.h" 00031 #include "net/instaweb/http/public/logging_proto_impl.h" 00032 #include "net/instaweb/http/public/request_context.h" 00033 #include "net/instaweb/http/public/response_headers.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/rewrite_driver.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/server_context.h" 00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h" 00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" 00042 #include "net/instaweb/util/public/basictypes.h" 00043 #include "net/instaweb/util/public/md5_hasher.h" 00044 #include "net/instaweb/util/public/mem_file_system.h" 00045 #include "net/instaweb/util/public/mock_hasher.h" 00046 #include "net/instaweb/util/public/mock_message_handler.h" 00047 #include "net/instaweb/util/public/mock_property_page.h" 00049 #include "net/instaweb/util/public/mock_timer.h" 00050 #include "net/instaweb/util/public/property_cache.h" 00051 #include "net/instaweb/util/public/scoped_ptr.h" 00052 #include "net/instaweb/util/public/string.h" 00053 #include "net/instaweb/util/public/string_util.h" 00054 #include "net/instaweb/util/public/timer.h" 00055 #include "net/instaweb/util/public/url_segment_encoder.h" 00056 #include "pagespeed/kernel/http/content_type.h" 00057 00058 00059 namespace net_instaweb { 00060 00061 class AbstractLogRecord; 00062 class CountingUrlAsyncFetcher; 00063 class DelayCache; 00064 class HTTPValue; 00065 class Hasher; 00066 class HtmlWriterFilter; 00067 class LRUCache; 00068 class MessageHandler; 00069 class MockLogRecord; 00070 class MockScheduler; 00071 class ProcessContext; 00072 class RequestHeaders; 00073 class ResourceNamer; 00074 class RewriteFilter; 00075 class Statistics; 00076 class WaitUrlAsyncFetcher; 00077 00078 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc { 00079 protected: 00080 RewriteOptionsTestBase() { 00081 RewriteOptions::Initialize(); 00082 } 00083 ~RewriteOptionsTestBase() { 00084 RewriteOptions::Terminate(); 00085 } 00086 }; 00087 00088 class RewriteTestBase : public RewriteOptionsTestBase { 00089 public: 00090 static const char kTestData[]; 00091 00093 static const char kConfiguredBeaconingKey[]; 00094 static const char kWrongBeaconingKey[]; 00095 00102 enum ActiveServerFlag { 00103 kPrimary, 00104 kSecondary, 00105 }; 00106 00107 RewriteTestBase(); 00108 explicit RewriteTestBase(Statistics* statistics); 00109 00115 explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*, 00116 TestRewriteDriverFactory*> factories); 00117 virtual ~RewriteTestBase(); 00118 00119 virtual void SetUp(); 00120 virtual void TearDown(); 00121 00125 virtual bool AddBody() const { return false; } 00126 00136 virtual TestRewriteDriverFactory* MakeTestFactory(); 00137 00140 void AddRecompressImageFilters(); 00141 00143 void AddFilter(RewriteOptions::Filter filter); 00144 00146 void AddOtherFilter(RewriteOptions::Filter filter); 00147 00150 void AddRewriteFilter(RewriteFilter* filter); 00151 00154 void AddFetchOnlyRewriteFilter(RewriteFilter* filter); 00155 00158 void AddOtherRewriteFilter(RewriteFilter* filter); 00159 00162 void SetBaseUrlForFetch(const StringPiece& url); 00163 00165 void SetDummyRequestHeaders(); 00166 00169 void SetDownstreamCacheDirectives( 00170 StringPiece downstream_cache_purge_method, 00171 StringPiece downstream_cache_purge_location_prefix, 00172 StringPiece rebeaconing_key); 00173 00175 void SetShouldBeaconHeader(StringPiece rebeaconing_key); 00176 00177 ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url); 00178 00183 Timer* timer() { return factory()->mock_timer(); } 00184 00186 void AppendDefaultHeaders(const ContentType& content_type, 00187 GoogleString* text); 00188 00191 void AppendDefaultHeaders(const ContentType& content_type, 00192 int64 original_content_length, 00193 GoogleString* text); 00194 00195 void ServeResourceFromManyContexts(const GoogleString& resource_url, 00196 const StringPiece& expected_content); 00197 00198 void ServeResourceFromManyContextsWithUA( 00199 const GoogleString& resource_url, 00200 const StringPiece& expected_content, 00201 const StringPiece& user_agent); 00202 00205 void ServeResourceFromNewContext( 00206 const GoogleString& resource_url, 00207 const StringPiece& expected_content); 00208 00212 virtual RewriteDriver* html_parse() { return rewrite_driver_; } 00213 00215 void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec, 00216 ResponseHeaders* response_headers); 00217 00221 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00222 const StringPiece& name, const StringPiece& ext, 00223 GoogleString* content); 00224 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00225 const StringPiece& name, const StringPiece& ext, 00226 GoogleString* content, ResponseHeaders* response); 00227 00228 bool FetchResourceUrl(const StringPiece& url, GoogleString* content, 00229 ResponseHeaders* response); 00230 bool FetchResourceUrl(const StringPiece& url, 00231 RequestHeaders* request_headers, 00232 GoogleString* content, 00233 ResponseHeaders* response_headers); 00234 bool FetchResourceUrl(const StringPiece& url, GoogleString* content); 00235 00237 bool TryFetchResource(const StringPiece& url); 00238 00243 void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers); 00244 00245 GoogleString CssLinkHref(const StringPiece& url) { 00246 return StrCat("<link rel=stylesheet href=", url, ">"); 00247 } 00248 00250 class CssLink { 00251 public: 00252 CssLink(const StringPiece& url, const StringPiece& content, 00253 const StringPiece& media, bool supply_mock); 00254 00256 class Vector : public std::vector<CssLink*> { 00257 public: 00258 ~Vector(); 00259 void Add(const StringPiece& url, const StringPiece& content, 00260 const StringPiece& media, bool supply_mock); 00261 }; 00262 00265 bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base, 00266 StringVector* segments, MessageHandler* handler); 00267 00268 GoogleString url_; 00269 GoogleString content_; 00270 GoogleString media_; 00271 bool supply_mock_; 00272 }; 00273 00275 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00276 StringVector* css_links); 00277 00279 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00280 CssLink::Vector* css_links); 00281 00283 void EncodePathAndLeaf(const StringPiece& filter_id, 00284 const StringPiece& hash, 00285 const StringVector& name_vector, 00286 const StringPiece& ext, 00287 ResourceNamer* namer); 00288 00289 StringVector MultiUrl(const StringPiece& url1) { 00290 StringVector v; 00291 v.push_back(url1.as_string()); 00292 return v; 00293 } 00294 00295 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) { 00296 StringVector v; 00297 v.push_back(url1.as_string()); 00298 v.push_back(url2.as_string()); 00299 return v; 00300 } 00301 00302 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00303 const StringPiece& url3) { 00304 StringVector v; 00305 v.push_back(url1.as_string()); 00306 v.push_back(url2.as_string()); 00307 v.push_back(url3.as_string()); 00308 return v; 00309 } 00310 00311 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00312 const StringPiece& url3, const StringPiece& url4) { 00313 StringVector v; 00314 v.push_back(url1.as_string()); 00315 v.push_back(url2.as_string()); 00316 v.push_back(url3.as_string()); 00317 v.push_back(url4.as_string()); 00318 return v; 00319 } 00320 00323 GoogleString Encode(const StringPiece& path, 00324 const StringPiece& filter_id, 00325 const StringPiece& hash, 00326 const StringPiece& name, 00327 const StringPiece& ext) { 00328 return Encode(path, filter_id, hash, MultiUrl(name), ext); 00329 } 00330 GoogleString Encode(const StringPiece& path, 00331 const StringPiece& filter_id, 00332 const StringPiece& hash, 00333 const StringVector& name_vector, 00334 const StringPiece& ext); 00335 00337 GoogleString EncodeNormal(const StringPiece& path, 00338 const StringPiece& filter_id, 00339 const StringPiece& hash, 00340 const StringPiece& name, 00341 const StringPiece& ext) { 00342 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext); 00343 } 00344 GoogleString EncodeNormal(const StringPiece& path, 00345 const StringPiece& filter_id, 00346 const StringPiece& hash, 00347 const StringVector& name_vector, 00348 const StringPiece& ext); 00349 00352 GoogleString EncodeWithBase(const StringPiece& base, 00353 const StringPiece& path, 00354 const StringPiece& filter_id, 00355 const StringPiece& hash, 00356 const StringPiece& name, 00357 const StringPiece& ext) { 00358 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext); 00359 } 00360 GoogleString EncodeWithBase(const StringPiece& base, 00361 const StringPiece& path, 00362 const StringPiece& filter_id, 00363 const StringPiece& hash, 00364 const StringVector& name_vector, 00365 const StringPiece& ext); 00366 00368 GoogleString AddOptionsToEncodedUrl(const StringPiece& url, 00369 const StringPiece& options); 00370 00375 static GoogleString ChangeSuffix( 00376 StringPiece old_url, bool append_new_suffix, 00377 StringPiece old_suffix, StringPiece new_suffix); 00378 00383 void SetupWaitFetcher(); 00384 void CallFetcherCallbacks(); 00385 void OtherCallFetcherCallbacks(); 00386 RewriteOptions* options() { return options_; } 00387 RewriteOptions* other_options() { return other_options_; } 00388 00390 void SetRewriteOptions(RewriteOptions* opts); 00391 00394 bool AddDomain(StringPiece domain); 00395 00398 bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain); 00399 00402 bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain); 00403 00406 bool AddShard(StringPiece domain, StringPiece shards); 00407 00409 void TestServeFiles(const ContentType* content_type, 00410 const StringPiece& filter_id, 00411 const StringPiece& rewritten_ext, 00412 const StringPiece& orig_name, 00413 const StringPiece& orig_content, 00414 const StringPiece& rewritten_name, 00415 const StringPiece& rewritten_content); 00416 00417 TestRewriteDriverFactory* factory() { return factory_.get(); } 00418 TestRewriteDriverFactory* other_factory() { return other_factory_.get(); } 00419 00420 void UseMd5Hasher() { 00421 server_context_->set_hasher(&md5_hasher_); 00422 server_context_->http_cache()->set_hasher(&md5_hasher_); 00423 other_server_context_->set_hasher(&md5_hasher_); 00424 other_server_context_->http_cache()->set_hasher(&md5_hasher_); 00425 } 00426 00427 00428 void SetDefaultLongCacheHeaders(const ContentType* content_type, 00429 ResponseHeaders* header) { 00430 server_context_->SetDefaultLongCacheHeaders(content_type, header); 00431 } 00432 00433 void SetFetchResponse(const StringPiece& url, 00434 const ResponseHeaders& response_header, 00435 const StringPiece& response_body) { 00436 mock_url_fetcher()->SetResponse(url, response_header, response_body); 00437 } 00438 00440 void SetResponseWithDefaultHeaders(const StringPiece& relative_url, 00441 const ContentType& content_type, 00442 const StringPiece& content, 00443 int64 ttl_sec); 00444 00447 bool LoadFile(const StringPiece& filename, GoogleString* contents); 00448 00450 void AddFileToMockFetcher(const StringPiece& url, 00451 const StringPiece& filename, 00452 const ContentType& content_type, int64 ttl_sec); 00453 00454 void AddToResponse(const StringPiece& url, 00455 const StringPiece& name, 00456 const StringPiece& value) { 00457 mock_url_fetcher()->AddToResponse(url, name, value); 00458 } 00459 00460 void SetFetchResponse404(const StringPiece& url); 00461 00462 void SetFetchFailOnUnexpected(bool fail) { 00463 mock_url_fetcher()->set_fail_on_unexpected(fail); 00464 } 00465 void FetcherUpdateDateHeaders() { 00466 mock_url_fetcher()->set_timer(timer()); 00467 mock_url_fetcher()->set_update_date_headers(true); 00468 } 00469 void ClearFetcherResponses() { mock_url_fetcher()->Clear(); } 00470 00471 virtual void ClearStats(); 00472 00475 void ClearRewriteDriver(); 00476 00477 MockUrlFetcher* mock_url_fetcher() { 00478 return &mock_url_fetcher_; 00479 } 00480 TestDistributedFetcher* test_distributed_fetcher() { 00481 return &test_distributed_fetcher_; 00482 } 00483 Hasher* hasher() { return server_context_->hasher(); } 00484 DelayCache* delay_cache() { return factory_->delay_cache(); } 00485 LRUCache* lru_cache() { return factory_->lru_cache(); } 00486 Statistics* statistics() { return factory_->statistics(); } 00487 MemFileSystem* file_system() { return factory_->mem_file_system(); } 00488 HTTPCache* http_cache() { return server_context_->http_cache(); } 00489 PropertyCache* page_property_cache() { 00490 return server_context_->page_property_cache(); 00491 } 00492 MockMessageHandler* message_handler() { 00493 return factory_->mock_message_handler(); 00494 } 00495 00503 RewriteDriver* rewrite_driver() { return rewrite_driver_; } 00504 RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; } 00505 00507 MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); } 00508 00509 int64 start_time_ms() const { return factory_->kStartTimeMs; } 00510 00511 bool ReadFile(const char* filename, GoogleString* contents) { 00512 return file_system()->ReadFile(filename, contents, message_handler()); 00513 } 00514 bool WriteFile(const char* filename, const StringPiece& contents) { 00515 return file_system()->WriteFile(filename, contents, message_handler()); 00516 } 00517 00518 ServerContext* server_context() { return server_context_; } 00519 ServerContext* other_server_context() { return other_server_context_; } 00520 CountingUrlAsyncFetcher* counting_url_async_fetcher() { 00521 return factory_->counting_url_async_fetcher(); 00522 } 00523 CountingUrlAsyncFetcher* counting_distributed_fetcher() { 00524 return factory_->counting_distributed_async_fetcher(); 00525 } 00526 void SetMockHashValue(const GoogleString& value) { 00527 factory_->mock_hasher()->set_hash_value(value); 00528 } 00529 00530 void SetCacheDelayUs(int64 delay_us); 00531 00536 RewriteDriver* MakeDriver(ServerContext* server_context, 00537 RewriteOptions* options); 00538 00540 GoogleString AbsolutifyUrl(const StringPiece& in); 00541 00546 void TestRetainExtraHeaders(const StringPiece& name, 00547 const StringPiece& filter_id, 00548 const StringPiece& ext); 00549 00553 const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const; 00554 00556 void SetUseTestUrlNamer(bool use_test_url_namer); 00557 00560 GoogleString EncodeCssName(const StringPiece& name, 00561 bool supports_webp, 00562 bool can_inline); 00563 00567 bool ReadIfCached(const ResourcePtr& resource); 00568 00573 void InitiateResourceRead(const ResourcePtr& resource); 00574 00578 HTTPCache::FindResult HttpBlockingFind( 00579 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out, 00580 ResponseHeaders* headers); 00581 00584 HTTPCache::FindResult HttpBlockingFindStatus( 00585 const GoogleString& key, HTTPCache* http_cache); 00586 00588 void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); } 00589 00591 void SetHtmlMimetype() { SetMimetype("text/html"); } 00592 00594 void SetMimetype(const StringPiece& mimetype); 00595 00598 void CheckFetchFromHttpCache( 00599 StringPiece url, 00600 StringPiece expected_contents, 00601 int64 expected_expiration_ms); 00602 00604 const PropertyCache::Cohort* SetupCohort( 00605 PropertyCache* cache, const GoogleString& cohort) { 00606 return factory()->SetupCohort(cache, cohort); 00607 } 00608 00611 void SetupSharedCache(); 00612 00614 MockPropertyPage* NewMockPage(const StringPiece& url, 00615 const StringPiece& options_signature_hash, 00616 UserAgentMatcher::DeviceType device_type) { 00617 return new MockPropertyPage( 00618 server_context_->thread_system(), 00619 server_context_->page_property_cache(), 00620 url, 00621 options_signature_hash, 00622 UserAgentMatcher::DeviceTypeSuffix(device_type)); 00623 } 00624 00625 MockPropertyPage* NewMockPage(const StringPiece& url) { 00626 return NewMockPage(url, "hash", UserAgentMatcher::kDesktop); 00627 } 00628 00630 void SetMockLogRecord(); 00631 00633 MockLogRecord* mock_log_record(); 00634 00636 GoogleString GetLazyloadScriptHtml(); 00637 GoogleString GetLazyloadPostscriptHtml(); 00638 00644 void SetCacheInvalidationTimestamp(); 00645 00648 void SetCacheInvalidationTimestampForUrl( 00649 StringPiece url, bool ignores_metadata_and_pcache); 00650 00654 void EnableCachePurge(); 00655 00658 static const ProcessContext& process_context(); 00659 00660 protected: 00661 void Init(); 00662 00665 virtual RequestContextPtr CreateRequestContext(); 00666 00669 void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher, 00670 RewriteDriver* driver); 00671 00674 void PopulateDefaultHeaders(const ContentType& content_type, 00675 int64 original_content_length, 00676 ResponseHeaders* headers); 00677 00680 void SetActiveServer(ActiveServerFlag server_to_use); 00681 00684 void AdvanceTimeUs(int64 delay_ms); 00685 void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); } 00686 void SetTimeUs(int64 time_us); 00687 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); } 00688 00690 void AdjustTimeUsWithoutWakingAlarms(int64 time_us); 00691 00693 const RequestContext::TimingInfo& timing_info(); 00694 RequestContext::TimingInfo* mutable_timing_info(); 00695 00700 LoggingInfo* logging_info(); 00701 00703 const MetadataCacheInfo& metadata_cache_info() { 00704 return logging_info()->metadata_cache_info(); 00705 } 00706 00709 GoogleString AppliedRewriterStringFromLog(); 00710 00713 void VerifyRewriterInfoEntry(AbstractLogRecord* log_record, 00714 const GoogleString& id, 00715 int url_index, 00716 int rewriter_info_index, 00717 int rewriter_info_size, 00718 int url_list_size, 00719 const GoogleString& url); 00720 00722 void SetCurrentUserAgent(const StringPiece& user_agent) { 00723 current_user_agent_ = user_agent; 00724 } 00725 00726 GoogleString ExpectedNonce(); 00727 00730 GoogleString HttpCacheKey(StringPiece url) { 00731 return HTTPCache::CompositeKey(url, rewrite_driver_->CacheFragment()); 00732 } 00733 00736 MockUrlFetcher mock_url_fetcher_; 00737 TestDistributedFetcher test_distributed_fetcher_; 00738 scoped_ptr<Statistics> statistics_; 00739 00745 scoped_ptr<TestRewriteDriverFactory> factory_; 00746 scoped_ptr<TestRewriteDriverFactory> other_factory_; 00747 ServerContext* server_context_; 00748 RewriteDriver* rewrite_driver_; 00749 ServerContext* other_server_context_; 00750 RewriteDriver* other_rewrite_driver_; 00751 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_; 00752 ActiveServerFlag active_server_; 00753 bool use_managed_rewrite_drivers_; 00754 StringPiece current_user_agent_; 00755 MD5Hasher md5_hasher_; 00756 00757 RewriteOptions* options_; 00758 RewriteOptions* other_options_; 00759 UrlSegmentEncoder default_encoder_; 00760 ResponseHeaders response_headers_; 00761 const GoogleString kEtag0; 00762 uint64 expected_nonce_; 00763 }; 00764 00765 } 00766 00767 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_