Page Speed Optimization Libraries
1.7.30.1
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00020 00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00023 00024 #include <utility> 00025 #include <vector> 00026 00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h" 00028 #include "net/instaweb/http/public/mock_url_fetcher.h" 00029 #include "net/instaweb/http/public/http_cache.h" 00030 #include "net/instaweb/http/public/logging_proto.h" 00031 #include "net/instaweb/http/public/logging_proto_impl.h" 00032 #include "net/instaweb/http/public/request_context.h" 00033 #include "net/instaweb/http/public/response_headers.h" 00034 #include "net/instaweb/http/public/user_agent_matcher.h" 00036 #include "net/instaweb/rewriter/public/resource.h" 00037 #include "net/instaweb/rewriter/public/rewrite_driver.h" 00038 #include "net/instaweb/rewriter/public/rewrite_options.h" 00039 #include "net/instaweb/rewriter/public/server_context.h" 00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h" 00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" 00042 #include "net/instaweb/util/public/basictypes.h" 00043 #include "net/instaweb/util/public/md5_hasher.h" 00044 #include "net/instaweb/util/public/mem_file_system.h" 00045 #include "net/instaweb/util/public/mock_hasher.h" 00046 #include "net/instaweb/util/public/mock_message_handler.h" 00047 #include "net/instaweb/util/public/mock_property_page.h" 00049 #include "net/instaweb/util/public/mock_timer.h" 00050 #include "net/instaweb/util/public/property_cache.h" 00051 #include "net/instaweb/util/public/scoped_ptr.h" 00052 #include "net/instaweb/util/public/string.h" 00053 #include "net/instaweb/util/public/string_util.h" 00054 #include "net/instaweb/util/public/timer.h" 00055 #include "net/instaweb/util/public/url_segment_encoder.h" 00056 #include "pagespeed/kernel/http/content_type.h" 00057 00058 00059 namespace net_instaweb { 00060 00061 class AbstractLogRecord; 00062 class CountingUrlAsyncFetcher; 00063 class DelayCache; 00064 class HTTPValue; 00065 class Hasher; 00066 class HtmlWriterFilter; 00067 class LRUCache; 00068 class MessageHandler; 00069 class MockLogRecord; 00070 class MockScheduler; 00071 class RequestHeaders; 00072 class ResourceNamer; 00073 class RewriteFilter; 00074 class Statistics; 00075 class WaitUrlAsyncFetcher; 00076 00077 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc { 00078 protected: 00079 RewriteOptionsTestBase() { 00080 RewriteOptions::Initialize(); 00081 } 00082 ~RewriteOptionsTestBase() { 00083 RewriteOptions::Terminate(); 00084 } 00085 }; 00086 00087 class RewriteTestBase : public RewriteOptionsTestBase { 00088 public: 00089 static const char kTestData[]; 00090 00091 00092 00093 00094 00095 00096 enum ActiveServerFlag { 00097 kPrimary, 00098 kSecondary, 00099 }; 00100 00101 RewriteTestBase(); 00102 explicit RewriteTestBase(Statistics* statistics); 00103 00109 explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*, 00110 TestRewriteDriverFactory*> factories); 00111 virtual ~RewriteTestBase(); 00112 00113 virtual void SetUp(); 00114 virtual void TearDown(); 00115 00119 virtual bool AddBody() const { return false; } 00120 00130 virtual TestRewriteDriverFactory* MakeTestFactory(); 00131 00134 void AddRecompressImageFilters(); 00135 00137 void AddFilter(RewriteOptions::Filter filter); 00138 00140 void AddOtherFilter(RewriteOptions::Filter filter); 00141 00144 void AddRewriteFilter(RewriteFilter* filter); 00145 00148 void AddFetchOnlyRewriteFilter(RewriteFilter* filter); 00149 00152 void AddOtherRewriteFilter(RewriteFilter* filter); 00153 00156 void SetBaseUrlForFetch(const StringPiece& url); 00157 00158 ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url); 00159 00164 Timer* timer() { return factory()->mock_timer(); } 00165 00167 void AppendDefaultHeaders(const ContentType& content_type, 00168 GoogleString* text); 00169 00172 void AppendDefaultHeaders(const ContentType& content_type, 00173 int64 original_content_length, 00174 GoogleString* text); 00175 00176 void ServeResourceFromManyContexts(const GoogleString& resource_url, 00177 const StringPiece& expected_content); 00178 00179 void ServeResourceFromManyContextsWithUA( 00180 const GoogleString& resource_url, 00181 const StringPiece& expected_content, 00182 const StringPiece& user_agent); 00183 00186 void ServeResourceFromNewContext( 00187 const GoogleString& resource_url, 00188 const StringPiece& expected_content); 00189 00193 virtual RewriteDriver* html_parse() { return rewrite_driver_; } 00194 00196 void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec, 00197 ResponseHeaders* response_headers); 00198 00202 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00203 const StringPiece& name, const StringPiece& ext, 00204 GoogleString* content); 00205 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00206 const StringPiece& name, const StringPiece& ext, 00207 GoogleString* content, ResponseHeaders* response); 00208 00209 bool FetchResourceUrl(const StringPiece& url, GoogleString* content, 00210 ResponseHeaders* response); 00211 bool FetchResourceUrl(const StringPiece& url, 00212 RequestHeaders* request_headers, 00213 GoogleString* content, 00214 ResponseHeaders* response_headers); 00215 bool FetchResourceUrl(const StringPiece& url, GoogleString* content); 00216 00218 bool TryFetchResource(const StringPiece& url); 00219 00224 void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers); 00225 00226 GoogleString CssLinkHref(const StringPiece& url) { 00227 return StrCat("<link rel=stylesheet href=", url, ">"); 00228 } 00229 00231 class CssLink { 00232 public: 00233 CssLink(const StringPiece& url, const StringPiece& content, 00234 const StringPiece& media, bool supply_mock); 00235 00237 class Vector : public std::vector<CssLink*> { 00238 public: 00239 ~Vector(); 00240 void Add(const StringPiece& url, const StringPiece& content, 00241 const StringPiece& media, bool supply_mock); 00242 }; 00243 00246 bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base, 00247 StringVector* segments, MessageHandler* handler); 00248 00249 GoogleString url_; 00250 GoogleString content_; 00251 GoogleString media_; 00252 bool supply_mock_; 00253 }; 00254 00256 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00257 StringVector* css_links); 00258 00260 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00261 CssLink::Vector* css_links); 00262 00264 void EncodePathAndLeaf(const StringPiece& filter_id, 00265 const StringPiece& hash, 00266 const StringVector& name_vector, 00267 const StringPiece& ext, 00268 ResourceNamer* namer); 00269 00270 StringVector MultiUrl(const StringPiece& url1) { 00271 StringVector v; 00272 v.push_back(url1.as_string()); 00273 return v; 00274 } 00275 00276 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) { 00277 StringVector v; 00278 v.push_back(url1.as_string()); 00279 v.push_back(url2.as_string()); 00280 return v; 00281 } 00282 00283 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00284 const StringPiece& url3) { 00285 StringVector v; 00286 v.push_back(url1.as_string()); 00287 v.push_back(url2.as_string()); 00288 v.push_back(url3.as_string()); 00289 return v; 00290 } 00291 00292 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00293 const StringPiece& url3, const StringPiece& url4) { 00294 StringVector v; 00295 v.push_back(url1.as_string()); 00296 v.push_back(url2.as_string()); 00297 v.push_back(url3.as_string()); 00298 v.push_back(url4.as_string()); 00299 return v; 00300 } 00301 00304 GoogleString Encode(const StringPiece& path, 00305 const StringPiece& filter_id, 00306 const StringPiece& hash, 00307 const StringPiece& name, 00308 const StringPiece& ext) { 00309 return Encode(path, filter_id, hash, MultiUrl(name), ext); 00310 } 00311 GoogleString Encode(const StringPiece& path, 00312 const StringPiece& filter_id, 00313 const StringPiece& hash, 00314 const StringVector& name_vector, 00315 const StringPiece& ext); 00316 00318 GoogleString EncodeNormal(const StringPiece& path, 00319 const StringPiece& filter_id, 00320 const StringPiece& hash, 00321 const StringPiece& name, 00322 const StringPiece& ext) { 00323 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext); 00324 } 00325 GoogleString EncodeNormal(const StringPiece& path, 00326 const StringPiece& filter_id, 00327 const StringPiece& hash, 00328 const StringVector& name_vector, 00329 const StringPiece& ext); 00330 00333 GoogleString EncodeWithBase(const StringPiece& base, 00334 const StringPiece& path, 00335 const StringPiece& filter_id, 00336 const StringPiece& hash, 00337 const StringPiece& name, 00338 const StringPiece& ext) { 00339 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext); 00340 } 00341 GoogleString EncodeWithBase(const StringPiece& base, 00342 const StringPiece& path, 00343 const StringPiece& filter_id, 00344 const StringPiece& hash, 00345 const StringVector& name_vector, 00346 const StringPiece& ext); 00347 00349 GoogleString AddOptionsToEncodedUrl(const StringPiece& url, 00350 const StringPiece& options); 00351 00356 static GoogleString ChangeSuffix( 00357 StringPiece old_url, bool append_new_suffix, 00358 StringPiece old_suffix, StringPiece new_suffix); 00359 00364 void SetupWaitFetcher(); 00365 void CallFetcherCallbacks(); 00366 void OtherCallFetcherCallbacks(); 00367 RewriteOptions* options() { return options_; } 00368 RewriteOptions* other_options() { return other_options_; } 00369 00371 void SetRewriteOptions(RewriteOptions* opts); 00372 00375 bool AddDomain(StringPiece domain); 00376 00379 bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain); 00380 00383 bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain); 00384 00387 bool AddShard(StringPiece domain, StringPiece shards); 00388 00390 void TestServeFiles(const ContentType* content_type, 00391 const StringPiece& filter_id, 00392 const StringPiece& rewritten_ext, 00393 const StringPiece& orig_name, 00394 const StringPiece& orig_content, 00395 const StringPiece& rewritten_name, 00396 const StringPiece& rewritten_content); 00397 00398 TestRewriteDriverFactory* factory() { return factory_.get(); } 00399 TestRewriteDriverFactory* other_factory() { return other_factory_.get(); } 00400 00401 void UseMd5Hasher() { 00402 server_context_->set_hasher(&md5_hasher_); 00403 server_context_->http_cache()->set_hasher(&md5_hasher_); 00404 other_server_context_->set_hasher(&md5_hasher_); 00405 other_server_context_->http_cache()->set_hasher(&md5_hasher_); 00406 } 00407 00408 00409 void SetDefaultLongCacheHeaders(const ContentType* content_type, 00410 ResponseHeaders* header) { 00411 server_context_->SetDefaultLongCacheHeaders(content_type, header); 00412 } 00413 00414 void SetFetchResponse(const StringPiece& url, 00415 const ResponseHeaders& response_header, 00416 const StringPiece& response_body) { 00417 mock_url_fetcher()->SetResponse(url, response_header, response_body); 00418 } 00419 00421 void SetResponseWithDefaultHeaders(const StringPiece& relative_url, 00422 const ContentType& content_type, 00423 const StringPiece& content, 00424 int64 ttl_sec); 00425 00428 bool LoadFile(const StringPiece& filename, GoogleString* contents); 00429 00431 void AddFileToMockFetcher(const StringPiece& url, 00432 const StringPiece& filename, 00433 const ContentType& content_type, int64 ttl_sec); 00434 00435 void AddToResponse(const StringPiece& url, 00436 const StringPiece& name, 00437 const StringPiece& value) { 00438 mock_url_fetcher()->AddToResponse(url, name, value); 00439 } 00440 00441 void SetFetchResponse404(const StringPiece& url); 00442 00443 void SetFetchFailOnUnexpected(bool fail) { 00444 mock_url_fetcher()->set_fail_on_unexpected(fail); 00445 } 00446 void FetcherUpdateDateHeaders() { 00447 mock_url_fetcher()->set_timer(timer()); 00448 mock_url_fetcher()->set_update_date_headers(true); 00449 } 00450 void ClearFetcherResponses() { mock_url_fetcher()->Clear(); } 00451 00452 virtual void ClearStats(); 00453 00456 void ClearRewriteDriver(); 00457 00458 MockUrlFetcher* mock_url_fetcher() { 00459 return &mock_url_fetcher_; 00460 } 00461 TestDistributedFetcher* test_distributed_fetcher() { 00462 return &test_distributed_fetcher_; 00463 } 00464 Hasher* hasher() { return server_context_->hasher(); } 00465 DelayCache* delay_cache() { return factory_->delay_cache(); } 00466 LRUCache* lru_cache() { return factory_->lru_cache(); } 00467 Statistics* statistics() { return factory_->statistics(); } 00468 MemFileSystem* file_system() { return factory_->mem_file_system(); } 00469 HTTPCache* http_cache() { return server_context_->http_cache(); } 00470 PropertyCache* page_property_cache() { 00471 return server_context_->page_property_cache(); 00472 } 00473 MockMessageHandler* message_handler() { 00474 return factory_->mock_message_handler(); 00475 } 00476 00484 RewriteDriver* rewrite_driver() { return rewrite_driver_; } 00485 RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; } 00486 00488 MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); } 00489 00490 int64 start_time_ms() const { return factory_->kStartTimeMs; } 00491 00492 bool ReadFile(const char* filename, GoogleString* contents) { 00493 return file_system()->ReadFile(filename, contents, message_handler()); 00494 } 00495 bool WriteFile(const char* filename, const StringPiece& contents) { 00496 return file_system()->WriteFile(filename, contents, message_handler()); 00497 } 00498 00499 ServerContext* server_context() { return server_context_; } 00500 ServerContext* other_server_context() { return other_server_context_; } 00501 CountingUrlAsyncFetcher* counting_url_async_fetcher() { 00502 return factory_->counting_url_async_fetcher(); 00503 } 00504 CountingUrlAsyncFetcher* counting_distributed_fetcher() { 00505 return factory_->counting_distributed_async_fetcher(); 00506 } 00507 void SetMockHashValue(const GoogleString& value) { 00508 factory_->mock_hasher()->set_hash_value(value); 00509 } 00510 00511 void SetCacheDelayUs(int64 delay_us); 00512 00517 RewriteDriver* MakeDriver(ServerContext* server_context, 00518 RewriteOptions* options); 00519 00521 GoogleString AbsolutifyUrl(const StringPiece& in); 00522 00527 void TestRetainExtraHeaders(const StringPiece& name, 00528 const StringPiece& filter_id, 00529 const StringPiece& ext); 00530 00534 const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const; 00535 00537 void SetUseTestUrlNamer(bool use_test_url_namer); 00538 00541 GoogleString EncodeCssName(const StringPiece& name, 00542 bool supports_webp, 00543 bool can_inline); 00544 00548 bool ReadIfCached(const ResourcePtr& resource); 00549 00554 void InitiateResourceRead(const ResourcePtr& resource); 00555 00559 HTTPCache::FindResult HttpBlockingFind( 00560 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out, 00561 ResponseHeaders* headers); 00562 00565 HTTPCache::FindResult HttpBlockingFindStatus( 00566 const GoogleString& key, HTTPCache* http_cache); 00567 00569 void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); } 00570 00572 void SetHtmlMimetype() { SetMimetype("text/html"); } 00573 00575 void SetMimetype(const StringPiece& mimetype); 00576 00579 void CheckFetchFromHttpCache( 00580 StringPiece url, 00581 StringPiece expected_contents, 00582 int64 expected_expiration_ms); 00583 00585 const PropertyCache::Cohort* SetupCohort( 00586 PropertyCache* cache, const GoogleString& cohort) { 00587 return factory()->SetupCohort(cache, cohort); 00588 } 00589 00592 void SetupSharedCache(); 00593 00595 MockPropertyPage* NewMockPage(const StringPiece& url, 00596 const StringPiece& options_signature_hash, 00597 UserAgentMatcher::DeviceType device_type) { 00598 return new MockPropertyPage( 00599 server_context_->thread_system(), 00600 server_context_->page_property_cache(), 00601 url, 00602 options_signature_hash, 00603 UserAgentMatcher::DeviceTypeSuffix(device_type)); 00604 } 00605 00606 MockPropertyPage* NewMockPage(const StringPiece& url) { 00607 return NewMockPage(url, "hash", UserAgentMatcher::kDesktop); 00608 } 00609 00611 void SetMockLogRecord(); 00612 00614 MockLogRecord* mock_log_record(); 00615 00617 GoogleString GetLazyloadScriptHtml(); 00618 GoogleString GetLazyloadPostscriptHtml(); 00619 00625 void SetCacheInvalidationTimestamp(); 00626 00629 void SetCacheInvalidationTimestampForUrl( 00630 StringPiece url, bool ignores_metadata_and_pcache); 00631 00635 void EnableCachePurge(); 00636 00637 protected: 00638 void Init(); 00639 00642 virtual RequestContextPtr CreateRequestContext(); 00643 00646 void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher, 00647 RewriteDriver* driver); 00648 00651 void PopulateDefaultHeaders(const ContentType& content_type, 00652 int64 original_content_length, 00653 ResponseHeaders* headers); 00654 00657 void SetActiveServer(ActiveServerFlag server_to_use); 00658 00661 void AdvanceTimeUs(int64 delay_ms); 00662 void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); } 00663 void SetTimeUs(int64 time_us); 00664 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); } 00665 00667 void AdjustTimeUsWithoutWakingAlarms(int64 time_us); 00668 00670 const RequestContext::TimingInfo& timing_info(); 00671 RequestContext::TimingInfo* mutable_timing_info(); 00672 00677 LoggingInfo* logging_info(); 00678 00680 const MetadataCacheInfo& metadata_cache_info() { 00681 return logging_info()->metadata_cache_info(); 00682 } 00683 00686 GoogleString AppliedRewriterStringFromLog(); 00687 00690 void VerifyRewriterInfoEntry(AbstractLogRecord* log_record, 00691 const GoogleString& id, 00692 int url_index, 00693 int rewriter_info_index, 00694 int rewriter_info_size, 00695 int url_list_size, 00696 const GoogleString& url); 00697 00699 void SetCurrentUserAgent(const StringPiece& user_agent) { 00700 current_user_agent_ = user_agent; 00701 } 00702 00703 GoogleString ExpectedNonce(); 00704 00707 MockUrlFetcher mock_url_fetcher_; 00708 TestDistributedFetcher test_distributed_fetcher_; 00709 scoped_ptr<Statistics> statistics_; 00710 00716 scoped_ptr<TestRewriteDriverFactory> factory_; 00717 scoped_ptr<TestRewriteDriverFactory> other_factory_; 00718 ServerContext* server_context_; 00719 RewriteDriver* rewrite_driver_; 00720 ServerContext* other_server_context_; 00721 RewriteDriver* other_rewrite_driver_; 00722 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_; 00723 ActiveServerFlag active_server_; 00724 bool use_managed_rewrite_drivers_; 00725 StringPiece current_user_agent_; 00726 MD5Hasher md5_hasher_; 00727 00728 RewriteOptions* options_; 00729 RewriteOptions* other_options_; 00730 UrlSegmentEncoder default_encoder_; 00731 ResponseHeaders response_headers_; 00732 const GoogleString kEtag0; 00733 uint64 expected_nonce_; 00734 }; 00735 00736 } 00737 00738 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_