Page Speed Optimization Libraries
1.6.29.3
|
00001 /* 00002 * Copyright 2010 Google Inc. 00003 * 00004 * Licensed under the Apache License, Version 2.0 (the "License"); 00005 * you may not use this file except in compliance with the License. 00006 * You may obtain a copy of the License at 00007 * 00008 * http:///www.apache.org/licenses/LICENSE-2.0 00009 * 00010 * Unless required by applicable law or agreed to in writing, software 00011 * distributed under the License is distributed on an "AS IS" BASIS, 00012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 00013 * See the License for the specific language governing permissions and 00014 * limitations under the License. 00015 */ 00016 00018 00020 00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_ 00023 00024 #include <utility> 00025 #include <vector> 00026 00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h" 00028 #include "net/instaweb/http/public/mock_url_fetcher.h" 00029 #include "net/instaweb/http/public/http_cache.h" 00030 #include "net/instaweb/http/public/logging_proto.h" 00031 #include "net/instaweb/http/public/logging_proto_impl.h" 00032 #include "net/instaweb/http/public/response_headers.h" 00034 #include "net/instaweb/rewriter/public/resource.h" 00035 #include "net/instaweb/http/public/request_context.h" 00036 #include "net/instaweb/rewriter/public/rewrite_driver.h" 00037 #include "net/instaweb/rewriter/public/rewrite_options.h" 00038 #include "net/instaweb/rewriter/public/server_context.h" 00039 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h" 00040 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h" 00041 #include "net/instaweb/util/public/basictypes.h" 00042 #include "net/instaweb/util/public/md5_hasher.h" 00043 #include "net/instaweb/util/public/mem_file_system.h" 00044 #include "net/instaweb/util/public/mock_hasher.h" 00045 #include "net/instaweb/util/public/mock_message_handler.h" 00046 #include "net/instaweb/util/public/mock_property_page.h" 00048 #include "net/instaweb/util/public/mock_timer.h" 00049 #include "net/instaweb/util/public/property_cache.h" 00050 #include "net/instaweb/util/public/scoped_ptr.h" 00051 #include "net/instaweb/util/public/string.h" 00052 #include "net/instaweb/util/public/string_util.h" 00053 #include "net/instaweb/util/public/timer.h" 00054 #include "net/instaweb/util/public/url_segment_encoder.h" 00055 #include "pagespeed/kernel/http/content_type.h" 00056 00057 00058 namespace net_instaweb { 00059 00060 class AbstractLogRecord; 00061 class CountingUrlAsyncFetcher; 00062 class DelayCache; 00063 class HTTPValue; 00064 class Hasher; 00065 class HtmlWriterFilter; 00066 class LRUCache; 00067 class MessageHandler; 00068 class MockLogRecord; 00069 class MockScheduler; 00070 class RequestHeaders; 00071 class ResourceNamer; 00072 class RewriteFilter; 00073 class Statistics; 00074 class WaitUrlAsyncFetcher; 00075 00076 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc { 00077 protected: 00078 RewriteOptionsTestBase() { 00079 RewriteOptions::Initialize(); 00080 } 00081 ~RewriteOptionsTestBase() { 00082 RewriteOptions::Terminate(); 00083 } 00084 }; 00085 00086 class RewriteTestBase : public RewriteOptionsTestBase { 00087 public: 00088 static const char kTestData[]; 00089 00090 00091 00092 00093 00094 00095 enum ActiveServerFlag { 00096 kPrimary, 00097 kSecondary, 00098 }; 00099 00100 RewriteTestBase(); 00101 explicit RewriteTestBase(Statistics* statistics); 00102 00108 explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*, 00109 TestRewriteDriverFactory*> factories); 00110 virtual ~RewriteTestBase(); 00111 00112 virtual void SetUp(); 00113 virtual void TearDown(); 00114 00118 virtual bool AddBody() const { return false; } 00119 00129 virtual TestRewriteDriverFactory* MakeTestFactory(); 00130 00133 void AddRecompressImageFilters(); 00134 00136 void AddFilter(RewriteOptions::Filter filter); 00137 00139 void AddOtherFilter(RewriteOptions::Filter filter); 00140 00143 void AddRewriteFilter(RewriteFilter* filter); 00144 00147 void AddFetchOnlyRewriteFilter(RewriteFilter* filter); 00148 00151 void AddOtherRewriteFilter(RewriteFilter* filter); 00152 00155 void SetBaseUrlForFetch(const StringPiece& url); 00156 00157 ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url); 00158 00163 Timer* timer() { return factory()->mock_timer(); } 00164 00166 void AppendDefaultHeaders(const ContentType& content_type, 00167 GoogleString* text); 00168 00171 void AppendDefaultHeaders(const ContentType& content_type, 00172 int64 original_content_length, 00173 GoogleString* text); 00174 00175 void ServeResourceFromManyContexts(const GoogleString& resource_url, 00176 const StringPiece& expected_content); 00177 00178 void ServeResourceFromManyContextsWithUA( 00179 const GoogleString& resource_url, 00180 const StringPiece& expected_content, 00181 const StringPiece& user_agent); 00182 00185 void ServeResourceFromNewContext( 00186 const GoogleString& resource_url, 00187 const StringPiece& expected_content); 00188 00192 virtual RewriteDriver* html_parse() { return rewrite_driver_; } 00193 00195 void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec, 00196 ResponseHeaders* response_headers); 00197 00201 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00202 const StringPiece& name, const StringPiece& ext, 00203 GoogleString* content); 00204 bool FetchResource(const StringPiece& path, const StringPiece& filter_id, 00205 const StringPiece& name, const StringPiece& ext, 00206 GoogleString* content, ResponseHeaders* response); 00207 00208 bool FetchResourceUrl(const StringPiece& url, GoogleString* content, 00209 ResponseHeaders* response); 00210 bool FetchResourceUrl(const StringPiece& url, 00211 RequestHeaders* request_headers, 00212 GoogleString* content, 00213 ResponseHeaders* response_headers); 00214 bool FetchResourceUrl(const StringPiece& url, GoogleString* content); 00215 00217 bool TryFetchResource(const StringPiece& url); 00218 00223 void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers); 00224 00225 GoogleString CssLinkHref(const StringPiece& url) { 00226 return StrCat("<link rel=stylesheet href=", url, ">"); 00227 } 00228 00230 class CssLink { 00231 public: 00232 CssLink(const StringPiece& url, const StringPiece& content, 00233 const StringPiece& media, bool supply_mock); 00234 00236 class Vector : public std::vector<CssLink*> { 00237 public: 00238 ~Vector(); 00239 void Add(const StringPiece& url, const StringPiece& content, 00240 const StringPiece& media, bool supply_mock); 00241 }; 00242 00245 bool DecomposeCombinedUrl(GoogleString* base, StringVector* segments, 00246 MessageHandler* handler); 00247 00248 GoogleString url_; 00249 GoogleString content_; 00250 GoogleString media_; 00251 bool supply_mock_; 00252 }; 00253 00255 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00256 StringVector* css_links); 00257 00259 void CollectCssLinks(const StringPiece& id, const StringPiece& html, 00260 CssLink::Vector* css_links); 00261 00263 void EncodePathAndLeaf(const StringPiece& filter_id, 00264 const StringPiece& hash, 00265 const StringVector& name_vector, 00266 const StringPiece& ext, 00267 ResourceNamer* namer); 00268 00269 StringVector MultiUrl(const StringPiece& url1) { 00270 StringVector v; 00271 v.push_back(url1.as_string()); 00272 return v; 00273 } 00274 00275 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) { 00276 StringVector v; 00277 v.push_back(url1.as_string()); 00278 v.push_back(url2.as_string()); 00279 return v; 00280 } 00281 00282 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00283 const StringPiece& url3) { 00284 StringVector v; 00285 v.push_back(url1.as_string()); 00286 v.push_back(url2.as_string()); 00287 v.push_back(url3.as_string()); 00288 return v; 00289 } 00290 00291 StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2, 00292 const StringPiece& url3, const StringPiece& url4) { 00293 StringVector v; 00294 v.push_back(url1.as_string()); 00295 v.push_back(url2.as_string()); 00296 v.push_back(url3.as_string()); 00297 v.push_back(url4.as_string()); 00298 return v; 00299 } 00300 00303 GoogleString Encode(const StringPiece& path, 00304 const StringPiece& filter_id, 00305 const StringPiece& hash, 00306 const StringPiece& name, 00307 const StringPiece& ext) { 00308 return Encode(path, filter_id, hash, MultiUrl(name), ext); 00309 } 00310 GoogleString Encode(const StringPiece& path, 00311 const StringPiece& filter_id, 00312 const StringPiece& hash, 00313 const StringVector& name_vector, 00314 const StringPiece& ext); 00315 00317 GoogleString EncodeNormal(const StringPiece& path, 00318 const StringPiece& filter_id, 00319 const StringPiece& hash, 00320 const StringPiece& name, 00321 const StringPiece& ext) { 00322 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext); 00323 } 00324 GoogleString EncodeNormal(const StringPiece& path, 00325 const StringPiece& filter_id, 00326 const StringPiece& hash, 00327 const StringVector& name_vector, 00328 const StringPiece& ext); 00329 00332 GoogleString EncodeWithBase(const StringPiece& base, 00333 const StringPiece& path, 00334 const StringPiece& filter_id, 00335 const StringPiece& hash, 00336 const StringPiece& name, 00337 const StringPiece& ext) { 00338 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext); 00339 } 00340 GoogleString EncodeWithBase(const StringPiece& base, 00341 const StringPiece& path, 00342 const StringPiece& filter_id, 00343 const StringPiece& hash, 00344 const StringVector& name_vector, 00345 const StringPiece& ext); 00346 00348 GoogleString AddOptionsToEncodedUrl(const StringPiece& url, 00349 const StringPiece& options); 00350 00355 static GoogleString ChangeSuffix( 00356 GoogleString old_url, bool append_new_suffix, 00357 StringPiece old_suffix, StringPiece new_suffix); 00358 00363 void SetupWaitFetcher(); 00364 void CallFetcherCallbacks(); 00365 void OtherCallFetcherCallbacks(); 00366 RewriteOptions* options() { return options_; } 00367 RewriteOptions* other_options() { return other_options_; } 00368 00370 void SetRewriteOptions(RewriteOptions* opts); 00371 00374 bool AddDomain(StringPiece domain); 00375 00378 bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain); 00379 00382 bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain); 00383 00386 bool AddShard(StringPiece domain, StringPiece shards); 00387 00389 void TestServeFiles(const ContentType* content_type, 00390 const StringPiece& filter_id, 00391 const StringPiece& rewritten_ext, 00392 const StringPiece& orig_name, 00393 const StringPiece& orig_content, 00394 const StringPiece& rewritten_name, 00395 const StringPiece& rewritten_content); 00396 00397 TestRewriteDriverFactory* factory() { return factory_.get(); } 00398 TestRewriteDriverFactory* other_factory() { return other_factory_.get(); } 00399 00400 void UseMd5Hasher() { 00401 server_context_->set_hasher(&md5_hasher_); 00402 server_context_->http_cache()->set_hasher(&md5_hasher_); 00403 other_server_context_->set_hasher(&md5_hasher_); 00404 other_server_context_->http_cache()->set_hasher(&md5_hasher_); 00405 } 00406 00407 00408 void SetDefaultLongCacheHeaders(const ContentType* content_type, 00409 ResponseHeaders* header) { 00410 server_context_->SetDefaultLongCacheHeaders(content_type, header); 00411 } 00412 00413 void SetFetchResponse(const StringPiece& url, 00414 const ResponseHeaders& response_header, 00415 const StringPiece& response_body) { 00416 mock_url_fetcher()->SetResponse(url, response_header, response_body); 00417 } 00418 00420 void SetResponseWithDefaultHeaders(const StringPiece& relative_url, 00421 const ContentType& content_type, 00422 const StringPiece& content, 00423 int64 ttl_sec); 00424 00427 bool LoadFile(const StringPiece& filename, GoogleString* contents); 00428 00430 void AddFileToMockFetcher(const StringPiece& url, 00431 const StringPiece& filename, 00432 const ContentType& content_type, int64 ttl_sec); 00433 00434 void AddToResponse(const StringPiece& url, 00435 const StringPiece& name, 00436 const StringPiece& value) { 00437 mock_url_fetcher()->AddToResponse(url, name, value); 00438 } 00439 00440 void SetFetchResponse404(const StringPiece& url); 00441 00442 void SetFetchFailOnUnexpected(bool fail) { 00443 mock_url_fetcher()->set_fail_on_unexpected(fail); 00444 } 00445 void FetcherUpdateDateHeaders() { 00446 mock_url_fetcher()->set_timer(timer()); 00447 mock_url_fetcher()->set_update_date_headers(true); 00448 } 00449 void ClearFetcherResponses() { mock_url_fetcher()->Clear(); } 00450 00451 virtual void ClearStats(); 00452 00455 void ClearRewriteDriver(); 00456 00457 MockUrlFetcher* mock_url_fetcher() { 00458 return &mock_url_fetcher_; 00459 } 00460 TestDistributedFetcher* test_distributed_fetcher() { 00461 return &test_distributed_fetcher_; 00462 } 00463 Hasher* hasher() { return server_context_->hasher(); } 00464 DelayCache* delay_cache() { return factory_->delay_cache(); } 00465 LRUCache* lru_cache() { return factory_->lru_cache(); } 00466 Statistics* statistics() { return factory_->statistics(); } 00467 MemFileSystem* file_system() { return factory_->mem_file_system(); } 00468 HTTPCache* http_cache() { return server_context_->http_cache(); } 00469 PropertyCache* page_property_cache() { 00470 return server_context_->page_property_cache(); 00471 } 00472 MockMessageHandler* message_handler() { 00473 return factory_->mock_message_handler(); 00474 } 00475 00483 RewriteDriver* rewrite_driver() { return rewrite_driver_; } 00484 RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; } 00485 00487 MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); } 00488 00489 int64 start_time_ms() const { return factory_->kStartTimeMs; } 00490 00491 bool ReadFile(const char* filename, GoogleString* contents) { 00492 return file_system()->ReadFile(filename, contents, message_handler()); 00493 } 00494 bool WriteFile(const char* filename, const StringPiece& contents) { 00495 return file_system()->WriteFile(filename, contents, message_handler()); 00496 } 00497 00498 ServerContext* server_context() { return server_context_; } 00499 ServerContext* other_server_context() { return other_server_context_; } 00500 CountingUrlAsyncFetcher* counting_url_async_fetcher() { 00501 return factory_->counting_url_async_fetcher(); 00502 } 00503 CountingUrlAsyncFetcher* counting_distributed_fetcher() { 00504 return factory_->counting_distributed_async_fetcher(); 00505 } 00506 void SetMockHashValue(const GoogleString& value) { 00507 factory_->mock_hasher()->set_hash_value(value); 00508 } 00509 00510 void SetCacheDelayUs(int64 delay_us); 00511 00516 RewriteDriver* MakeDriver(ServerContext* server_context, 00517 RewriteOptions* options); 00518 00520 GoogleString AbsolutifyUrl(const StringPiece& in); 00521 00526 void TestRetainExtraHeaders(const StringPiece& name, 00527 const StringPiece& filter_id, 00528 const StringPiece& ext); 00529 00533 const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const; 00534 00536 void SetUseTestUrlNamer(bool use_test_url_namer); 00537 00540 GoogleString EncodeCssName(const StringPiece& name, 00541 bool supports_webp, 00542 bool can_inline); 00543 00547 bool ReadIfCached(const ResourcePtr& resource); 00548 00553 void InitiateResourceRead(const ResourcePtr& resource); 00554 00558 HTTPCache::FindResult HttpBlockingFind( 00559 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out, 00560 ResponseHeaders* headers); 00561 00564 HTTPCache::FindResult HttpBlockingFindStatus( 00565 const GoogleString& key, HTTPCache* http_cache); 00566 00568 void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); } 00569 00571 void SetHtmlMimetype() { SetMimetype("text/html"); } 00572 00574 void SetMimetype(const StringPiece& mimetype); 00575 00578 void CheckFetchFromHttpCache( 00579 StringPiece url, 00580 StringPiece expected_contents, 00581 int64 expected_expiration_ms); 00582 00584 const PropertyCache::Cohort* SetupCohort( 00585 PropertyCache* cache, const GoogleString& cohort) { 00586 return factory()->SetupCohort(cache, cohort); 00587 } 00588 00591 void SetupSharedCache(); 00592 00594 MockPropertyPage* NewMockPage(const StringPiece& key) { 00595 return new MockPropertyPage( 00596 server_context_->thread_system(), 00597 server_context_->page_property_cache(), 00598 key); 00599 } 00600 00602 void SetMockLogRecord(); 00603 00605 MockLogRecord* mock_log_record(); 00606 00608 GoogleString GetLazyloadScriptHtml(); 00609 GoogleString GetLazyloadPostscriptHtml(); 00610 00611 protected: 00612 void Init(); 00613 00616 virtual RequestContextPtr CreateRequestContext(); 00617 00620 void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher, 00621 RewriteDriver* driver); 00622 00625 void PopulateDefaultHeaders(const ContentType& content_type, 00626 int64 original_content_length, 00627 ResponseHeaders* headers); 00628 00631 void SetActiveServer(ActiveServerFlag server_to_use); 00632 00635 void AdvanceTimeUs(int64 delay_ms); 00636 void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); } 00637 void SetTimeUs(int64 time_us); 00638 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); } 00639 00641 void AdjustTimeUsWithoutWakingAlarms(int64 time_us); 00642 00644 const RequestContext::TimingInfo& timing_info(); 00645 RequestContext::TimingInfo* mutable_timing_info(); 00646 00651 LoggingInfo* logging_info(); 00652 00654 const MetadataCacheInfo& metadata_cache_info() { 00655 return logging_info()->metadata_cache_info(); 00656 } 00657 00660 GoogleString AppliedRewriterStringFromLog(); 00661 00664 void VerifyRewriterInfoEntry(AbstractLogRecord* log_record, 00665 const GoogleString& id, 00666 int url_index, 00667 int rewriter_info_index, 00668 int rewriter_info_size, 00669 int url_list_size, 00670 const GoogleString& url); 00671 00673 void SetCurrentUserAgent(const StringPiece& user_agent) { 00674 current_user_agent_ = user_agent; 00675 } 00676 00679 MockUrlFetcher mock_url_fetcher_; 00680 TestDistributedFetcher test_distributed_fetcher_; 00681 scoped_ptr<Statistics> statistics_; 00682 00688 scoped_ptr<TestRewriteDriverFactory> factory_; 00689 scoped_ptr<TestRewriteDriverFactory> other_factory_; 00690 ServerContext* server_context_; 00691 RewriteDriver* rewrite_driver_; 00692 ServerContext* other_server_context_; 00693 RewriteDriver* other_rewrite_driver_; 00694 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_; 00695 ActiveServerFlag active_server_; 00696 bool use_managed_rewrite_drivers_; 00697 StringPiece current_user_agent_; 00698 MD5Hasher md5_hasher_; 00699 00700 RewriteOptions* options_; 00701 RewriteOptions* other_options_; 00702 UrlSegmentEncoder default_encoder_; 00703 ResponseHeaders response_headers_; 00704 const GoogleString kEtag0; 00705 }; 00706 00707 } 00708 00709 #endif ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_