Page Speed Optimization Libraries  1.7.30.4
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/request_context.h"
00033 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/server_context.h"
00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/md5_hasher.h"
00044 #include "net/instaweb/util/public/mem_file_system.h"
00045 #include "net/instaweb/util/public/mock_hasher.h"
00046 #include "net/instaweb/util/public/mock_message_handler.h"
00047 #include "net/instaweb/util/public/mock_property_page.h"
00049 #include "net/instaweb/util/public/mock_timer.h"
00050 #include "net/instaweb/util/public/property_cache.h"
00051 #include "net/instaweb/util/public/scoped_ptr.h"
00052 #include "net/instaweb/util/public/string.h"
00053 #include "net/instaweb/util/public/string_util.h"
00054 #include "net/instaweb/util/public/timer.h"
00055 #include "net/instaweb/util/public/url_segment_encoder.h"
00056 #include "pagespeed/kernel/http/content_type.h"
00057 
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class CountingUrlAsyncFetcher;
00063 class DelayCache;
00064 class HTTPValue;
00065 class Hasher;
00066 class HtmlWriterFilter;
00067 class LRUCache;
00068 class MessageHandler;
00069 class MockLogRecord;
00070 class MockScheduler;
00071 class ProcessContext;
00072 class RequestHeaders;
00073 class ResourceNamer;
00074 class RewriteFilter;
00075 class Statistics;
00076 class WaitUrlAsyncFetcher;
00077 
00078 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00079  protected:
00080   RewriteOptionsTestBase() {
00081     RewriteOptions::Initialize();
00082   }
00083   ~RewriteOptionsTestBase() {
00084     RewriteOptions::Terminate();
00085   }
00086 };
00087 
00088 class RewriteTestBase : public RewriteOptionsTestBase {
00089  public:
00090   static const char kTestData[]; 
00091 
00093   static const char kConfiguredBeaconingKey[];
00094   static const char kWrongBeaconingKey[];
00095 
00102   enum ActiveServerFlag {
00103     kPrimary,    
00104     kSecondary,  
00105   };
00106 
00107   RewriteTestBase();
00108   explicit RewriteTestBase(Statistics* statistics);
00109 
00115   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00116                                      TestRewriteDriverFactory*> factories);
00117   virtual ~RewriteTestBase();
00118 
00119   virtual void SetUp();
00120   virtual void TearDown();
00121 
00125   virtual bool AddBody() const { return false; }
00126 
00136   virtual TestRewriteDriverFactory* MakeTestFactory();
00137 
00140   void AddRecompressImageFilters();
00141 
00143   void AddFilter(RewriteOptions::Filter filter);
00144 
00146   void AddOtherFilter(RewriteOptions::Filter filter);
00147 
00150   void AddRewriteFilter(RewriteFilter* filter);
00151 
00154   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00155 
00158   void AddOtherRewriteFilter(RewriteFilter* filter);
00159 
00162   void SetBaseUrlForFetch(const StringPiece& url);
00163 
00165   void SetDummyRequestHeaders();
00166 
00169   void SetDownstreamCacheDirectives(
00170     StringPiece downstream_cache_location,
00171     StringPiece rebeaconing_key);
00172 
00174   void SetShouldBeaconHeader(StringPiece rebeaconing_key);
00175 
00176   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00177 
00182   Timer* timer() { return factory()->mock_timer(); }
00183 
00185   void AppendDefaultHeaders(const ContentType& content_type,
00186                             GoogleString* text);
00187 
00190   void AppendDefaultHeaders(const ContentType& content_type,
00191                             int64 original_content_length,
00192                             GoogleString* text);
00193 
00194   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00195                                      const StringPiece& expected_content);
00196 
00197   void ServeResourceFromManyContextsWithUA(
00198       const GoogleString& resource_url,
00199       const StringPiece& expected_content,
00200       const StringPiece& user_agent);
00201 
00204   void ServeResourceFromNewContext(
00205       const GoogleString& resource_url,
00206       const StringPiece& expected_content);
00207 
00211   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00212 
00214   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00215                               ResponseHeaders* response_headers);
00216 
00220   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00221                      const StringPiece& name, const StringPiece& ext,
00222                      GoogleString* content);
00223   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00224                      const StringPiece& name, const StringPiece& ext,
00225                      GoogleString* content, ResponseHeaders* response);
00226 
00227   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00228                         ResponseHeaders* response);
00229   bool FetchResourceUrl(const StringPiece& url,
00230                         RequestHeaders* request_headers,
00231                         GoogleString* content,
00232                         ResponseHeaders* response_headers);
00233   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00234 
00236   bool TryFetchResource(const StringPiece& url);
00237 
00242   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00243 
00244   GoogleString CssLinkHref(const StringPiece& url) {
00245     return StrCat("<link rel=stylesheet href=", url, ">");
00246   }
00247 
00249   class CssLink {
00250    public:
00251     CssLink(const StringPiece& url, const StringPiece& content,
00252             const StringPiece& media, bool supply_mock);
00253 
00255     class Vector : public std::vector<CssLink*> {
00256      public:
00257       ~Vector();
00258       void Add(const StringPiece& url, const StringPiece& content,
00259                const StringPiece& media, bool supply_mock);
00260     };
00261 
00264     bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
00265                               StringVector* segments, MessageHandler* handler);
00266 
00267     GoogleString url_;
00268     GoogleString content_;
00269     GoogleString media_;
00270     bool supply_mock_;
00271   };
00272 
00274   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00275                        StringVector* css_links);
00276 
00278   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00279                        CssLink::Vector* css_links);
00280 
00282   void EncodePathAndLeaf(const StringPiece& filter_id,
00283                          const StringPiece& hash,
00284                          const StringVector& name_vector,
00285                          const StringPiece& ext,
00286                          ResourceNamer* namer);
00287 
00288   StringVector MultiUrl(const StringPiece& url1) {
00289     StringVector v;
00290     v.push_back(url1.as_string());
00291     return v;
00292   }
00293 
00294   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00295     StringVector v;
00296     v.push_back(url1.as_string());
00297     v.push_back(url2.as_string());
00298     return v;
00299   }
00300 
00301   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00302                         const StringPiece& url3) {
00303     StringVector v;
00304     v.push_back(url1.as_string());
00305     v.push_back(url2.as_string());
00306     v.push_back(url3.as_string());
00307     return v;
00308   }
00309 
00310   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00311                         const StringPiece& url3, const StringPiece& url4) {
00312     StringVector v;
00313     v.push_back(url1.as_string());
00314     v.push_back(url2.as_string());
00315     v.push_back(url3.as_string());
00316     v.push_back(url4.as_string());
00317     return v;
00318   }
00319 
00322   GoogleString Encode(const StringPiece& path,
00323                       const StringPiece& filter_id,
00324                       const StringPiece& hash,
00325                       const StringPiece& name,
00326                       const StringPiece& ext) {
00327     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00328   }
00329   GoogleString Encode(const StringPiece& path,
00330                       const StringPiece& filter_id,
00331                       const StringPiece& hash,
00332                       const StringVector& name_vector,
00333                       const StringPiece& ext);
00334 
00336   GoogleString EncodeNormal(const StringPiece& path,
00337                             const StringPiece& filter_id,
00338                             const StringPiece& hash,
00339                             const StringPiece& name,
00340                             const StringPiece& ext) {
00341     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00342   }
00343   GoogleString EncodeNormal(const StringPiece& path,
00344                             const StringPiece& filter_id,
00345                             const StringPiece& hash,
00346                             const StringVector& name_vector,
00347                             const StringPiece& ext);
00348 
00351   GoogleString EncodeWithBase(const StringPiece& base,
00352                               const StringPiece& path,
00353                               const StringPiece& filter_id,
00354                               const StringPiece& hash,
00355                               const StringPiece& name,
00356                               const StringPiece& ext) {
00357     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00358   }
00359   GoogleString EncodeWithBase(const StringPiece& base,
00360                               const StringPiece& path,
00361                               const StringPiece& filter_id,
00362                               const StringPiece& hash,
00363                               const StringVector& name_vector,
00364                               const StringPiece& ext);
00365 
00367   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00368                                       const StringPiece& options);
00369 
00374   static GoogleString ChangeSuffix(
00375       StringPiece old_url, bool append_new_suffix,
00376       StringPiece old_suffix, StringPiece new_suffix);
00377 
00382   void SetupWaitFetcher();
00383   void CallFetcherCallbacks();
00384   void OtherCallFetcherCallbacks();
00385   RewriteOptions* options() { return options_; }
00386   RewriteOptions* other_options() { return other_options_; }
00387 
00389   void SetRewriteOptions(RewriteOptions* opts);
00390 
00393   bool AddDomain(StringPiece domain);
00394 
00397   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00398 
00401   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00402 
00405   bool AddShard(StringPiece domain, StringPiece shards);
00406 
00408   void TestServeFiles(const ContentType* content_type,
00409                       const StringPiece& filter_id,
00410                       const StringPiece& rewritten_ext,
00411                       const StringPiece& orig_name,
00412                       const StringPiece& orig_content,
00413                       const StringPiece& rewritten_name,
00414                       const StringPiece& rewritten_content);
00415 
00416   TestRewriteDriverFactory* factory() { return factory_.get(); }
00417   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00418 
00419   void UseMd5Hasher() {
00420     server_context_->set_hasher(&md5_hasher_);
00421     server_context_->http_cache()->set_hasher(&md5_hasher_);
00422     other_server_context_->set_hasher(&md5_hasher_);
00423     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00424   }
00425 
00426 
00427   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00428                                   ResponseHeaders* header) {
00429     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00430   }
00431 
00432   void SetFetchResponse(const StringPiece& url,
00433                         const ResponseHeaders& response_header,
00434                         const StringPiece& response_body) {
00435     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00436   }
00437 
00439   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00440                                      const ContentType& content_type,
00441                                      const StringPiece& content,
00442                                      int64 ttl_sec);
00443 
00446   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00447 
00449   void AddFileToMockFetcher(const StringPiece& url,
00450                             const StringPiece& filename,
00451                             const ContentType& content_type, int64 ttl_sec);
00452 
00453   void AddToResponse(const StringPiece& url,
00454                      const StringPiece& name,
00455                      const StringPiece& value) {
00456     mock_url_fetcher()->AddToResponse(url, name, value);
00457   }
00458 
00459   void SetFetchResponse404(const StringPiece& url);
00460 
00461   void SetFetchFailOnUnexpected(bool fail) {
00462     mock_url_fetcher()->set_fail_on_unexpected(fail);
00463   }
00464   void FetcherUpdateDateHeaders() {
00465     mock_url_fetcher()->set_timer(timer());
00466     mock_url_fetcher()->set_update_date_headers(true);
00467   }
00468   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00469 
00470   virtual void ClearStats();
00471 
00474   void ClearRewriteDriver();
00475 
00476   MockUrlFetcher* mock_url_fetcher() {
00477     return &mock_url_fetcher_;
00478   }
00479   TestDistributedFetcher* test_distributed_fetcher() {
00480     return &test_distributed_fetcher_;
00481   }
00482   Hasher* hasher() { return server_context_->hasher(); }
00483   DelayCache* delay_cache() { return factory_->delay_cache(); }
00484   LRUCache* lru_cache() { return factory_->lru_cache(); }
00485   Statistics* statistics() { return factory_->statistics(); }
00486   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00487   HTTPCache* http_cache() { return server_context_->http_cache(); }
00488   PropertyCache* page_property_cache() {
00489     return server_context_->page_property_cache();
00490   }
00491   MockMessageHandler* message_handler() {
00492     return factory_->mock_message_handler();
00493   }
00494 
00502   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00503   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00504 
00506   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00507 
00508   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00509 
00510   bool ReadFile(const char* filename, GoogleString* contents) {
00511     return file_system()->ReadFile(filename, contents, message_handler());
00512   }
00513   bool WriteFile(const char* filename, const StringPiece& contents) {
00514     return file_system()->WriteFile(filename, contents, message_handler());
00515   }
00516 
00517   ServerContext* server_context() { return server_context_; }
00518   ServerContext* other_server_context() { return other_server_context_; }
00519   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00520     return factory_->counting_url_async_fetcher();
00521   }
00522   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00523     return factory_->counting_distributed_async_fetcher();
00524   }
00525   void SetMockHashValue(const GoogleString& value) {
00526     factory_->mock_hasher()->set_hash_value(value);
00527   }
00528 
00529   void SetCacheDelayUs(int64 delay_us);
00530 
00535   RewriteDriver* MakeDriver(ServerContext* server_context,
00536                             RewriteOptions* options);
00537 
00539   GoogleString AbsolutifyUrl(const StringPiece& in);
00540 
00545   void TestRetainExtraHeaders(const StringPiece& name,
00546                               const StringPiece& filter_id,
00547                               const StringPiece& ext);
00548 
00552   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00553 
00555   void SetUseTestUrlNamer(bool use_test_url_namer);
00556 
00559   GoogleString EncodeCssName(const StringPiece& name,
00560                              bool supports_webp,
00561                              bool can_inline);
00562 
00566   bool ReadIfCached(const ResourcePtr& resource);
00567 
00572   void InitiateResourceRead(const ResourcePtr& resource);
00573 
00577   HTTPCache::FindResult HttpBlockingFind(
00578       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00579       ResponseHeaders* headers);
00580 
00583   HTTPCache::FindResult HttpBlockingFindStatus(
00584       const GoogleString& key, HTTPCache* http_cache);
00585 
00587   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00588 
00590   void SetHtmlMimetype() { SetMimetype("text/html"); }
00591 
00593   void SetMimetype(const StringPiece& mimetype);
00594 
00597   void CheckFetchFromHttpCache(
00598       StringPiece url,
00599       StringPiece expected_contents,
00600       int64 expected_expiration_ms);
00601 
00603   const PropertyCache::Cohort*  SetupCohort(
00604       PropertyCache* cache, const GoogleString& cohort) {
00605     return factory()->SetupCohort(cache, cohort);
00606   }
00607 
00610   void SetupSharedCache();
00611 
00613   MockPropertyPage* NewMockPage(const StringPiece& url,
00614                                 const StringPiece& options_signature_hash,
00615                                 UserAgentMatcher::DeviceType device_type) {
00616     return new MockPropertyPage(
00617         server_context_->thread_system(),
00618         server_context_->page_property_cache(),
00619         url,
00620         options_signature_hash,
00621         UserAgentMatcher::DeviceTypeSuffix(device_type));
00622   }
00623 
00624   MockPropertyPage* NewMockPage(const StringPiece& url) {
00625     return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
00626   }
00627 
00629   void SetMockLogRecord();
00630 
00632   MockLogRecord* mock_log_record();
00633 
00635   GoogleString GetLazyloadScriptHtml();
00636   GoogleString GetLazyloadPostscriptHtml();
00637 
00643   void SetCacheInvalidationTimestamp();
00644 
00647   void SetCacheInvalidationTimestampForUrl(
00648       StringPiece url, bool ignores_metadata_and_pcache);
00649 
00653   void EnableCachePurge();
00654 
00657   static const ProcessContext& process_context();
00658 
00659  protected:
00660   void Init();
00661 
00664   virtual RequestContextPtr CreateRequestContext();
00665 
00668   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00669                                      RewriteDriver* driver);
00670 
00673   void PopulateDefaultHeaders(const ContentType& content_type,
00674                               int64 original_content_length,
00675                               ResponseHeaders* headers);
00676 
00679   void SetActiveServer(ActiveServerFlag server_to_use);
00680 
00683   void AdvanceTimeUs(int64 delay_ms);
00684   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00685   void SetTimeUs(int64 time_us);
00686   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00687 
00689   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00690 
00692   const RequestContext::TimingInfo& timing_info();
00693   RequestContext::TimingInfo* mutable_timing_info();
00694 
00699   LoggingInfo* logging_info();
00700 
00702   const MetadataCacheInfo& metadata_cache_info() {
00703     return logging_info()->metadata_cache_info();
00704   }
00705 
00708   GoogleString AppliedRewriterStringFromLog();
00709 
00712   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00713                                const GoogleString& id,
00714                                int url_index,
00715                                int rewriter_info_index,
00716                                int rewriter_info_size,
00717                                int url_list_size,
00718                                const GoogleString& url);
00719 
00721   void SetCurrentUserAgent(const StringPiece& user_agent) {
00722     current_user_agent_ = user_agent;
00723   }
00724 
00725   GoogleString ExpectedNonce();
00726 
00729   GoogleString HttpCacheKey(StringPiece url) {
00730     return HTTPCache::CompositeKey(url, rewrite_driver_->CacheFragment());
00731   }
00732 
00735   MockUrlFetcher mock_url_fetcher_;
00736   TestDistributedFetcher test_distributed_fetcher_;
00737   scoped_ptr<Statistics> statistics_;
00738 
00744   scoped_ptr<TestRewriteDriverFactory> factory_;
00745   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00746   ServerContext* server_context_;
00747   RewriteDriver* rewrite_driver_;
00748   ServerContext* other_server_context_;
00749   RewriteDriver* other_rewrite_driver_;
00750   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00751   ActiveServerFlag active_server_;
00752   bool use_managed_rewrite_drivers_;
00753   StringPiece current_user_agent_;
00754   MD5Hasher md5_hasher_;
00755 
00756   RewriteOptions* options_; 
00757   RewriteOptions* other_options_; 
00758   UrlSegmentEncoder default_encoder_;
00759   ResponseHeaders response_headers_;
00760   const GoogleString kEtag0; 
00761   uint64 expected_nonce_;
00762 };
00763 
00764 }  
00765 
00766 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines