Page Speed Optimization Libraries  1.7.30.2
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/request_context.h"
00033 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/server_context.h"
00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/md5_hasher.h"
00044 #include "net/instaweb/util/public/mem_file_system.h"
00045 #include "net/instaweb/util/public/mock_hasher.h"
00046 #include "net/instaweb/util/public/mock_message_handler.h"
00047 #include "net/instaweb/util/public/mock_property_page.h"
00049 #include "net/instaweb/util/public/mock_timer.h"
00050 #include "net/instaweb/util/public/property_cache.h"
00051 #include "net/instaweb/util/public/scoped_ptr.h"
00052 #include "net/instaweb/util/public/string.h"
00053 #include "net/instaweb/util/public/string_util.h"
00054 #include "net/instaweb/util/public/timer.h"
00055 #include "net/instaweb/util/public/url_segment_encoder.h"
00056 #include "pagespeed/kernel/http/content_type.h"
00057 
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class CountingUrlAsyncFetcher;
00063 class DelayCache;
00064 class HTTPValue;
00065 class Hasher;
00066 class HtmlWriterFilter;
00067 class LRUCache;
00068 class MessageHandler;
00069 class MockLogRecord;
00070 class MockScheduler;
00071 class RequestHeaders;
00072 class ResourceNamer;
00073 class RewriteFilter;
00074 class Statistics;
00075 class WaitUrlAsyncFetcher;
00076 
00077 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00078  protected:
00079   RewriteOptionsTestBase() {
00080     RewriteOptions::Initialize();
00081   }
00082   ~RewriteOptionsTestBase() {
00083     RewriteOptions::Terminate();
00084   }
00085 };
00086 
00087 class RewriteTestBase : public RewriteOptionsTestBase {
00088  public:
00089   static const char kTestData[]; 
00090 
00091 
00092 
00093 
00094 
00095 
00096   enum ActiveServerFlag {
00097     kPrimary,    
00098     kSecondary,  
00099   };
00100 
00101   RewriteTestBase();
00102   explicit RewriteTestBase(Statistics* statistics);
00103 
00109   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00110                                      TestRewriteDriverFactory*> factories);
00111   virtual ~RewriteTestBase();
00112 
00113   virtual void SetUp();
00114   virtual void TearDown();
00115 
00119   virtual bool AddBody() const { return false; }
00120 
00130   virtual TestRewriteDriverFactory* MakeTestFactory();
00131 
00134   void AddRecompressImageFilters();
00135 
00137   void AddFilter(RewriteOptions::Filter filter);
00138 
00140   void AddOtherFilter(RewriteOptions::Filter filter);
00141 
00144   void AddRewriteFilter(RewriteFilter* filter);
00145 
00148   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00149 
00152   void AddOtherRewriteFilter(RewriteFilter* filter);
00153 
00156   void SetBaseUrlForFetch(const StringPiece& url);
00157 
00159   void SetDummyRequestHeaders();
00160 
00163   void SetDownstreamCacheDirectives(
00164     StringPiece downstream_cache_location,
00165     StringPiece rebeaconing_key);
00166 
00167   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00168 
00173   Timer* timer() { return factory()->mock_timer(); }
00174 
00176   void AppendDefaultHeaders(const ContentType& content_type,
00177                             GoogleString* text);
00178 
00181   void AppendDefaultHeaders(const ContentType& content_type,
00182                             int64 original_content_length,
00183                             GoogleString* text);
00184 
00185   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00186                                      const StringPiece& expected_content);
00187 
00188   void ServeResourceFromManyContextsWithUA(
00189       const GoogleString& resource_url,
00190       const StringPiece& expected_content,
00191       const StringPiece& user_agent);
00192 
00195   void ServeResourceFromNewContext(
00196       const GoogleString& resource_url,
00197       const StringPiece& expected_content);
00198 
00202   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00203 
00205   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00206                               ResponseHeaders* response_headers);
00207 
00211   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00212                      const StringPiece& name, const StringPiece& ext,
00213                      GoogleString* content);
00214   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00215                      const StringPiece& name, const StringPiece& ext,
00216                      GoogleString* content, ResponseHeaders* response);
00217 
00218   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00219                         ResponseHeaders* response);
00220   bool FetchResourceUrl(const StringPiece& url,
00221                         RequestHeaders* request_headers,
00222                         GoogleString* content,
00223                         ResponseHeaders* response_headers);
00224   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00225 
00227   bool TryFetchResource(const StringPiece& url);
00228 
00233   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00234 
00235   GoogleString CssLinkHref(const StringPiece& url) {
00236     return StrCat("<link rel=stylesheet href=", url, ">");
00237   }
00238 
00240   class CssLink {
00241    public:
00242     CssLink(const StringPiece& url, const StringPiece& content,
00243             const StringPiece& media, bool supply_mock);
00244 
00246     class Vector : public std::vector<CssLink*> {
00247      public:
00248       ~Vector();
00249       void Add(const StringPiece& url, const StringPiece& content,
00250                const StringPiece& media, bool supply_mock);
00251     };
00252 
00255     bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
00256                               StringVector* segments, MessageHandler* handler);
00257 
00258     GoogleString url_;
00259     GoogleString content_;
00260     GoogleString media_;
00261     bool supply_mock_;
00262   };
00263 
00265   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00266                        StringVector* css_links);
00267 
00269   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00270                        CssLink::Vector* css_links);
00271 
00273   void EncodePathAndLeaf(const StringPiece& filter_id,
00274                          const StringPiece& hash,
00275                          const StringVector& name_vector,
00276                          const StringPiece& ext,
00277                          ResourceNamer* namer);
00278 
00279   StringVector MultiUrl(const StringPiece& url1) {
00280     StringVector v;
00281     v.push_back(url1.as_string());
00282     return v;
00283   }
00284 
00285   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00286     StringVector v;
00287     v.push_back(url1.as_string());
00288     v.push_back(url2.as_string());
00289     return v;
00290   }
00291 
00292   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00293                         const StringPiece& url3) {
00294     StringVector v;
00295     v.push_back(url1.as_string());
00296     v.push_back(url2.as_string());
00297     v.push_back(url3.as_string());
00298     return v;
00299   }
00300 
00301   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00302                         const StringPiece& url3, const StringPiece& url4) {
00303     StringVector v;
00304     v.push_back(url1.as_string());
00305     v.push_back(url2.as_string());
00306     v.push_back(url3.as_string());
00307     v.push_back(url4.as_string());
00308     return v;
00309   }
00310 
00313   GoogleString Encode(const StringPiece& path,
00314                       const StringPiece& filter_id,
00315                       const StringPiece& hash,
00316                       const StringPiece& name,
00317                       const StringPiece& ext) {
00318     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00319   }
00320   GoogleString Encode(const StringPiece& path,
00321                       const StringPiece& filter_id,
00322                       const StringPiece& hash,
00323                       const StringVector& name_vector,
00324                       const StringPiece& ext);
00325 
00327   GoogleString EncodeNormal(const StringPiece& path,
00328                             const StringPiece& filter_id,
00329                             const StringPiece& hash,
00330                             const StringPiece& name,
00331                             const StringPiece& ext) {
00332     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00333   }
00334   GoogleString EncodeNormal(const StringPiece& path,
00335                             const StringPiece& filter_id,
00336                             const StringPiece& hash,
00337                             const StringVector& name_vector,
00338                             const StringPiece& ext);
00339 
00342   GoogleString EncodeWithBase(const StringPiece& base,
00343                               const StringPiece& path,
00344                               const StringPiece& filter_id,
00345                               const StringPiece& hash,
00346                               const StringPiece& name,
00347                               const StringPiece& ext) {
00348     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00349   }
00350   GoogleString EncodeWithBase(const StringPiece& base,
00351                               const StringPiece& path,
00352                               const StringPiece& filter_id,
00353                               const StringPiece& hash,
00354                               const StringVector& name_vector,
00355                               const StringPiece& ext);
00356 
00358   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00359                                       const StringPiece& options);
00360 
00365   static GoogleString ChangeSuffix(
00366       StringPiece old_url, bool append_new_suffix,
00367       StringPiece old_suffix, StringPiece new_suffix);
00368 
00373   void SetupWaitFetcher();
00374   void CallFetcherCallbacks();
00375   void OtherCallFetcherCallbacks();
00376   RewriteOptions* options() { return options_; }
00377   RewriteOptions* other_options() { return other_options_; }
00378 
00380   void SetRewriteOptions(RewriteOptions* opts);
00381 
00384   bool AddDomain(StringPiece domain);
00385 
00388   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00389 
00392   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00393 
00396   bool AddShard(StringPiece domain, StringPiece shards);
00397 
00399   void TestServeFiles(const ContentType* content_type,
00400                       const StringPiece& filter_id,
00401                       const StringPiece& rewritten_ext,
00402                       const StringPiece& orig_name,
00403                       const StringPiece& orig_content,
00404                       const StringPiece& rewritten_name,
00405                       const StringPiece& rewritten_content);
00406 
00407   TestRewriteDriverFactory* factory() { return factory_.get(); }
00408   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00409 
00410   void UseMd5Hasher() {
00411     server_context_->set_hasher(&md5_hasher_);
00412     server_context_->http_cache()->set_hasher(&md5_hasher_);
00413     other_server_context_->set_hasher(&md5_hasher_);
00414     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00415   }
00416 
00417 
00418   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00419                                   ResponseHeaders* header) {
00420     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00421   }
00422 
00423   void SetFetchResponse(const StringPiece& url,
00424                         const ResponseHeaders& response_header,
00425                         const StringPiece& response_body) {
00426     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00427   }
00428 
00430   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00431                                      const ContentType& content_type,
00432                                      const StringPiece& content,
00433                                      int64 ttl_sec);
00434 
00437   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00438 
00440   void AddFileToMockFetcher(const StringPiece& url,
00441                             const StringPiece& filename,
00442                             const ContentType& content_type, int64 ttl_sec);
00443 
00444   void AddToResponse(const StringPiece& url,
00445                      const StringPiece& name,
00446                      const StringPiece& value) {
00447     mock_url_fetcher()->AddToResponse(url, name, value);
00448   }
00449 
00450   void SetFetchResponse404(const StringPiece& url);
00451 
00452   void SetFetchFailOnUnexpected(bool fail) {
00453     mock_url_fetcher()->set_fail_on_unexpected(fail);
00454   }
00455   void FetcherUpdateDateHeaders() {
00456     mock_url_fetcher()->set_timer(timer());
00457     mock_url_fetcher()->set_update_date_headers(true);
00458   }
00459   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00460 
00461   virtual void ClearStats();
00462 
00465   void ClearRewriteDriver();
00466 
00467   MockUrlFetcher* mock_url_fetcher() {
00468     return &mock_url_fetcher_;
00469   }
00470   TestDistributedFetcher* test_distributed_fetcher() {
00471     return &test_distributed_fetcher_;
00472   }
00473   Hasher* hasher() { return server_context_->hasher(); }
00474   DelayCache* delay_cache() { return factory_->delay_cache(); }
00475   LRUCache* lru_cache() { return factory_->lru_cache(); }
00476   Statistics* statistics() { return factory_->statistics(); }
00477   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00478   HTTPCache* http_cache() { return server_context_->http_cache(); }
00479   PropertyCache* page_property_cache() {
00480     return server_context_->page_property_cache();
00481   }
00482   MockMessageHandler* message_handler() {
00483     return factory_->mock_message_handler();
00484   }
00485 
00493   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00494   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00495 
00497   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00498 
00499   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00500 
00501   bool ReadFile(const char* filename, GoogleString* contents) {
00502     return file_system()->ReadFile(filename, contents, message_handler());
00503   }
00504   bool WriteFile(const char* filename, const StringPiece& contents) {
00505     return file_system()->WriteFile(filename, contents, message_handler());
00506   }
00507 
00508   ServerContext* server_context() { return server_context_; }
00509   ServerContext* other_server_context() { return other_server_context_; }
00510   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00511     return factory_->counting_url_async_fetcher();
00512   }
00513   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00514     return factory_->counting_distributed_async_fetcher();
00515   }
00516   void SetMockHashValue(const GoogleString& value) {
00517     factory_->mock_hasher()->set_hash_value(value);
00518   }
00519 
00520   void SetCacheDelayUs(int64 delay_us);
00521 
00526   RewriteDriver* MakeDriver(ServerContext* server_context,
00527                             RewriteOptions* options);
00528 
00530   GoogleString AbsolutifyUrl(const StringPiece& in);
00531 
00536   void TestRetainExtraHeaders(const StringPiece& name,
00537                               const StringPiece& filter_id,
00538                               const StringPiece& ext);
00539 
00543   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00544 
00546   void SetUseTestUrlNamer(bool use_test_url_namer);
00547 
00550   GoogleString EncodeCssName(const StringPiece& name,
00551                              bool supports_webp,
00552                              bool can_inline);
00553 
00557   bool ReadIfCached(const ResourcePtr& resource);
00558 
00563   void InitiateResourceRead(const ResourcePtr& resource);
00564 
00568   HTTPCache::FindResult HttpBlockingFind(
00569       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00570       ResponseHeaders* headers);
00571 
00574   HTTPCache::FindResult HttpBlockingFindStatus(
00575       const GoogleString& key, HTTPCache* http_cache);
00576 
00578   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00579 
00581   void SetHtmlMimetype() { SetMimetype("text/html"); }
00582 
00584   void SetMimetype(const StringPiece& mimetype);
00585 
00588   void CheckFetchFromHttpCache(
00589       StringPiece url,
00590       StringPiece expected_contents,
00591       int64 expected_expiration_ms);
00592 
00594   const PropertyCache::Cohort*  SetupCohort(
00595       PropertyCache* cache, const GoogleString& cohort) {
00596     return factory()->SetupCohort(cache, cohort);
00597   }
00598 
00601   void SetupSharedCache();
00602 
00604   MockPropertyPage* NewMockPage(const StringPiece& url,
00605                                 const StringPiece& options_signature_hash,
00606                                 UserAgentMatcher::DeviceType device_type) {
00607     return new MockPropertyPage(
00608         server_context_->thread_system(),
00609         server_context_->page_property_cache(),
00610         url,
00611         options_signature_hash,
00612         UserAgentMatcher::DeviceTypeSuffix(device_type));
00613   }
00614 
00615   MockPropertyPage* NewMockPage(const StringPiece& url) {
00616     return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
00617   }
00618 
00620   void SetMockLogRecord();
00621 
00623   MockLogRecord* mock_log_record();
00624 
00626   GoogleString GetLazyloadScriptHtml();
00627   GoogleString GetLazyloadPostscriptHtml();
00628 
00634   void SetCacheInvalidationTimestamp();
00635 
00638   void SetCacheInvalidationTimestampForUrl(
00639       StringPiece url, bool ignores_metadata_and_pcache);
00640 
00644   void EnableCachePurge();
00645 
00646  protected:
00647   void Init();
00648 
00651   virtual RequestContextPtr CreateRequestContext();
00652 
00655   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00656                                      RewriteDriver* driver);
00657 
00660   void PopulateDefaultHeaders(const ContentType& content_type,
00661                               int64 original_content_length,
00662                               ResponseHeaders* headers);
00663 
00666   void SetActiveServer(ActiveServerFlag server_to_use);
00667 
00670   void AdvanceTimeUs(int64 delay_ms);
00671   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00672   void SetTimeUs(int64 time_us);
00673   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00674 
00676   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00677 
00679   const RequestContext::TimingInfo& timing_info();
00680   RequestContext::TimingInfo* mutable_timing_info();
00681 
00686   LoggingInfo* logging_info();
00687 
00689   const MetadataCacheInfo& metadata_cache_info() {
00690     return logging_info()->metadata_cache_info();
00691   }
00692 
00695   GoogleString AppliedRewriterStringFromLog();
00696 
00699   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00700                                const GoogleString& id,
00701                                int url_index,
00702                                int rewriter_info_index,
00703                                int rewriter_info_size,
00704                                int url_list_size,
00705                                const GoogleString& url);
00706 
00708   void SetCurrentUserAgent(const StringPiece& user_agent) {
00709     current_user_agent_ = user_agent;
00710   }
00711 
00712   GoogleString ExpectedNonce();
00713 
00716   MockUrlFetcher mock_url_fetcher_;
00717   TestDistributedFetcher test_distributed_fetcher_;
00718   scoped_ptr<Statistics> statistics_;
00719 
00725   scoped_ptr<TestRewriteDriverFactory> factory_;
00726   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00727   ServerContext* server_context_;
00728   RewriteDriver* rewrite_driver_;
00729   ServerContext* other_server_context_;
00730   RewriteDriver* other_rewrite_driver_;
00731   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00732   ActiveServerFlag active_server_;
00733   bool use_managed_rewrite_drivers_;
00734   StringPiece current_user_agent_;
00735   MD5Hasher md5_hasher_;
00736 
00737   RewriteOptions* options_; 
00738   RewriteOptions* other_options_; 
00739   UrlSegmentEncoder default_encoder_;
00740   ResponseHeaders response_headers_;
00741   const GoogleString kEtag0; 
00742   uint64 expected_nonce_;
00743 };
00744 
00745 }  
00746 
00747 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines