Page Speed Optimization Libraries  1.7.30.1
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/request_context.h"
00033 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/server_context.h"
00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/md5_hasher.h"
00044 #include "net/instaweb/util/public/mem_file_system.h"
00045 #include "net/instaweb/util/public/mock_hasher.h"
00046 #include "net/instaweb/util/public/mock_message_handler.h"
00047 #include "net/instaweb/util/public/mock_property_page.h"
00049 #include "net/instaweb/util/public/mock_timer.h"
00050 #include "net/instaweb/util/public/property_cache.h"
00051 #include "net/instaweb/util/public/scoped_ptr.h"
00052 #include "net/instaweb/util/public/string.h"
00053 #include "net/instaweb/util/public/string_util.h"
00054 #include "net/instaweb/util/public/timer.h"
00055 #include "net/instaweb/util/public/url_segment_encoder.h"
00056 #include "pagespeed/kernel/http/content_type.h"
00057 
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class CountingUrlAsyncFetcher;
00063 class DelayCache;
00064 class HTTPValue;
00065 class Hasher;
00066 class HtmlWriterFilter;
00067 class LRUCache;
00068 class MessageHandler;
00069 class MockLogRecord;
00070 class MockScheduler;
00071 class RequestHeaders;
00072 class ResourceNamer;
00073 class RewriteFilter;
00074 class Statistics;
00075 class WaitUrlAsyncFetcher;
00076 
00077 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00078  protected:
00079   RewriteOptionsTestBase() {
00080     RewriteOptions::Initialize();
00081   }
00082   ~RewriteOptionsTestBase() {
00083     RewriteOptions::Terminate();
00084   }
00085 };
00086 
00087 class RewriteTestBase : public RewriteOptionsTestBase {
00088  public:
00089   static const char kTestData[]; 
00090 
00091 
00092 
00093 
00094 
00095 
00096   enum ActiveServerFlag {
00097     kPrimary,    
00098     kSecondary,  
00099   };
00100 
00101   RewriteTestBase();
00102   explicit RewriteTestBase(Statistics* statistics);
00103 
00109   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00110                                      TestRewriteDriverFactory*> factories);
00111   virtual ~RewriteTestBase();
00112 
00113   virtual void SetUp();
00114   virtual void TearDown();
00115 
00119   virtual bool AddBody() const { return false; }
00120 
00130   virtual TestRewriteDriverFactory* MakeTestFactory();
00131 
00134   void AddRecompressImageFilters();
00135 
00137   void AddFilter(RewriteOptions::Filter filter);
00138 
00140   void AddOtherFilter(RewriteOptions::Filter filter);
00141 
00144   void AddRewriteFilter(RewriteFilter* filter);
00145 
00148   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00149 
00152   void AddOtherRewriteFilter(RewriteFilter* filter);
00153 
00156   void SetBaseUrlForFetch(const StringPiece& url);
00157 
00158   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00159 
00164   Timer* timer() { return factory()->mock_timer(); }
00165 
00167   void AppendDefaultHeaders(const ContentType& content_type,
00168                             GoogleString* text);
00169 
00172   void AppendDefaultHeaders(const ContentType& content_type,
00173                             int64 original_content_length,
00174                             GoogleString* text);
00175 
00176   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00177                                      const StringPiece& expected_content);
00178 
00179   void ServeResourceFromManyContextsWithUA(
00180       const GoogleString& resource_url,
00181       const StringPiece& expected_content,
00182       const StringPiece& user_agent);
00183 
00186   void ServeResourceFromNewContext(
00187       const GoogleString& resource_url,
00188       const StringPiece& expected_content);
00189 
00193   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00194 
00196   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00197                               ResponseHeaders* response_headers);
00198 
00202   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00203                      const StringPiece& name, const StringPiece& ext,
00204                      GoogleString* content);
00205   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00206                      const StringPiece& name, const StringPiece& ext,
00207                      GoogleString* content, ResponseHeaders* response);
00208 
00209   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00210                         ResponseHeaders* response);
00211   bool FetchResourceUrl(const StringPiece& url,
00212                         RequestHeaders* request_headers,
00213                         GoogleString* content,
00214                         ResponseHeaders* response_headers);
00215   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00216 
00218   bool TryFetchResource(const StringPiece& url);
00219 
00224   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00225 
00226   GoogleString CssLinkHref(const StringPiece& url) {
00227     return StrCat("<link rel=stylesheet href=", url, ">");
00228   }
00229 
00231   class CssLink {
00232    public:
00233     CssLink(const StringPiece& url, const StringPiece& content,
00234             const StringPiece& media, bool supply_mock);
00235 
00237     class Vector : public std::vector<CssLink*> {
00238      public:
00239       ~Vector();
00240       void Add(const StringPiece& url, const StringPiece& content,
00241                const StringPiece& media, bool supply_mock);
00242     };
00243 
00246     bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
00247                               StringVector* segments, MessageHandler* handler);
00248 
00249     GoogleString url_;
00250     GoogleString content_;
00251     GoogleString media_;
00252     bool supply_mock_;
00253   };
00254 
00256   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00257                        StringVector* css_links);
00258 
00260   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00261                        CssLink::Vector* css_links);
00262 
00264   void EncodePathAndLeaf(const StringPiece& filter_id,
00265                          const StringPiece& hash,
00266                          const StringVector& name_vector,
00267                          const StringPiece& ext,
00268                          ResourceNamer* namer);
00269 
00270   StringVector MultiUrl(const StringPiece& url1) {
00271     StringVector v;
00272     v.push_back(url1.as_string());
00273     return v;
00274   }
00275 
00276   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00277     StringVector v;
00278     v.push_back(url1.as_string());
00279     v.push_back(url2.as_string());
00280     return v;
00281   }
00282 
00283   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00284                         const StringPiece& url3) {
00285     StringVector v;
00286     v.push_back(url1.as_string());
00287     v.push_back(url2.as_string());
00288     v.push_back(url3.as_string());
00289     return v;
00290   }
00291 
00292   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00293                         const StringPiece& url3, const StringPiece& url4) {
00294     StringVector v;
00295     v.push_back(url1.as_string());
00296     v.push_back(url2.as_string());
00297     v.push_back(url3.as_string());
00298     v.push_back(url4.as_string());
00299     return v;
00300   }
00301 
00304   GoogleString Encode(const StringPiece& path,
00305                       const StringPiece& filter_id,
00306                       const StringPiece& hash,
00307                       const StringPiece& name,
00308                       const StringPiece& ext) {
00309     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00310   }
00311   GoogleString Encode(const StringPiece& path,
00312                       const StringPiece& filter_id,
00313                       const StringPiece& hash,
00314                       const StringVector& name_vector,
00315                       const StringPiece& ext);
00316 
00318   GoogleString EncodeNormal(const StringPiece& path,
00319                             const StringPiece& filter_id,
00320                             const StringPiece& hash,
00321                             const StringPiece& name,
00322                             const StringPiece& ext) {
00323     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00324   }
00325   GoogleString EncodeNormal(const StringPiece& path,
00326                             const StringPiece& filter_id,
00327                             const StringPiece& hash,
00328                             const StringVector& name_vector,
00329                             const StringPiece& ext);
00330 
00333   GoogleString EncodeWithBase(const StringPiece& base,
00334                               const StringPiece& path,
00335                               const StringPiece& filter_id,
00336                               const StringPiece& hash,
00337                               const StringPiece& name,
00338                               const StringPiece& ext) {
00339     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00340   }
00341   GoogleString EncodeWithBase(const StringPiece& base,
00342                               const StringPiece& path,
00343                               const StringPiece& filter_id,
00344                               const StringPiece& hash,
00345                               const StringVector& name_vector,
00346                               const StringPiece& ext);
00347 
00349   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00350                                       const StringPiece& options);
00351 
00356   static GoogleString ChangeSuffix(
00357       StringPiece old_url, bool append_new_suffix,
00358       StringPiece old_suffix, StringPiece new_suffix);
00359 
00364   void SetupWaitFetcher();
00365   void CallFetcherCallbacks();
00366   void OtherCallFetcherCallbacks();
00367   RewriteOptions* options() { return options_; }
00368   RewriteOptions* other_options() { return other_options_; }
00369 
00371   void SetRewriteOptions(RewriteOptions* opts);
00372 
00375   bool AddDomain(StringPiece domain);
00376 
00379   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00380 
00383   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00384 
00387   bool AddShard(StringPiece domain, StringPiece shards);
00388 
00390   void TestServeFiles(const ContentType* content_type,
00391                       const StringPiece& filter_id,
00392                       const StringPiece& rewritten_ext,
00393                       const StringPiece& orig_name,
00394                       const StringPiece& orig_content,
00395                       const StringPiece& rewritten_name,
00396                       const StringPiece& rewritten_content);
00397 
00398   TestRewriteDriverFactory* factory() { return factory_.get(); }
00399   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00400 
00401   void UseMd5Hasher() {
00402     server_context_->set_hasher(&md5_hasher_);
00403     server_context_->http_cache()->set_hasher(&md5_hasher_);
00404     other_server_context_->set_hasher(&md5_hasher_);
00405     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00406   }
00407 
00408 
00409   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00410                                   ResponseHeaders* header) {
00411     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00412   }
00413 
00414   void SetFetchResponse(const StringPiece& url,
00415                         const ResponseHeaders& response_header,
00416                         const StringPiece& response_body) {
00417     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00418   }
00419 
00421   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00422                                      const ContentType& content_type,
00423                                      const StringPiece& content,
00424                                      int64 ttl_sec);
00425 
00428   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00429 
00431   void AddFileToMockFetcher(const StringPiece& url,
00432                             const StringPiece& filename,
00433                             const ContentType& content_type, int64 ttl_sec);
00434 
00435   void AddToResponse(const StringPiece& url,
00436                      const StringPiece& name,
00437                      const StringPiece& value) {
00438     mock_url_fetcher()->AddToResponse(url, name, value);
00439   }
00440 
00441   void SetFetchResponse404(const StringPiece& url);
00442 
00443   void SetFetchFailOnUnexpected(bool fail) {
00444     mock_url_fetcher()->set_fail_on_unexpected(fail);
00445   }
00446   void FetcherUpdateDateHeaders() {
00447     mock_url_fetcher()->set_timer(timer());
00448     mock_url_fetcher()->set_update_date_headers(true);
00449   }
00450   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00451 
00452   virtual void ClearStats();
00453 
00456   void ClearRewriteDriver();
00457 
00458   MockUrlFetcher* mock_url_fetcher() {
00459     return &mock_url_fetcher_;
00460   }
00461   TestDistributedFetcher* test_distributed_fetcher() {
00462     return &test_distributed_fetcher_;
00463   }
00464   Hasher* hasher() { return server_context_->hasher(); }
00465   DelayCache* delay_cache() { return factory_->delay_cache(); }
00466   LRUCache* lru_cache() { return factory_->lru_cache(); }
00467   Statistics* statistics() { return factory_->statistics(); }
00468   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00469   HTTPCache* http_cache() { return server_context_->http_cache(); }
00470   PropertyCache* page_property_cache() {
00471     return server_context_->page_property_cache();
00472   }
00473   MockMessageHandler* message_handler() {
00474     return factory_->mock_message_handler();
00475   }
00476 
00484   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00485   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00486 
00488   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00489 
00490   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00491 
00492   bool ReadFile(const char* filename, GoogleString* contents) {
00493     return file_system()->ReadFile(filename, contents, message_handler());
00494   }
00495   bool WriteFile(const char* filename, const StringPiece& contents) {
00496     return file_system()->WriteFile(filename, contents, message_handler());
00497   }
00498 
00499   ServerContext* server_context() { return server_context_; }
00500   ServerContext* other_server_context() { return other_server_context_; }
00501   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00502     return factory_->counting_url_async_fetcher();
00503   }
00504   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00505     return factory_->counting_distributed_async_fetcher();
00506   }
00507   void SetMockHashValue(const GoogleString& value) {
00508     factory_->mock_hasher()->set_hash_value(value);
00509   }
00510 
00511   void SetCacheDelayUs(int64 delay_us);
00512 
00517   RewriteDriver* MakeDriver(ServerContext* server_context,
00518                             RewriteOptions* options);
00519 
00521   GoogleString AbsolutifyUrl(const StringPiece& in);
00522 
00527   void TestRetainExtraHeaders(const StringPiece& name,
00528                               const StringPiece& filter_id,
00529                               const StringPiece& ext);
00530 
00534   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00535 
00537   void SetUseTestUrlNamer(bool use_test_url_namer);
00538 
00541   GoogleString EncodeCssName(const StringPiece& name,
00542                              bool supports_webp,
00543                              bool can_inline);
00544 
00548   bool ReadIfCached(const ResourcePtr& resource);
00549 
00554   void InitiateResourceRead(const ResourcePtr& resource);
00555 
00559   HTTPCache::FindResult HttpBlockingFind(
00560       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00561       ResponseHeaders* headers);
00562 
00565   HTTPCache::FindResult HttpBlockingFindStatus(
00566       const GoogleString& key, HTTPCache* http_cache);
00567 
00569   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00570 
00572   void SetHtmlMimetype() { SetMimetype("text/html"); }
00573 
00575   void SetMimetype(const StringPiece& mimetype);
00576 
00579   void CheckFetchFromHttpCache(
00580       StringPiece url,
00581       StringPiece expected_contents,
00582       int64 expected_expiration_ms);
00583 
00585   const PropertyCache::Cohort*  SetupCohort(
00586       PropertyCache* cache, const GoogleString& cohort) {
00587     return factory()->SetupCohort(cache, cohort);
00588   }
00589 
00592   void SetupSharedCache();
00593 
00595   MockPropertyPage* NewMockPage(const StringPiece& url,
00596                                 const StringPiece& options_signature_hash,
00597                                 UserAgentMatcher::DeviceType device_type) {
00598     return new MockPropertyPage(
00599         server_context_->thread_system(),
00600         server_context_->page_property_cache(),
00601         url,
00602         options_signature_hash,
00603         UserAgentMatcher::DeviceTypeSuffix(device_type));
00604   }
00605 
00606   MockPropertyPage* NewMockPage(const StringPiece& url) {
00607     return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
00608   }
00609 
00611   void SetMockLogRecord();
00612 
00614   MockLogRecord* mock_log_record();
00615 
00617   GoogleString GetLazyloadScriptHtml();
00618   GoogleString GetLazyloadPostscriptHtml();
00619 
00625   void SetCacheInvalidationTimestamp();
00626 
00629   void SetCacheInvalidationTimestampForUrl(
00630       StringPiece url, bool ignores_metadata_and_pcache);
00631 
00635   void EnableCachePurge();
00636 
00637  protected:
00638   void Init();
00639 
00642   virtual RequestContextPtr CreateRequestContext();
00643 
00646   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00647                                      RewriteDriver* driver);
00648 
00651   void PopulateDefaultHeaders(const ContentType& content_type,
00652                               int64 original_content_length,
00653                               ResponseHeaders* headers);
00654 
00657   void SetActiveServer(ActiveServerFlag server_to_use);
00658 
00661   void AdvanceTimeUs(int64 delay_ms);
00662   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00663   void SetTimeUs(int64 time_us);
00664   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00665 
00667   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00668 
00670   const RequestContext::TimingInfo& timing_info();
00671   RequestContext::TimingInfo* mutable_timing_info();
00672 
00677   LoggingInfo* logging_info();
00678 
00680   const MetadataCacheInfo& metadata_cache_info() {
00681     return logging_info()->metadata_cache_info();
00682   }
00683 
00686   GoogleString AppliedRewriterStringFromLog();
00687 
00690   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00691                                const GoogleString& id,
00692                                int url_index,
00693                                int rewriter_info_index,
00694                                int rewriter_info_size,
00695                                int url_list_size,
00696                                const GoogleString& url);
00697 
00699   void SetCurrentUserAgent(const StringPiece& user_agent) {
00700     current_user_agent_ = user_agent;
00701   }
00702 
00703   GoogleString ExpectedNonce();
00704 
00707   MockUrlFetcher mock_url_fetcher_;
00708   TestDistributedFetcher test_distributed_fetcher_;
00709   scoped_ptr<Statistics> statistics_;
00710 
00716   scoped_ptr<TestRewriteDriverFactory> factory_;
00717   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00718   ServerContext* server_context_;
00719   RewriteDriver* rewrite_driver_;
00720   ServerContext* other_server_context_;
00721   RewriteDriver* other_rewrite_driver_;
00722   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00723   ActiveServerFlag active_server_;
00724   bool use_managed_rewrite_drivers_;
00725   StringPiece current_user_agent_;
00726   MD5Hasher md5_hasher_;
00727 
00728   RewriteOptions* options_; 
00729   RewriteOptions* other_options_; 
00730   UrlSegmentEncoder default_encoder_;
00731   ResponseHeaders response_headers_;
00732   const GoogleString kEtag0; 
00733   uint64 expected_nonce_;
00734 };
00735 
00736 }  
00737 
00738 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines