Page Speed Optimization Libraries  1.8.31.3
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/request_context.h"
00033 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/server_context.h"
00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/md5_hasher.h"
00044 #include "net/instaweb/util/public/mem_file_system.h"
00045 #include "net/instaweb/util/public/mock_hasher.h"
00046 #include "net/instaweb/util/public/mock_message_handler.h"
00047 #include "net/instaweb/util/public/mock_property_page.h"
00049 #include "net/instaweb/util/public/mock_timer.h"
00050 #include "net/instaweb/util/public/property_cache.h"
00051 #include "net/instaweb/util/public/scoped_ptr.h"
00052 #include "net/instaweb/util/public/string.h"
00053 #include "net/instaweb/util/public/string_util.h"
00054 #include "net/instaweb/util/public/timer.h"
00055 #include "net/instaweb/util/public/url_segment_encoder.h"
00056 #include "pagespeed/kernel/http/content_type.h"
00057 
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class CountingUrlAsyncFetcher;
00063 class DelayCache;
00064 class HTTPValue;
00065 class Hasher;
00066 class HtmlWriterFilter;
00067 class LRUCache;
00068 class MessageHandler;
00069 class MockLogRecord;
00070 class MockScheduler;
00071 class ProcessContext;
00072 class RequestHeaders;
00073 class ResourceNamer;
00074 class RewriteFilter;
00075 class Statistics;
00076 class WaitUrlAsyncFetcher;
00077 
00078 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00079  protected:
00080   RewriteOptionsTestBase() {
00081     RewriteOptions::Initialize();
00082   }
00083   ~RewriteOptionsTestBase() {
00084     RewriteOptions::Terminate();
00085   }
00086 };
00087 
00088 class RewriteTestBase : public RewriteOptionsTestBase {
00089  public:
00090   static const char kTestData[]; 
00091 
00093   static const char kConfiguredBeaconingKey[];
00094   static const char kWrongBeaconingKey[];
00095 
00102   enum ActiveServerFlag {
00103     kPrimary,    
00104     kSecondary,  
00105   };
00106 
00107   RewriteTestBase();
00108   explicit RewriteTestBase(Statistics* statistics);
00109 
00115   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00116                                      TestRewriteDriverFactory*> factories);
00117   virtual ~RewriteTestBase();
00118 
00119   virtual void SetUp();
00120   virtual void TearDown();
00121 
00125   virtual bool AddBody() const { return false; }
00126 
00136   virtual TestRewriteDriverFactory* MakeTestFactory();
00137 
00140   void AddRecompressImageFilters();
00141 
00143   void AddFilter(RewriteOptions::Filter filter);
00144 
00146   void AddOtherFilter(RewriteOptions::Filter filter);
00147 
00150   void AddRewriteFilter(RewriteFilter* filter);
00151 
00154   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00155 
00158   void AddOtherRewriteFilter(RewriteFilter* filter);
00159 
00162   void SetBaseUrlForFetch(const StringPiece& url);
00163 
00165   void SetDummyRequestHeaders();
00166 
00169   void SetDownstreamCacheDirectives(
00170     StringPiece downstream_cache_purge_method,
00171     StringPiece downstream_cache_purge_location_prefix,
00172     StringPiece rebeaconing_key);
00173 
00175   void SetShouldBeaconHeader(StringPiece rebeaconing_key);
00176 
00177   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00178 
00183   Timer* timer() { return factory()->mock_timer(); }
00184 
00186   void AppendDefaultHeaders(const ContentType& content_type,
00187                             GoogleString* text);
00188 
00191   void AppendDefaultHeaders(const ContentType& content_type,
00192                             int64 original_content_length,
00193                             GoogleString* text);
00194 
00195   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00196                                      const StringPiece& expected_content);
00197 
00198   void ServeResourceFromManyContextsWithUA(
00199       const GoogleString& resource_url,
00200       const StringPiece& expected_content,
00201       const StringPiece& user_agent);
00202 
00205   void ServeResourceFromNewContext(
00206       const GoogleString& resource_url,
00207       const StringPiece& expected_content);
00208 
00212   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00213 
00215   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00216                               ResponseHeaders* response_headers);
00217 
00221   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00222                      const StringPiece& name, const StringPiece& ext,
00223                      GoogleString* content);
00224   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00225                      const StringPiece& name, const StringPiece& ext,
00226                      GoogleString* content, ResponseHeaders* response);
00227 
00228   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00229                         ResponseHeaders* response);
00230   bool FetchResourceUrl(const StringPiece& url,
00231                         RequestHeaders* request_headers,
00232                         GoogleString* content,
00233                         ResponseHeaders* response_headers);
00234   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00235 
00237   bool TryFetchResource(const StringPiece& url);
00238 
00243   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00244 
00245   GoogleString CssLinkHref(const StringPiece& url) {
00246     return StrCat("<link rel=stylesheet href=", url, ">");
00247   }
00248 
00250   class CssLink {
00251    public:
00252     CssLink(const StringPiece& url, const StringPiece& content,
00253             const StringPiece& media, bool supply_mock);
00254 
00256     class Vector : public std::vector<CssLink*> {
00257      public:
00258       ~Vector();
00259       void Add(const StringPiece& url, const StringPiece& content,
00260                const StringPiece& media, bool supply_mock);
00261     };
00262 
00265     bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
00266                               StringVector* segments, MessageHandler* handler);
00267 
00268     GoogleString url_;
00269     GoogleString content_;
00270     GoogleString media_;
00271     bool supply_mock_;
00272   };
00273 
00275   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00276                        StringVector* css_links);
00277 
00279   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00280                        CssLink::Vector* css_links);
00281 
00283   void EncodePathAndLeaf(const StringPiece& filter_id,
00284                          const StringPiece& hash,
00285                          const StringVector& name_vector,
00286                          const StringPiece& ext,
00287                          ResourceNamer* namer);
00288 
00289   StringVector MultiUrl(const StringPiece& url1) {
00290     StringVector v;
00291     v.push_back(url1.as_string());
00292     return v;
00293   }
00294 
00295   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00296     StringVector v;
00297     v.push_back(url1.as_string());
00298     v.push_back(url2.as_string());
00299     return v;
00300   }
00301 
00302   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00303                         const StringPiece& url3) {
00304     StringVector v;
00305     v.push_back(url1.as_string());
00306     v.push_back(url2.as_string());
00307     v.push_back(url3.as_string());
00308     return v;
00309   }
00310 
00311   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00312                         const StringPiece& url3, const StringPiece& url4) {
00313     StringVector v;
00314     v.push_back(url1.as_string());
00315     v.push_back(url2.as_string());
00316     v.push_back(url3.as_string());
00317     v.push_back(url4.as_string());
00318     return v;
00319   }
00320 
00323   GoogleString Encode(const StringPiece& path,
00324                       const StringPiece& filter_id,
00325                       const StringPiece& hash,
00326                       const StringPiece& name,
00327                       const StringPiece& ext) {
00328     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00329   }
00330   GoogleString Encode(const StringPiece& path,
00331                       const StringPiece& filter_id,
00332                       const StringPiece& hash,
00333                       const StringVector& name_vector,
00334                       const StringPiece& ext);
00335 
00337   GoogleString EncodeNormal(const StringPiece& path,
00338                             const StringPiece& filter_id,
00339                             const StringPiece& hash,
00340                             const StringPiece& name,
00341                             const StringPiece& ext) {
00342     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00343   }
00344   GoogleString EncodeNormal(const StringPiece& path,
00345                             const StringPiece& filter_id,
00346                             const StringPiece& hash,
00347                             const StringVector& name_vector,
00348                             const StringPiece& ext);
00349 
00352   GoogleString EncodeWithBase(const StringPiece& base,
00353                               const StringPiece& path,
00354                               const StringPiece& filter_id,
00355                               const StringPiece& hash,
00356                               const StringPiece& name,
00357                               const StringPiece& ext) {
00358     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00359   }
00360   GoogleString EncodeWithBase(const StringPiece& base,
00361                               const StringPiece& path,
00362                               const StringPiece& filter_id,
00363                               const StringPiece& hash,
00364                               const StringVector& name_vector,
00365                               const StringPiece& ext);
00366 
00368   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00369                                       const StringPiece& options);
00370 
00375   static GoogleString ChangeSuffix(
00376       StringPiece old_url, bool append_new_suffix,
00377       StringPiece old_suffix, StringPiece new_suffix);
00378 
00383   void SetupWaitFetcher();
00384   void CallFetcherCallbacks();
00385   void OtherCallFetcherCallbacks();
00386   RewriteOptions* options() { return options_; }
00387   RewriteOptions* other_options() { return other_options_; }
00388 
00390   void SetRewriteOptions(RewriteOptions* opts);
00391 
00394   bool AddDomain(StringPiece domain);
00395 
00398   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00399 
00402   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00403 
00406   bool AddShard(StringPiece domain, StringPiece shards);
00407 
00409   void TestServeFiles(const ContentType* content_type,
00410                       const StringPiece& filter_id,
00411                       const StringPiece& rewritten_ext,
00412                       const StringPiece& orig_name,
00413                       const StringPiece& orig_content,
00414                       const StringPiece& rewritten_name,
00415                       const StringPiece& rewritten_content);
00416 
00417   TestRewriteDriverFactory* factory() { return factory_.get(); }
00418   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00419 
00420   void UseMd5Hasher() {
00421     server_context_->set_hasher(&md5_hasher_);
00422     server_context_->http_cache()->set_hasher(&md5_hasher_);
00423     other_server_context_->set_hasher(&md5_hasher_);
00424     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00425   }
00426 
00427 
00428   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00429                                   ResponseHeaders* header) {
00430     server_context_->SetDefaultLongCacheHeaders(
00431         content_type, StringPiece(), StringPiece(), header);
00432   }
00433 
00434   void SetFetchResponse(const StringPiece& url,
00435                         const ResponseHeaders& response_header,
00436                         const StringPiece& response_body) {
00437     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00438   }
00439 
00441   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00442                                      const ContentType& content_type,
00443                                      const StringPiece& content,
00444                                      int64 ttl_sec);
00445 
00448   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00449 
00451   void AddFileToMockFetcher(const StringPiece& url,
00452                             const StringPiece& filename,
00453                             const ContentType& content_type, int64 ttl_sec);
00454 
00455   void AddToResponse(const StringPiece& url,
00456                      const StringPiece& name,
00457                      const StringPiece& value) {
00458     mock_url_fetcher()->AddToResponse(url, name, value);
00459   }
00460 
00461   void SetFetchResponse404(const StringPiece& url);
00462 
00463   void SetFetchFailOnUnexpected(bool fail) {
00464     mock_url_fetcher()->set_fail_on_unexpected(fail);
00465   }
00466   void FetcherUpdateDateHeaders() {
00467     mock_url_fetcher()->set_timer(timer());
00468     mock_url_fetcher()->set_update_date_headers(true);
00469   }
00470   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00471 
00472   virtual void ClearStats();
00473 
00476   void ClearRewriteDriver();
00477 
00478   MockUrlFetcher* mock_url_fetcher() {
00479     return &mock_url_fetcher_;
00480   }
00481   TestDistributedFetcher* test_distributed_fetcher() {
00482     return &test_distributed_fetcher_;
00483   }
00484   Hasher* hasher() { return server_context_->hasher(); }
00485   DelayCache* delay_cache() { return factory_->delay_cache(); }
00486   LRUCache* lru_cache() { return factory_->lru_cache(); }
00487   Statistics* statistics() { return factory_->statistics(); }
00488   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00489   HTTPCache* http_cache() { return server_context_->http_cache(); }
00490   PropertyCache* page_property_cache() {
00491     return server_context_->page_property_cache();
00492   }
00493   MockMessageHandler* message_handler() {
00494     return factory_->mock_message_handler();
00495   }
00496 
00504   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00505   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00506 
00508   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00509 
00510   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00511 
00512   bool ReadFile(const char* filename, GoogleString* contents) {
00513     return file_system()->ReadFile(filename, contents, message_handler());
00514   }
00515   bool WriteFile(const char* filename, const StringPiece& contents) {
00516     return file_system()->WriteFile(filename, contents, message_handler());
00517   }
00518 
00519   ServerContext* server_context() { return server_context_; }
00520   ServerContext* other_server_context() { return other_server_context_; }
00521   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00522     return factory_->counting_url_async_fetcher();
00523   }
00524   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00525     return factory_->counting_distributed_async_fetcher();
00526   }
00527   void SetMockHashValue(const GoogleString& value) {
00528     factory_->mock_hasher()->set_hash_value(value);
00529   }
00530 
00531   void SetCacheDelayUs(int64 delay_us);
00532 
00537   RewriteDriver* MakeDriver(ServerContext* server_context,
00538                             RewriteOptions* options);
00539 
00541   GoogleString AbsolutifyUrl(const StringPiece& in);
00542 
00547   void TestRetainExtraHeaders(const StringPiece& name,
00548                               const StringPiece& filter_id,
00549                               const StringPiece& ext);
00550 
00554   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00555 
00557   void SetUseTestUrlNamer(bool use_test_url_namer);
00558 
00561   GoogleString EncodeCssName(const StringPiece& name,
00562                              bool supports_webp,
00563                              bool can_inline);
00564 
00568   bool ReadIfCached(const ResourcePtr& resource);
00569 
00574   void InitiateResourceRead(const ResourcePtr& resource);
00575 
00579   HTTPCache::FindResult HttpBlockingFind(
00580       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00581       ResponseHeaders* headers);
00582 
00585   HTTPCache::FindResult HttpBlockingFindStatus(
00586       const GoogleString& key, HTTPCache* http_cache);
00587 
00589   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00590 
00592   void SetHtmlMimetype() { SetMimetype("text/html"); }
00593 
00595   void SetMimetype(const StringPiece& mimetype);
00596 
00599   void CheckFetchFromHttpCache(
00600       StringPiece url,
00601       StringPiece expected_contents,
00602       int64 expected_expiration_ms);
00603 
00605   const PropertyCache::Cohort*  SetupCohort(
00606       PropertyCache* cache, const GoogleString& cohort) {
00607     return factory()->SetupCohort(cache, cohort);
00608   }
00609 
00612   void SetupSharedCache();
00613 
00615   MockPropertyPage* NewMockPage(const StringPiece& url,
00616                                 const StringPiece& options_signature_hash,
00617                                 UserAgentMatcher::DeviceType device_type) {
00618     return new MockPropertyPage(
00619         server_context_->thread_system(),
00620         server_context_->page_property_cache(),
00621         url,
00622         options_signature_hash,
00623         UserAgentMatcher::DeviceTypeSuffix(device_type));
00624   }
00625 
00626   MockPropertyPage* NewMockPage(const StringPiece& url) {
00627     return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
00628   }
00629 
00631   void SetMockLogRecord();
00632 
00634   MockLogRecord* mock_log_record();
00635 
00637   GoogleString GetLazyloadScriptHtml();
00638   GoogleString GetLazyloadPostscriptHtml();
00639 
00645   void SetCacheInvalidationTimestamp();
00646 
00649   void SetCacheInvalidationTimestampForUrl(
00650       StringPiece url, bool ignores_metadata_and_pcache);
00651 
00655   void EnableCachePurge();
00656 
00659   static const ProcessContext& process_context();
00660 
00661  protected:
00662   void Init();
00663 
00666   virtual RequestContextPtr CreateRequestContext();
00667 
00670   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00671                                      RewriteDriver* driver);
00672 
00675   void PopulateDefaultHeaders(const ContentType& content_type,
00676                               int64 original_content_length,
00677                               ResponseHeaders* headers);
00678 
00681   void SetActiveServer(ActiveServerFlag server_to_use);
00682 
00685   void AdvanceTimeUs(int64 delay_ms);
00686   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00687   void SetTimeUs(int64 time_us);
00688   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00689 
00691   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00692 
00694   const RequestContext::TimingInfo& timing_info();
00695   RequestContext::TimingInfo* mutable_timing_info();
00696 
00701   LoggingInfo* logging_info();
00702 
00704   const MetadataCacheInfo& metadata_cache_info() {
00705     return logging_info()->metadata_cache_info();
00706   }
00707 
00710   GoogleString AppliedRewriterStringFromLog();
00711 
00714   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00715                                const GoogleString& id,
00716                                int url_index,
00717                                int rewriter_info_index,
00718                                int rewriter_info_size,
00719                                int url_list_size,
00720                                const GoogleString& url);
00721 
00723   void SetCurrentUserAgent(const StringPiece& user_agent) {
00724     current_user_agent_ = user_agent;
00725   }
00726 
00727   GoogleString ExpectedNonce();
00728 
00731   GoogleString HttpCacheKey(StringPiece url) {
00732     return HTTPCache::CompositeKey(url, rewrite_driver_->CacheFragment());
00733   }
00734 
00736   int TimedValue(StringPiece name);
00737 
00740   MockUrlFetcher mock_url_fetcher_;
00741   TestDistributedFetcher test_distributed_fetcher_;
00742   scoped_ptr<Statistics> statistics_;
00743 
00749   scoped_ptr<TestRewriteDriverFactory> factory_;
00750   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00751   ServerContext* server_context_;
00752   RewriteDriver* rewrite_driver_;
00753   ServerContext* other_server_context_;
00754   RewriteDriver* other_rewrite_driver_;
00755   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00756   ActiveServerFlag active_server_;
00757   bool use_managed_rewrite_drivers_;
00758   StringPiece current_user_agent_;
00759   MD5Hasher md5_hasher_;
00760 
00761   RewriteOptions* options_; 
00762   RewriteOptions* other_options_; 
00763   UrlSegmentEncoder default_encoder_;
00764   ResponseHeaders response_headers_;
00765   const GoogleString kEtag0; 
00766   uint64 expected_nonce_;
00767 };
00768 
00769 }  
00770 
00771 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines