Page Speed Optimization Libraries  1.8.31.2
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/request_context.h"
00033 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/http/public/user_agent_matcher.h"
00036 #include "net/instaweb/rewriter/public/resource.h"
00037 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00038 #include "net/instaweb/rewriter/public/rewrite_options.h"
00039 #include "net/instaweb/rewriter/public/server_context.h"
00040 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00041 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00042 #include "net/instaweb/util/public/basictypes.h"
00043 #include "net/instaweb/util/public/md5_hasher.h"
00044 #include "net/instaweb/util/public/mem_file_system.h"
00045 #include "net/instaweb/util/public/mock_hasher.h"
00046 #include "net/instaweb/util/public/mock_message_handler.h"
00047 #include "net/instaweb/util/public/mock_property_page.h"
00049 #include "net/instaweb/util/public/mock_timer.h"
00050 #include "net/instaweb/util/public/property_cache.h"
00051 #include "net/instaweb/util/public/scoped_ptr.h"
00052 #include "net/instaweb/util/public/string.h"
00053 #include "net/instaweb/util/public/string_util.h"
00054 #include "net/instaweb/util/public/timer.h"
00055 #include "net/instaweb/util/public/url_segment_encoder.h"
00056 #include "pagespeed/kernel/http/content_type.h"
00057 
00058 
00059 namespace net_instaweb {
00060 
00061 class AbstractLogRecord;
00062 class CountingUrlAsyncFetcher;
00063 class DelayCache;
00064 class HTTPValue;
00065 class Hasher;
00066 class HtmlWriterFilter;
00067 class LRUCache;
00068 class MessageHandler;
00069 class MockLogRecord;
00070 class MockScheduler;
00071 class ProcessContext;
00072 class RequestHeaders;
00073 class ResourceNamer;
00074 class RewriteFilter;
00075 class Statistics;
00076 class WaitUrlAsyncFetcher;
00077 
00078 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00079  protected:
00080   RewriteOptionsTestBase() {
00081     RewriteOptions::Initialize();
00082   }
00083   ~RewriteOptionsTestBase() {
00084     RewriteOptions::Terminate();
00085   }
00086 };
00087 
00088 class RewriteTestBase : public RewriteOptionsTestBase {
00089  public:
00090   static const char kTestData[]; 
00091 
00093   static const char kConfiguredBeaconingKey[];
00094   static const char kWrongBeaconingKey[];
00095 
00102   enum ActiveServerFlag {
00103     kPrimary,    
00104     kSecondary,  
00105   };
00106 
00107   RewriteTestBase();
00108   explicit RewriteTestBase(Statistics* statistics);
00109 
00115   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00116                                      TestRewriteDriverFactory*> factories);
00117   virtual ~RewriteTestBase();
00118 
00119   virtual void SetUp();
00120   virtual void TearDown();
00121 
00125   virtual bool AddBody() const { return false; }
00126 
00136   virtual TestRewriteDriverFactory* MakeTestFactory();
00137 
00140   void AddRecompressImageFilters();
00141 
00143   void AddFilter(RewriteOptions::Filter filter);
00144 
00146   void AddOtherFilter(RewriteOptions::Filter filter);
00147 
00150   void AddRewriteFilter(RewriteFilter* filter);
00151 
00154   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00155 
00158   void AddOtherRewriteFilter(RewriteFilter* filter);
00159 
00162   void SetBaseUrlForFetch(const StringPiece& url);
00163 
00165   void SetDummyRequestHeaders();
00166 
00169   void SetDownstreamCacheDirectives(
00170     StringPiece downstream_cache_purge_method,
00171     StringPiece downstream_cache_purge_location_prefix,
00172     StringPiece rebeaconing_key);
00173 
00175   void SetShouldBeaconHeader(StringPiece rebeaconing_key);
00176 
00177   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00178 
00183   Timer* timer() { return factory()->mock_timer(); }
00184 
00186   void AppendDefaultHeaders(const ContentType& content_type,
00187                             GoogleString* text);
00188 
00191   void AppendDefaultHeaders(const ContentType& content_type,
00192                             int64 original_content_length,
00193                             GoogleString* text);
00194 
00195   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00196                                      const StringPiece& expected_content);
00197 
00198   void ServeResourceFromManyContextsWithUA(
00199       const GoogleString& resource_url,
00200       const StringPiece& expected_content,
00201       const StringPiece& user_agent);
00202 
00205   void ServeResourceFromNewContext(
00206       const GoogleString& resource_url,
00207       const StringPiece& expected_content);
00208 
00212   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00213 
00215   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00216                               ResponseHeaders* response_headers);
00217 
00221   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00222                      const StringPiece& name, const StringPiece& ext,
00223                      GoogleString* content);
00224   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00225                      const StringPiece& name, const StringPiece& ext,
00226                      GoogleString* content, ResponseHeaders* response);
00227 
00228   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00229                         ResponseHeaders* response);
00230   bool FetchResourceUrl(const StringPiece& url,
00231                         RequestHeaders* request_headers,
00232                         GoogleString* content,
00233                         ResponseHeaders* response_headers);
00234   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00235 
00237   bool TryFetchResource(const StringPiece& url);
00238 
00243   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00244 
00245   GoogleString CssLinkHref(const StringPiece& url) {
00246     return StrCat("<link rel=stylesheet href=", url, ">");
00247   }
00248 
00250   class CssLink {
00251    public:
00252     CssLink(const StringPiece& url, const StringPiece& content,
00253             const StringPiece& media, bool supply_mock);
00254 
00256     class Vector : public std::vector<CssLink*> {
00257      public:
00258       ~Vector();
00259       void Add(const StringPiece& url, const StringPiece& content,
00260                const StringPiece& media, bool supply_mock);
00261     };
00262 
00265     bool DecomposeCombinedUrl(StringPiece base_url, GoogleString* base,
00266                               StringVector* segments, MessageHandler* handler);
00267 
00268     GoogleString url_;
00269     GoogleString content_;
00270     GoogleString media_;
00271     bool supply_mock_;
00272   };
00273 
00275   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00276                        StringVector* css_links);
00277 
00279   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00280                        CssLink::Vector* css_links);
00281 
00283   void EncodePathAndLeaf(const StringPiece& filter_id,
00284                          const StringPiece& hash,
00285                          const StringVector& name_vector,
00286                          const StringPiece& ext,
00287                          ResourceNamer* namer);
00288 
00289   StringVector MultiUrl(const StringPiece& url1) {
00290     StringVector v;
00291     v.push_back(url1.as_string());
00292     return v;
00293   }
00294 
00295   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00296     StringVector v;
00297     v.push_back(url1.as_string());
00298     v.push_back(url2.as_string());
00299     return v;
00300   }
00301 
00302   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00303                         const StringPiece& url3) {
00304     StringVector v;
00305     v.push_back(url1.as_string());
00306     v.push_back(url2.as_string());
00307     v.push_back(url3.as_string());
00308     return v;
00309   }
00310 
00311   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00312                         const StringPiece& url3, const StringPiece& url4) {
00313     StringVector v;
00314     v.push_back(url1.as_string());
00315     v.push_back(url2.as_string());
00316     v.push_back(url3.as_string());
00317     v.push_back(url4.as_string());
00318     return v;
00319   }
00320 
00323   GoogleString Encode(const StringPiece& path,
00324                       const StringPiece& filter_id,
00325                       const StringPiece& hash,
00326                       const StringPiece& name,
00327                       const StringPiece& ext) {
00328     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00329   }
00330   GoogleString Encode(const StringPiece& path,
00331                       const StringPiece& filter_id,
00332                       const StringPiece& hash,
00333                       const StringVector& name_vector,
00334                       const StringPiece& ext);
00335 
00337   GoogleString EncodeNormal(const StringPiece& path,
00338                             const StringPiece& filter_id,
00339                             const StringPiece& hash,
00340                             const StringPiece& name,
00341                             const StringPiece& ext) {
00342     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00343   }
00344   GoogleString EncodeNormal(const StringPiece& path,
00345                             const StringPiece& filter_id,
00346                             const StringPiece& hash,
00347                             const StringVector& name_vector,
00348                             const StringPiece& ext);
00349 
00352   GoogleString EncodeWithBase(const StringPiece& base,
00353                               const StringPiece& path,
00354                               const StringPiece& filter_id,
00355                               const StringPiece& hash,
00356                               const StringPiece& name,
00357                               const StringPiece& ext) {
00358     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00359   }
00360   GoogleString EncodeWithBase(const StringPiece& base,
00361                               const StringPiece& path,
00362                               const StringPiece& filter_id,
00363                               const StringPiece& hash,
00364                               const StringVector& name_vector,
00365                               const StringPiece& ext);
00366 
00368   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00369                                       const StringPiece& options);
00370 
00375   static GoogleString ChangeSuffix(
00376       StringPiece old_url, bool append_new_suffix,
00377       StringPiece old_suffix, StringPiece new_suffix);
00378 
00383   void SetupWaitFetcher();
00384   void CallFetcherCallbacks();
00385   void OtherCallFetcherCallbacks();
00386   RewriteOptions* options() { return options_; }
00387   RewriteOptions* other_options() { return other_options_; }
00388 
00390   void SetRewriteOptions(RewriteOptions* opts);
00391 
00394   bool AddDomain(StringPiece domain);
00395 
00398   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00399 
00402   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00403 
00406   bool AddShard(StringPiece domain, StringPiece shards);
00407 
00409   void TestServeFiles(const ContentType* content_type,
00410                       const StringPiece& filter_id,
00411                       const StringPiece& rewritten_ext,
00412                       const StringPiece& orig_name,
00413                       const StringPiece& orig_content,
00414                       const StringPiece& rewritten_name,
00415                       const StringPiece& rewritten_content);
00416 
00417   TestRewriteDriverFactory* factory() { return factory_.get(); }
00418   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00419 
00420   void UseMd5Hasher() {
00421     server_context_->set_hasher(&md5_hasher_);
00422     server_context_->http_cache()->set_hasher(&md5_hasher_);
00423     other_server_context_->set_hasher(&md5_hasher_);
00424     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00425   }
00426 
00427 
00428   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00429                                   ResponseHeaders* header) {
00430     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00431   }
00432 
00433   void SetFetchResponse(const StringPiece& url,
00434                         const ResponseHeaders& response_header,
00435                         const StringPiece& response_body) {
00436     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00437   }
00438 
00440   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00441                                      const ContentType& content_type,
00442                                      const StringPiece& content,
00443                                      int64 ttl_sec);
00444 
00447   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00448 
00450   void AddFileToMockFetcher(const StringPiece& url,
00451                             const StringPiece& filename,
00452                             const ContentType& content_type, int64 ttl_sec);
00453 
00454   void AddToResponse(const StringPiece& url,
00455                      const StringPiece& name,
00456                      const StringPiece& value) {
00457     mock_url_fetcher()->AddToResponse(url, name, value);
00458   }
00459 
00460   void SetFetchResponse404(const StringPiece& url);
00461 
00462   void SetFetchFailOnUnexpected(bool fail) {
00463     mock_url_fetcher()->set_fail_on_unexpected(fail);
00464   }
00465   void FetcherUpdateDateHeaders() {
00466     mock_url_fetcher()->set_timer(timer());
00467     mock_url_fetcher()->set_update_date_headers(true);
00468   }
00469   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00470 
00471   virtual void ClearStats();
00472 
00475   void ClearRewriteDriver();
00476 
00477   MockUrlFetcher* mock_url_fetcher() {
00478     return &mock_url_fetcher_;
00479   }
00480   TestDistributedFetcher* test_distributed_fetcher() {
00481     return &test_distributed_fetcher_;
00482   }
00483   Hasher* hasher() { return server_context_->hasher(); }
00484   DelayCache* delay_cache() { return factory_->delay_cache(); }
00485   LRUCache* lru_cache() { return factory_->lru_cache(); }
00486   Statistics* statistics() { return factory_->statistics(); }
00487   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00488   HTTPCache* http_cache() { return server_context_->http_cache(); }
00489   PropertyCache* page_property_cache() {
00490     return server_context_->page_property_cache();
00491   }
00492   MockMessageHandler* message_handler() {
00493     return factory_->mock_message_handler();
00494   }
00495 
00503   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00504   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00505 
00507   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00508 
00509   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00510 
00511   bool ReadFile(const char* filename, GoogleString* contents) {
00512     return file_system()->ReadFile(filename, contents, message_handler());
00513   }
00514   bool WriteFile(const char* filename, const StringPiece& contents) {
00515     return file_system()->WriteFile(filename, contents, message_handler());
00516   }
00517 
00518   ServerContext* server_context() { return server_context_; }
00519   ServerContext* other_server_context() { return other_server_context_; }
00520   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00521     return factory_->counting_url_async_fetcher();
00522   }
00523   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00524     return factory_->counting_distributed_async_fetcher();
00525   }
00526   void SetMockHashValue(const GoogleString& value) {
00527     factory_->mock_hasher()->set_hash_value(value);
00528   }
00529 
00530   void SetCacheDelayUs(int64 delay_us);
00531 
00536   RewriteDriver* MakeDriver(ServerContext* server_context,
00537                             RewriteOptions* options);
00538 
00540   GoogleString AbsolutifyUrl(const StringPiece& in);
00541 
00546   void TestRetainExtraHeaders(const StringPiece& name,
00547                               const StringPiece& filter_id,
00548                               const StringPiece& ext);
00549 
00553   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00554 
00556   void SetUseTestUrlNamer(bool use_test_url_namer);
00557 
00560   GoogleString EncodeCssName(const StringPiece& name,
00561                              bool supports_webp,
00562                              bool can_inline);
00563 
00567   bool ReadIfCached(const ResourcePtr& resource);
00568 
00573   void InitiateResourceRead(const ResourcePtr& resource);
00574 
00578   HTTPCache::FindResult HttpBlockingFind(
00579       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00580       ResponseHeaders* headers);
00581 
00584   HTTPCache::FindResult HttpBlockingFindStatus(
00585       const GoogleString& key, HTTPCache* http_cache);
00586 
00588   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00589 
00591   void SetHtmlMimetype() { SetMimetype("text/html"); }
00592 
00594   void SetMimetype(const StringPiece& mimetype);
00595 
00598   void CheckFetchFromHttpCache(
00599       StringPiece url,
00600       StringPiece expected_contents,
00601       int64 expected_expiration_ms);
00602 
00604   const PropertyCache::Cohort*  SetupCohort(
00605       PropertyCache* cache, const GoogleString& cohort) {
00606     return factory()->SetupCohort(cache, cohort);
00607   }
00608 
00611   void SetupSharedCache();
00612 
00614   MockPropertyPage* NewMockPage(const StringPiece& url,
00615                                 const StringPiece& options_signature_hash,
00616                                 UserAgentMatcher::DeviceType device_type) {
00617     return new MockPropertyPage(
00618         server_context_->thread_system(),
00619         server_context_->page_property_cache(),
00620         url,
00621         options_signature_hash,
00622         UserAgentMatcher::DeviceTypeSuffix(device_type));
00623   }
00624 
00625   MockPropertyPage* NewMockPage(const StringPiece& url) {
00626     return NewMockPage(url, "hash", UserAgentMatcher::kDesktop);
00627   }
00628 
00630   void SetMockLogRecord();
00631 
00633   MockLogRecord* mock_log_record();
00634 
00636   GoogleString GetLazyloadScriptHtml();
00637   GoogleString GetLazyloadPostscriptHtml();
00638 
00644   void SetCacheInvalidationTimestamp();
00645 
00648   void SetCacheInvalidationTimestampForUrl(
00649       StringPiece url, bool ignores_metadata_and_pcache);
00650 
00654   void EnableCachePurge();
00655 
00658   static const ProcessContext& process_context();
00659 
00660  protected:
00661   void Init();
00662 
00665   virtual RequestContextPtr CreateRequestContext();
00666 
00669   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00670                                      RewriteDriver* driver);
00671 
00674   void PopulateDefaultHeaders(const ContentType& content_type,
00675                               int64 original_content_length,
00676                               ResponseHeaders* headers);
00677 
00680   void SetActiveServer(ActiveServerFlag server_to_use);
00681 
00684   void AdvanceTimeUs(int64 delay_ms);
00685   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00686   void SetTimeUs(int64 time_us);
00687   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00688 
00690   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00691 
00693   const RequestContext::TimingInfo& timing_info();
00694   RequestContext::TimingInfo* mutable_timing_info();
00695 
00700   LoggingInfo* logging_info();
00701 
00703   const MetadataCacheInfo& metadata_cache_info() {
00704     return logging_info()->metadata_cache_info();
00705   }
00706 
00709   GoogleString AppliedRewriterStringFromLog();
00710 
00713   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00714                                const GoogleString& id,
00715                                int url_index,
00716                                int rewriter_info_index,
00717                                int rewriter_info_size,
00718                                int url_list_size,
00719                                const GoogleString& url);
00720 
00722   void SetCurrentUserAgent(const StringPiece& user_agent) {
00723     current_user_agent_ = user_agent;
00724   }
00725 
00726   GoogleString ExpectedNonce();
00727 
00730   GoogleString HttpCacheKey(StringPiece url) {
00731     return HTTPCache::CompositeKey(url, rewrite_driver_->CacheFragment());
00732   }
00733 
00736   MockUrlFetcher mock_url_fetcher_;
00737   TestDistributedFetcher test_distributed_fetcher_;
00738   scoped_ptr<Statistics> statistics_;
00739 
00745   scoped_ptr<TestRewriteDriverFactory> factory_;
00746   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00747   ServerContext* server_context_;
00748   RewriteDriver* rewrite_driver_;
00749   ServerContext* other_server_context_;
00750   RewriteDriver* other_rewrite_driver_;
00751   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00752   ActiveServerFlag active_server_;
00753   bool use_managed_rewrite_drivers_;
00754   StringPiece current_user_agent_;
00755   MD5Hasher md5_hasher_;
00756 
00757   RewriteOptions* options_; 
00758   RewriteOptions* other_options_; 
00759   UrlSegmentEncoder default_encoder_;
00760   ResponseHeaders response_headers_;
00761   const GoogleString kEtag0; 
00762   uint64 expected_nonce_;
00763 };
00764 
00765 }  
00766 
00767 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines