Page Speed Optimization Libraries  1.5.27.2
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/response_headers.h"
00033 #include "net/instaweb/rewriter/public/resource.h"
00034 #include "net/instaweb/http/public/request_context.h"
00035 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00036 #include "net/instaweb/rewriter/public/rewrite_options.h"
00037 #include "net/instaweb/rewriter/public/server_context.h"
00038 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00039 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00040 #include "net/instaweb/util/public/basictypes.h"
00041 #include "net/instaweb/util/public/md5_hasher.h"
00042 #include "net/instaweb/util/public/mem_file_system.h"
00043 #include "net/instaweb/util/public/mock_hasher.h"
00044 #include "net/instaweb/util/public/mock_message_handler.h"
00045 #include "net/instaweb/util/public/mock_property_page.h"
00047 #include "net/instaweb/util/public/mock_timer.h"
00048 #include "net/instaweb/util/public/scoped_ptr.h"
00049 #include "net/instaweb/util/public/string.h"
00050 #include "net/instaweb/util/public/string_util.h"
00051 #include "net/instaweb/util/public/timer.h"
00052 #include "net/instaweb/util/public/url_segment_encoder.h"
00053 
00054 
00055 namespace net_instaweb {
00056 
00057 class AbstractLogRecord;
00058 class CountingUrlAsyncFetcher;
00059 class DelayCache;
00060 class HTTPValue;
00061 class Hasher;
00062 class HtmlWriterFilter;
00063 class LRUCache;
00064 class MessageHandler;
00065 class MockScheduler;
00066 class PropertyCache;
00067 class RequestHeaders;
00068 class ResourceNamer;
00069 class RewriteFilter;
00070 class Statistics;
00071 class WaitUrlAsyncFetcher;
00072 struct ContentType;
00073 
00074 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00075  protected:
00076   RewriteOptionsTestBase() {
00077     RewriteOptions::Initialize();
00078   }
00079   ~RewriteOptionsTestBase() {
00080     RewriteOptions::Terminate();
00081   }
00082 };
00083 
00084 class RewriteTestBase : public RewriteOptionsTestBase {
00085  public:
00086   static const char kTestData[]; 
00087 
00088 
00089 
00090 
00091 
00092 
00093   enum ActiveServerFlag {
00094     kPrimary,    
00095     kSecondary,  
00096   };
00097 
00098   RewriteTestBase();
00099   explicit RewriteTestBase(Statistics* statistics);
00100 
00106   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00107                                      TestRewriteDriverFactory*> factories);
00108   virtual ~RewriteTestBase();
00109 
00110   virtual void SetUp();
00111   virtual void TearDown();
00112 
00116   virtual bool AddBody() const { return false; }
00117 
00127   virtual TestRewriteDriverFactory* MakeTestFactory();
00128 
00131   void AddRecompressImageFilters();
00132 
00134   void AddFilter(RewriteOptions::Filter filter);
00135 
00137   void AddOtherFilter(RewriteOptions::Filter filter);
00138 
00141   void AddRewriteFilter(RewriteFilter* filter);
00142 
00145   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00146 
00149   void AddOtherRewriteFilter(RewriteFilter* filter);
00150 
00153   void SetBaseUrlForFetch(const StringPiece& url);
00154 
00155   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00156 
00161   Timer* timer() { return factory()->mock_timer(); }
00162 
00164   void AppendDefaultHeaders(const ContentType& content_type,
00165                             GoogleString* text);
00166 
00169   void AppendDefaultHeaders(const ContentType& content_type,
00170                             int64 original_content_length,
00171                             GoogleString* text);
00172 
00173   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00174                                      const StringPiece& expected_content);
00175 
00176   void ServeResourceFromManyContextsWithUA(
00177       const GoogleString& resource_url,
00178       const StringPiece& expected_content,
00179       const StringPiece& user_agent);
00180 
00183   void ServeResourceFromNewContext(
00184       const GoogleString& resource_url,
00185       const StringPiece& expected_content);
00186 
00190   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00191 
00193   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00194                               ResponseHeaders* response_headers);
00195 
00199   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00200                      const StringPiece& name, const StringPiece& ext,
00201                      GoogleString* content);
00202   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00203                      const StringPiece& name, const StringPiece& ext,
00204                      GoogleString* content, ResponseHeaders* response);
00205 
00206   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00207                         ResponseHeaders* response);
00208   bool FetchResourceUrl(const StringPiece& url,
00209                         RequestHeaders* request_headers,
00210                         GoogleString* content,
00211                         ResponseHeaders* response_headers);
00212   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00213 
00215   bool TryFetchResource(const StringPiece& url);
00216 
00221   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00222 
00223   GoogleString CssLinkHref(const StringPiece& url) {
00224     return StrCat("<link rel=stylesheet href=", url, ">");
00225   }
00226 
00228   class CssLink {
00229    public:
00230     CssLink(const StringPiece& url, const StringPiece& content,
00231             const StringPiece& media, bool supply_mock);
00232 
00234     class Vector : public std::vector<CssLink*> {
00235      public:
00236       ~Vector();
00237       void Add(const StringPiece& url, const StringPiece& content,
00238                const StringPiece& media, bool supply_mock);
00239     };
00240 
00243     bool DecomposeCombinedUrl(GoogleString* base, StringVector* segments,
00244                               MessageHandler* handler);
00245 
00246     GoogleString url_;
00247     GoogleString content_;
00248     GoogleString media_;
00249     bool supply_mock_;
00250   };
00251 
00253   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00254                        StringVector* css_links);
00255 
00257   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00258                        CssLink::Vector* css_links);
00259 
00261   void EncodePathAndLeaf(const StringPiece& filter_id,
00262                          const StringPiece& hash,
00263                          const StringVector& name_vector,
00264                          const StringPiece& ext,
00265                          ResourceNamer* namer);
00266 
00267   StringVector MultiUrl(const StringPiece& url1) {
00268     StringVector v;
00269     v.push_back(url1.as_string());
00270     return v;
00271   }
00272 
00273   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00274     StringVector v;
00275     v.push_back(url1.as_string());
00276     v.push_back(url2.as_string());
00277     return v;
00278   }
00279 
00280   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00281                         const StringPiece& url3) {
00282     StringVector v;
00283     v.push_back(url1.as_string());
00284     v.push_back(url2.as_string());
00285     v.push_back(url3.as_string());
00286     return v;
00287   }
00288 
00289   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00290                         const StringPiece& url3, const StringPiece& url4) {
00291     StringVector v;
00292     v.push_back(url1.as_string());
00293     v.push_back(url2.as_string());
00294     v.push_back(url3.as_string());
00295     v.push_back(url4.as_string());
00296     return v;
00297   }
00298 
00301   GoogleString Encode(const StringPiece& path,
00302                       const StringPiece& filter_id,
00303                       const StringPiece& hash,
00304                       const StringPiece& name,
00305                       const StringPiece& ext) {
00306     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00307   }
00308   GoogleString Encode(const StringPiece& path,
00309                       const StringPiece& filter_id,
00310                       const StringPiece& hash,
00311                       const StringVector& name_vector,
00312                       const StringPiece& ext);
00313 
00315   GoogleString EncodeNormal(const StringPiece& path,
00316                             const StringPiece& filter_id,
00317                             const StringPiece& hash,
00318                             const StringPiece& name,
00319                             const StringPiece& ext) {
00320     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00321   }
00322   GoogleString EncodeNormal(const StringPiece& path,
00323                             const StringPiece& filter_id,
00324                             const StringPiece& hash,
00325                             const StringVector& name_vector,
00326                             const StringPiece& ext);
00327 
00330   GoogleString EncodeWithBase(const StringPiece& base,
00331                               const StringPiece& path,
00332                               const StringPiece& filter_id,
00333                               const StringPiece& hash,
00334                               const StringPiece& name,
00335                               const StringPiece& ext) {
00336     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00337   }
00338   GoogleString EncodeWithBase(const StringPiece& base,
00339                               const StringPiece& path,
00340                               const StringPiece& filter_id,
00341                               const StringPiece& hash,
00342                               const StringVector& name_vector,
00343                               const StringPiece& ext);
00344 
00346   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00347                                       const StringPiece& options);
00348 
00353   static GoogleString ChangeSuffix(
00354       GoogleString old_url, bool append_new_suffix,
00355       StringPiece old_suffix, StringPiece new_suffix);
00356 
00361   void SetupWaitFetcher();
00362   void CallFetcherCallbacks();
00363 
00364   RewriteOptions* options() { return options_; }
00365   RewriteOptions* other_options() { return other_options_; }
00366 
00368   void TestServeFiles(const ContentType* content_type,
00369                       const StringPiece& filter_id,
00370                       const StringPiece& rewritten_ext,
00371                       const StringPiece& orig_name,
00372                       const StringPiece& orig_content,
00373                       const StringPiece& rewritten_name,
00374                       const StringPiece& rewritten_content);
00375 
00376   TestRewriteDriverFactory* factory() { return factory_.get(); }
00377   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00378 
00379   void UseMd5Hasher() {
00380     server_context_->set_hasher(&md5_hasher_);
00381     server_context_->http_cache()->set_hasher(&md5_hasher_);
00382     other_server_context_->set_hasher(&md5_hasher_);
00383     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00384   }
00385 
00386 
00387   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00388                                   ResponseHeaders* header) {
00389     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00390   }
00391 
00392   void SetFetchResponse(const StringPiece& url,
00393                         const ResponseHeaders& response_header,
00394                         const StringPiece& response_body) {
00395     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00396   }
00397 
00399   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00400                                      const ContentType& content_type,
00401                                      const StringPiece& content,
00402                                      int64 ttl_sec);
00403 
00406   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00407 
00409   void AddFileToMockFetcher(const StringPiece& url,
00410                             const StringPiece& filename,
00411                             const ContentType& content_type, int64 ttl_sec);
00412 
00413   void AddToResponse(const StringPiece& url,
00414                      const StringPiece& name,
00415                      const StringPiece& value) {
00416     mock_url_fetcher()->AddToResponse(url, name, value);
00417   }
00418 
00419   void SetFetchResponse404(const StringPiece& url);
00420 
00421   void SetFetchFailOnUnexpected(bool fail) {
00422     mock_url_fetcher()->set_fail_on_unexpected(fail);
00423   }
00424   void FetcherUpdateDateHeaders() {
00425     mock_url_fetcher()->set_timer(timer());
00426     mock_url_fetcher()->set_update_date_headers(true);
00427   }
00428   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00429 
00430   virtual void ClearStats();
00431 
00434   void ClearRewriteDriver();
00435 
00436   MockUrlFetcher* mock_url_fetcher() {
00437     return &mock_url_fetcher_;
00438   }
00439   TestDistributedFetcher* test_distributed_fetcher() {
00440     return &test_distributed_fetcher_;
00441   }
00442   Hasher* hasher() { return server_context_->hasher(); }
00443   DelayCache* delay_cache() { return factory_->delay_cache(); }
00444   LRUCache* lru_cache() { return factory_->lru_cache(); }
00445   Statistics* statistics() { return factory_->statistics(); }
00446   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00447   HTTPCache* http_cache() { return server_context_->http_cache(); }
00448   PropertyCache* page_property_cache() {
00449     return server_context_->page_property_cache();
00450   }
00451   MockMessageHandler* message_handler() {
00452     return factory_->mock_message_handler();
00453   }
00454 
00462   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00463   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00464 
00466   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00467 
00468   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00469 
00470   bool ReadFile(const char* filename, GoogleString* contents) {
00471     return file_system()->ReadFile(filename, contents, message_handler());
00472   }
00473   bool WriteFile(const char* filename, const StringPiece& contents) {
00474     return file_system()->WriteFile(filename, contents, message_handler());
00475   }
00476 
00477   ServerContext* server_context() { return server_context_; }
00478   ServerContext* other_server_context() { return other_server_context_; }
00479   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00480     return factory_->counting_url_async_fetcher();
00481   }
00482   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00483     return factory_->counting_distributed_async_fetcher();
00484   }
00485   void SetMockHashValue(const GoogleString& value) {
00486     factory_->mock_hasher()->set_hash_value(value);
00487   }
00488 
00489   void SetCacheDelayUs(int64 delay_us);
00490 
00495   RewriteDriver* MakeDriver(ServerContext* server_context,
00496                             RewriteOptions* options);
00497 
00499   GoogleString AbsolutifyUrl(const StringPiece& in);
00500 
00505   void TestRetainExtraHeaders(const StringPiece& name,
00506                               const StringPiece& filter_id,
00507                               const StringPiece& ext);
00508 
00512   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00513 
00515   void SetUseTestUrlNamer(bool use_test_url_namer);
00516 
00519   GoogleString EncodeCssName(const StringPiece& name,
00520                              bool supports_webp,
00521                              bool can_inline);
00522 
00526   bool ReadIfCached(const ResourcePtr& resource);
00527 
00532   void InitiateResourceRead(const ResourcePtr& resource);
00533 
00537   HTTPCache::FindResult HttpBlockingFind(
00538       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00539       ResponseHeaders* headers);
00540 
00543   HTTPCache::FindResult HttpBlockingFindStatus(
00544       const GoogleString& key, HTTPCache* http_cache);
00545 
00547   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00548 
00550   void SetHtmlMimetype() { SetMimetype("text/html"); }
00551 
00553   void SetMimetype(const StringPiece& mimetype);
00554 
00557   void CheckFetchFromHttpCache(
00558       StringPiece url,
00559       StringPiece expected_contents,
00560       int64 expected_expiration_ms);
00561 
00563   void SetupCohort(PropertyCache* cache, const GoogleString& cohort) {
00564     factory()->SetupCohort(cache, cohort);
00565   }
00566 
00569   void SetupSharedCache();
00570 
00572   MockPropertyPage* NewMockPage(const StringPiece& key) {
00573     return new MockPropertyPage(
00574         server_context_->thread_system(),
00575         server_context_->page_property_cache(),
00576         key);
00577   }
00578 
00580   MockPropertyPage* NewMockClientPage(const StringPiece& key) {
00581     return new MockPropertyPage(
00582         server_context_->thread_system(),
00583         server_context_->client_property_cache(),
00584         key);
00585   }
00586 
00587  protected:
00588   void Init();
00589 
00592   virtual RequestContextPtr CreateRequestContext();
00593 
00596   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00597                                      RewriteDriver* driver);
00598 
00601   void PopulateDefaultHeaders(const ContentType& content_type,
00602                               int64 original_content_length,
00603                               ResponseHeaders* headers);
00604 
00607   void SetActiveServer(ActiveServerFlag server_to_use);
00608 
00611   void AdvanceTimeUs(int64 delay_ms);
00612   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00613   void SetTimeUs(int64 time_us);
00614   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00615 
00617   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00618 
00623   LoggingInfo* logging_info();
00624 
00626   const MetadataCacheInfo& metadata_cache_info() {
00627     return logging_info()->metadata_cache_info();
00628   }
00629 
00632   GoogleString AppliedRewriterStringFromLog();
00633 
00636   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00637                                const GoogleString& id,
00638                                int url_index,
00639                                int rewriter_info_index,
00640                                int rewriter_info_size,
00641                                int url_list_size,
00642                                const GoogleString& url);
00643 
00645   void SetCurrentUserAgent(const StringPiece& user_agent) {
00646     current_user_agent_ = user_agent;
00647   }
00648 
00651   MockUrlFetcher mock_url_fetcher_;
00652   TestDistributedFetcher test_distributed_fetcher_;
00653   scoped_ptr<Statistics> statistics_;
00654 
00660   scoped_ptr<TestRewriteDriverFactory> factory_;
00661   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00662   ServerContext* server_context_;
00663   RewriteDriver* rewrite_driver_;
00664   ServerContext* other_server_context_;
00665   RewriteDriver* other_rewrite_driver_;
00666   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00667   ActiveServerFlag active_server_;
00668   bool use_managed_rewrite_drivers_;
00669   StringPiece current_user_agent_;
00670   MD5Hasher md5_hasher_;
00671 
00672   RewriteOptions* options_; 
00673   RewriteOptions* other_options_; 
00674   UrlSegmentEncoder default_encoder_;
00675   ResponseHeaders response_headers_;
00676   const GoogleString kEtag0; 
00677 };
00678 
00679 }  
00680 
00681 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines