Page Speed Optimization Libraries  1.6.29.3
net/instaweb/rewriter/public/rewrite_test_base.h
Go to the documentation of this file.
00001 /*
00002  * Copyright 2010 Google Inc.
00003  *
00004  * Licensed under the Apache License, Version 2.0 (the "License");
00005  * you may not use this file except in compliance with the License.
00006  * You may obtain a copy of the License at
00007  *
00008  *      http:///www.apache.org/licenses/LICENSE-2.0
00009  *
00010  * Unless required by applicable law or agreed to in writing, software
00011  * distributed under the License is distributed on an "AS IS" BASIS,
00012  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  * See the License for the specific language governing permissions and
00014  * limitations under the License.
00015  */
00016 
00018 
00020 
00021 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00022 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
00023 
00024 #include <utility>
00025 #include <vector>
00026 
00027 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
00028 #include "net/instaweb/http/public/mock_url_fetcher.h"
00029 #include "net/instaweb/http/public/http_cache.h"
00030 #include "net/instaweb/http/public/logging_proto.h"
00031 #include "net/instaweb/http/public/logging_proto_impl.h"
00032 #include "net/instaweb/http/public/response_headers.h"
00034 #include "net/instaweb/rewriter/public/resource.h"
00035 #include "net/instaweb/http/public/request_context.h"
00036 #include "net/instaweb/rewriter/public/rewrite_driver.h"
00037 #include "net/instaweb/rewriter/public/rewrite_options.h"
00038 #include "net/instaweb/rewriter/public/server_context.h"
00039 #include "net/instaweb/rewriter/public/test_distributed_fetcher.h"
00040 #include "net/instaweb/rewriter/public/test_rewrite_driver_factory.h"
00041 #include "net/instaweb/util/public/basictypes.h"
00042 #include "net/instaweb/util/public/md5_hasher.h"
00043 #include "net/instaweb/util/public/mem_file_system.h"
00044 #include "net/instaweb/util/public/mock_hasher.h"
00045 #include "net/instaweb/util/public/mock_message_handler.h"
00046 #include "net/instaweb/util/public/mock_property_page.h"
00048 #include "net/instaweb/util/public/mock_timer.h"
00049 #include "net/instaweb/util/public/property_cache.h"
00050 #include "net/instaweb/util/public/scoped_ptr.h"
00051 #include "net/instaweb/util/public/string.h"
00052 #include "net/instaweb/util/public/string_util.h"
00053 #include "net/instaweb/util/public/timer.h"
00054 #include "net/instaweb/util/public/url_segment_encoder.h"
00055 #include "pagespeed/kernel/http/content_type.h"
00056 
00057 
00058 namespace net_instaweb {
00059 
00060 class AbstractLogRecord;
00061 class CountingUrlAsyncFetcher;
00062 class DelayCache;
00063 class HTTPValue;
00064 class Hasher;
00065 class HtmlWriterFilter;
00066 class LRUCache;
00067 class MessageHandler;
00068 class MockLogRecord;
00069 class MockScheduler;
00070 class RequestHeaders;
00071 class ResourceNamer;
00072 class RewriteFilter;
00073 class Statistics;
00074 class WaitUrlAsyncFetcher;
00075 
00076 class RewriteOptionsTestBase : public HtmlParseTestBaseNoAlloc {
00077  protected:
00078   RewriteOptionsTestBase() {
00079     RewriteOptions::Initialize();
00080   }
00081   ~RewriteOptionsTestBase() {
00082     RewriteOptions::Terminate();
00083   }
00084 };
00085 
00086 class RewriteTestBase : public RewriteOptionsTestBase {
00087  public:
00088   static const char kTestData[]; 
00089 
00090 
00091 
00092 
00093 
00094 
00095   enum ActiveServerFlag {
00096     kPrimary,    
00097     kSecondary,  
00098   };
00099 
00100   RewriteTestBase();
00101   explicit RewriteTestBase(Statistics* statistics);
00102 
00108   explicit RewriteTestBase(std::pair<TestRewriteDriverFactory*,
00109                                      TestRewriteDriverFactory*> factories);
00110   virtual ~RewriteTestBase();
00111 
00112   virtual void SetUp();
00113   virtual void TearDown();
00114 
00118   virtual bool AddBody() const { return false; }
00119 
00129   virtual TestRewriteDriverFactory* MakeTestFactory();
00130 
00133   void AddRecompressImageFilters();
00134 
00136   void AddFilter(RewriteOptions::Filter filter);
00137 
00139   void AddOtherFilter(RewriteOptions::Filter filter);
00140 
00143   void AddRewriteFilter(RewriteFilter* filter);
00144 
00147   void AddFetchOnlyRewriteFilter(RewriteFilter* filter);
00148 
00151   void AddOtherRewriteFilter(RewriteFilter* filter);
00152 
00155   void SetBaseUrlForFetch(const StringPiece& url);
00156 
00157   ResourcePtr CreateResource(const StringPiece& base, const StringPiece& url);
00158 
00163   Timer* timer() { return factory()->mock_timer(); }
00164 
00166   void AppendDefaultHeaders(const ContentType& content_type,
00167                             GoogleString* text);
00168 
00171   void AppendDefaultHeaders(const ContentType& content_type,
00172                             int64 original_content_length,
00173                             GoogleString* text);
00174 
00175   void ServeResourceFromManyContexts(const GoogleString& resource_url,
00176                                      const StringPiece& expected_content);
00177 
00178   void ServeResourceFromManyContextsWithUA(
00179       const GoogleString& resource_url,
00180       const StringPiece& expected_content,
00181       const StringPiece& user_agent);
00182 
00185   void ServeResourceFromNewContext(
00186       const GoogleString& resource_url,
00187       const StringPiece& expected_content);
00188 
00192   virtual RewriteDriver* html_parse() { return rewrite_driver_; }
00193 
00195   void DefaultResponseHeaders(const ContentType& content_type, int64 ttl_sec,
00196                               ResponseHeaders* response_headers);
00197 
00201   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00202                      const StringPiece& name, const StringPiece& ext,
00203                      GoogleString* content);
00204   bool FetchResource(const StringPiece& path, const StringPiece& filter_id,
00205                      const StringPiece& name, const StringPiece& ext,
00206                      GoogleString* content, ResponseHeaders* response);
00207 
00208   bool FetchResourceUrl(const StringPiece& url, GoogleString* content,
00209                         ResponseHeaders* response);
00210   bool FetchResourceUrl(const StringPiece& url,
00211                         RequestHeaders* request_headers,
00212                         GoogleString* content,
00213                         ResponseHeaders* response_headers);
00214   bool FetchResourceUrl(const StringPiece& url, GoogleString* content);
00215 
00217   bool TryFetchResource(const StringPiece& url);
00218 
00223   void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers);
00224 
00225   GoogleString CssLinkHref(const StringPiece& url) {
00226     return StrCat("<link rel=stylesheet href=", url, ">");
00227   }
00228 
00230   class CssLink {
00231    public:
00232     CssLink(const StringPiece& url, const StringPiece& content,
00233             const StringPiece& media, bool supply_mock);
00234 
00236     class Vector : public std::vector<CssLink*> {
00237      public:
00238       ~Vector();
00239       void Add(const StringPiece& url, const StringPiece& content,
00240                const StringPiece& media, bool supply_mock);
00241     };
00242 
00245     bool DecomposeCombinedUrl(GoogleString* base, StringVector* segments,
00246                               MessageHandler* handler);
00247 
00248     GoogleString url_;
00249     GoogleString content_;
00250     GoogleString media_;
00251     bool supply_mock_;
00252   };
00253 
00255   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00256                        StringVector* css_links);
00257 
00259   void CollectCssLinks(const StringPiece& id, const StringPiece& html,
00260                        CssLink::Vector* css_links);
00261 
00263   void EncodePathAndLeaf(const StringPiece& filter_id,
00264                          const StringPiece& hash,
00265                          const StringVector& name_vector,
00266                          const StringPiece& ext,
00267                          ResourceNamer* namer);
00268 
00269   StringVector MultiUrl(const StringPiece& url1) {
00270     StringVector v;
00271     v.push_back(url1.as_string());
00272     return v;
00273   }
00274 
00275   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2) {
00276     StringVector v;
00277     v.push_back(url1.as_string());
00278     v.push_back(url2.as_string());
00279     return v;
00280   }
00281 
00282   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00283                         const StringPiece& url3) {
00284     StringVector v;
00285     v.push_back(url1.as_string());
00286     v.push_back(url2.as_string());
00287     v.push_back(url3.as_string());
00288     return v;
00289   }
00290 
00291   StringVector MultiUrl(const StringPiece& url1, const StringPiece& url2,
00292                         const StringPiece& url3, const StringPiece& url4) {
00293     StringVector v;
00294     v.push_back(url1.as_string());
00295     v.push_back(url2.as_string());
00296     v.push_back(url3.as_string());
00297     v.push_back(url4.as_string());
00298     return v;
00299   }
00300 
00303   GoogleString Encode(const StringPiece& path,
00304                       const StringPiece& filter_id,
00305                       const StringPiece& hash,
00306                       const StringPiece& name,
00307                       const StringPiece& ext) {
00308     return Encode(path, filter_id, hash, MultiUrl(name), ext);
00309   }
00310   GoogleString Encode(const StringPiece& path,
00311                       const StringPiece& filter_id,
00312                       const StringPiece& hash,
00313                       const StringVector& name_vector,
00314                       const StringPiece& ext);
00315 
00317   GoogleString EncodeNormal(const StringPiece& path,
00318                             const StringPiece& filter_id,
00319                             const StringPiece& hash,
00320                             const StringPiece& name,
00321                             const StringPiece& ext) {
00322     return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
00323   }
00324   GoogleString EncodeNormal(const StringPiece& path,
00325                             const StringPiece& filter_id,
00326                             const StringPiece& hash,
00327                             const StringVector& name_vector,
00328                             const StringPiece& ext);
00329 
00332   GoogleString EncodeWithBase(const StringPiece& base,
00333                               const StringPiece& path,
00334                               const StringPiece& filter_id,
00335                               const StringPiece& hash,
00336                               const StringPiece& name,
00337                               const StringPiece& ext) {
00338     return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
00339   }
00340   GoogleString EncodeWithBase(const StringPiece& base,
00341                               const StringPiece& path,
00342                               const StringPiece& filter_id,
00343                               const StringPiece& hash,
00344                               const StringVector& name_vector,
00345                               const StringPiece& ext);
00346 
00348   GoogleString AddOptionsToEncodedUrl(const StringPiece& url,
00349                                       const StringPiece& options);
00350 
00355   static GoogleString ChangeSuffix(
00356       GoogleString old_url, bool append_new_suffix,
00357       StringPiece old_suffix, StringPiece new_suffix);
00358 
00363   void SetupWaitFetcher();
00364   void CallFetcherCallbacks();
00365   void OtherCallFetcherCallbacks();
00366   RewriteOptions* options() { return options_; }
00367   RewriteOptions* other_options() { return other_options_; }
00368 
00370   void SetRewriteOptions(RewriteOptions* opts);
00371 
00374   bool AddDomain(StringPiece domain);
00375 
00378   bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain);
00379 
00382   bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain);
00383 
00386   bool AddShard(StringPiece domain, StringPiece shards);
00387 
00389   void TestServeFiles(const ContentType* content_type,
00390                       const StringPiece& filter_id,
00391                       const StringPiece& rewritten_ext,
00392                       const StringPiece& orig_name,
00393                       const StringPiece& orig_content,
00394                       const StringPiece& rewritten_name,
00395                       const StringPiece& rewritten_content);
00396 
00397   TestRewriteDriverFactory* factory() { return factory_.get(); }
00398   TestRewriteDriverFactory* other_factory() { return other_factory_.get(); }
00399 
00400   void UseMd5Hasher() {
00401     server_context_->set_hasher(&md5_hasher_);
00402     server_context_->http_cache()->set_hasher(&md5_hasher_);
00403     other_server_context_->set_hasher(&md5_hasher_);
00404     other_server_context_->http_cache()->set_hasher(&md5_hasher_);
00405   }
00406 
00407 
00408   void SetDefaultLongCacheHeaders(const ContentType* content_type,
00409                                   ResponseHeaders* header) {
00410     server_context_->SetDefaultLongCacheHeaders(content_type, header);
00411   }
00412 
00413   void SetFetchResponse(const StringPiece& url,
00414                         const ResponseHeaders& response_header,
00415                         const StringPiece& response_body) {
00416     mock_url_fetcher()->SetResponse(url, response_header, response_body);
00417   }
00418 
00420   void SetResponseWithDefaultHeaders(const StringPiece& relative_url,
00421                                      const ContentType& content_type,
00422                                      const StringPiece& content,
00423                                      int64 ttl_sec);
00424 
00427   bool LoadFile(const StringPiece& filename, GoogleString* contents);
00428 
00430   void AddFileToMockFetcher(const StringPiece& url,
00431                             const StringPiece& filename,
00432                             const ContentType& content_type, int64 ttl_sec);
00433 
00434   void AddToResponse(const StringPiece& url,
00435                      const StringPiece& name,
00436                      const StringPiece& value) {
00437     mock_url_fetcher()->AddToResponse(url, name, value);
00438   }
00439 
00440   void SetFetchResponse404(const StringPiece& url);
00441 
00442   void SetFetchFailOnUnexpected(bool fail) {
00443     mock_url_fetcher()->set_fail_on_unexpected(fail);
00444   }
00445   void FetcherUpdateDateHeaders() {
00446     mock_url_fetcher()->set_timer(timer());
00447     mock_url_fetcher()->set_update_date_headers(true);
00448   }
00449   void ClearFetcherResponses() { mock_url_fetcher()->Clear(); }
00450 
00451   virtual void ClearStats();
00452 
00455   void ClearRewriteDriver();
00456 
00457   MockUrlFetcher* mock_url_fetcher() {
00458     return &mock_url_fetcher_;
00459   }
00460   TestDistributedFetcher* test_distributed_fetcher() {
00461     return &test_distributed_fetcher_;
00462   }
00463   Hasher* hasher() { return server_context_->hasher(); }
00464   DelayCache* delay_cache() { return factory_->delay_cache(); }
00465   LRUCache* lru_cache() { return factory_->lru_cache(); }
00466   Statistics* statistics() { return factory_->statistics(); }
00467   MemFileSystem* file_system() { return factory_->mem_file_system(); }
00468   HTTPCache* http_cache() { return server_context_->http_cache(); }
00469   PropertyCache* page_property_cache() {
00470     return server_context_->page_property_cache();
00471   }
00472   MockMessageHandler* message_handler() {
00473     return factory_->mock_message_handler();
00474   }
00475 
00483   RewriteDriver* rewrite_driver() { return rewrite_driver_; }
00484   RewriteDriver* other_rewrite_driver() { return other_rewrite_driver_; }
00485 
00487   MockScheduler* mock_scheduler() { return factory_->mock_scheduler(); }
00488 
00489   int64 start_time_ms() const { return factory_->kStartTimeMs; }
00490 
00491   bool ReadFile(const char* filename, GoogleString* contents) {
00492     return file_system()->ReadFile(filename, contents, message_handler());
00493   }
00494   bool WriteFile(const char* filename, const StringPiece& contents) {
00495     return file_system()->WriteFile(filename, contents, message_handler());
00496   }
00497 
00498   ServerContext* server_context() { return server_context_; }
00499   ServerContext* other_server_context() { return other_server_context_; }
00500   CountingUrlAsyncFetcher* counting_url_async_fetcher() {
00501     return factory_->counting_url_async_fetcher();
00502   }
00503   CountingUrlAsyncFetcher* counting_distributed_fetcher() {
00504     return factory_->counting_distributed_async_fetcher();
00505   }
00506   void SetMockHashValue(const GoogleString& value) {
00507     factory_->mock_hasher()->set_hash_value(value);
00508   }
00509 
00510   void SetCacheDelayUs(int64 delay_us);
00511 
00516   RewriteDriver* MakeDriver(ServerContext* server_context,
00517                             RewriteOptions* options);
00518 
00520   GoogleString AbsolutifyUrl(const StringPiece& in);
00521 
00526   void TestRetainExtraHeaders(const StringPiece& name,
00527                               const StringPiece& filter_id,
00528                               const StringPiece& ext);
00529 
00533   const UrlSegmentEncoder* FindEncoder(const StringPiece& id) const;
00534 
00536   void SetUseTestUrlNamer(bool use_test_url_namer);
00537 
00540   GoogleString EncodeCssName(const StringPiece& name,
00541                              bool supports_webp,
00542                              bool can_inline);
00543 
00547   bool ReadIfCached(const ResourcePtr& resource);
00548 
00553   void InitiateResourceRead(const ResourcePtr& resource);
00554 
00558   HTTPCache::FindResult HttpBlockingFind(
00559       const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
00560       ResponseHeaders* headers);
00561 
00564   HTTPCache::FindResult HttpBlockingFindStatus(
00565       const GoogleString& key, HTTPCache* http_cache);
00566 
00568   void SetXhtmlMimetype() { SetMimetype("application/xhtml+xml"); }
00569 
00571   void SetHtmlMimetype() { SetMimetype("text/html"); }
00572 
00574   void SetMimetype(const StringPiece& mimetype);
00575 
00578   void CheckFetchFromHttpCache(
00579       StringPiece url,
00580       StringPiece expected_contents,
00581       int64 expected_expiration_ms);
00582 
00584   const PropertyCache::Cohort*  SetupCohort(
00585       PropertyCache* cache, const GoogleString& cohort) {
00586     return factory()->SetupCohort(cache, cohort);
00587   }
00588 
00591   void SetupSharedCache();
00592 
00594   MockPropertyPage* NewMockPage(const StringPiece& key) {
00595     return new MockPropertyPage(
00596         server_context_->thread_system(),
00597         server_context_->page_property_cache(),
00598         key);
00599   }
00600 
00602   void SetMockLogRecord();
00603 
00605   MockLogRecord* mock_log_record();
00606 
00608   GoogleString GetLazyloadScriptHtml();
00609   GoogleString GetLazyloadPostscriptHtml();
00610 
00611  protected:
00612   void Init();
00613 
00616   virtual RequestContextPtr CreateRequestContext();
00617 
00620   void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher* fetcher,
00621                                      RewriteDriver* driver);
00622 
00625   void PopulateDefaultHeaders(const ContentType& content_type,
00626                               int64 original_content_length,
00627                               ResponseHeaders* headers);
00628 
00631   void SetActiveServer(ActiveServerFlag server_to_use);
00632 
00635   void AdvanceTimeUs(int64 delay_ms);
00636   void AdvanceTimeMs(int64 delay_ms) { AdvanceTimeUs(delay_ms * Timer::kMsUs); }
00637   void SetTimeUs(int64 time_us);
00638   void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
00639 
00641   void AdjustTimeUsWithoutWakingAlarms(int64 time_us);
00642 
00644   const RequestContext::TimingInfo& timing_info();
00645   RequestContext::TimingInfo* mutable_timing_info();
00646 
00651   LoggingInfo* logging_info();
00652 
00654   const MetadataCacheInfo& metadata_cache_info() {
00655     return logging_info()->metadata_cache_info();
00656   }
00657 
00660   GoogleString AppliedRewriterStringFromLog();
00661 
00664   void VerifyRewriterInfoEntry(AbstractLogRecord* log_record,
00665                                const GoogleString& id,
00666                                int url_index,
00667                                int rewriter_info_index,
00668                                int rewriter_info_size,
00669                                int url_list_size,
00670                                const GoogleString& url);
00671 
00673   void SetCurrentUserAgent(const StringPiece& user_agent) {
00674     current_user_agent_ = user_agent;
00675   }
00676 
00679   MockUrlFetcher mock_url_fetcher_;
00680   TestDistributedFetcher test_distributed_fetcher_;
00681   scoped_ptr<Statistics> statistics_;
00682 
00688   scoped_ptr<TestRewriteDriverFactory> factory_;
00689   scoped_ptr<TestRewriteDriverFactory> other_factory_;
00690   ServerContext* server_context_;
00691   RewriteDriver* rewrite_driver_;
00692   ServerContext* other_server_context_;
00693   RewriteDriver* other_rewrite_driver_;
00694   scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
00695   ActiveServerFlag active_server_;
00696   bool use_managed_rewrite_drivers_;
00697   StringPiece current_user_agent_;
00698   MD5Hasher md5_hasher_;
00699 
00700   RewriteOptions* options_; 
00701   RewriteOptions* other_options_; 
00702   UrlSegmentEncoder default_encoder_;
00703   ResponseHeaders response_headers_;
00704   const GoogleString kEtag0; 
00705 };
00706 
00707 }  
00708 
00709 #endif  ///< NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines