21 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
22 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
27 #include "net/instaweb/htmlparse/public/html_parse_test_base.h"
33 #include "net/instaweb/http/public/response_headers.h"
34 #include "net/instaweb/http/public/user_agent_matcher.h"
43 #include "net/instaweb/util/public/md5_hasher.h"
45 #include "net/instaweb/util/public/mock_hasher.h"
49 #include "net/instaweb/util/public/mock_timer.h"
55 #include "net/instaweb/util/public/url_segment_encoder.h"
56 #include "pagespeed/kernel/http/content_type.h"
59 namespace net_instaweb {
61 class AbstractLogRecord;
62 class CountingUrlAsyncFetcher;
66 class HtmlWriterFilter;
76 class WaitUrlAsyncFetcher;
78 class RewriteOptionsTestBase :
public HtmlParseTestBaseNoAlloc {
80 RewriteOptionsTestBase() {
83 ~RewriteOptionsTestBase() {
84 RewriteOptions::Terminate();
94 static const char kWrongBeaconingKey[];
119 virtual void SetUp();
120 virtual void TearDown();
125 virtual bool AddBody()
const {
return false; }
170 StringPiece downstream_cache_purge_method,
171 StringPiece downstream_cache_purge_location_prefix,
172 StringPiece rebeaconing_key);
177 ResourcePtr CreateResource(
const StringPiece& base,
const StringPiece& url);
183 Timer*
timer() {
return factory()->mock_timer(); }
192 int64 original_content_length,
195 void ServeResourceFromManyContexts(
const GoogleString& resource_url,
196 const StringPiece& expected_content);
198 void ServeResourceFromManyContextsWithUA(
199 const GoogleString& resource_url,
200 const StringPiece& expected_content,
201 const StringPiece& user_agent);
206 const GoogleString& resource_url,
207 const StringPiece& expected_content);
216 ResponseHeaders* response_headers);
221 bool FetchResource(
const StringPiece& path,
const StringPiece& filter_id,
222 const StringPiece& name,
const StringPiece& ext,
223 GoogleString* content);
224 bool FetchResource(
const StringPiece& path,
const StringPiece& filter_id,
225 const StringPiece& name,
const StringPiece& ext,
226 GoogleString* content, ResponseHeaders* response);
228 bool FetchResourceUrl(
const StringPiece& url, GoogleString* content,
229 ResponseHeaders* response);
230 bool FetchResourceUrl(
const StringPiece& url,
231 RequestHeaders* request_headers,
232 GoogleString* content,
233 ResponseHeaders* response_headers);
234 bool FetchResourceUrl(
const StringPiece& url, GoogleString* content);
245 GoogleString CssLinkHref(
const StringPiece& url) {
246 return StrCat(
"<link rel=stylesheet href=", url,
">");
252 CssLink(
const StringPiece& url,
const StringPiece& content,
253 const StringPiece& media,
bool supply_mock);
256 class Vector :
public std::vector<CssLink*> {
259 void Add(
const StringPiece& url,
const StringPiece& content,
260 const StringPiece& media,
bool supply_mock);
266 StringVector* segments, MessageHandler* handler);
269 GoogleString content_;
276 StringVector* css_links);
284 const StringPiece& hash,
285 const StringVector& name_vector,
286 const StringPiece& ext,
289 StringVector MultiUrl(
const StringPiece& url1) {
291 v.push_back(url1.as_string());
295 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2) {
297 v.push_back(url1.as_string());
298 v.push_back(url2.as_string());
302 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2,
303 const StringPiece& url3) {
305 v.push_back(url1.as_string());
306 v.push_back(url2.as_string());
307 v.push_back(url3.as_string());
311 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2,
312 const StringPiece& url3,
const StringPiece& url4) {
314 v.push_back(url1.as_string());
315 v.push_back(url2.as_string());
316 v.push_back(url3.as_string());
317 v.push_back(url4.as_string());
323 GoogleString
Encode(
const StringPiece& path,
324 const StringPiece& filter_id,
325 const StringPiece& hash,
326 const StringPiece& name,
327 const StringPiece& ext) {
328 return Encode(path, filter_id, hash, MultiUrl(name), ext);
330 GoogleString
Encode(
const StringPiece& path,
331 const StringPiece& filter_id,
332 const StringPiece& hash,
333 const StringVector& name_vector,
334 const StringPiece& ext);
338 const StringPiece& filter_id,
339 const StringPiece& hash,
340 const StringPiece& name,
341 const StringPiece& ext) {
342 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
345 const StringPiece& filter_id,
346 const StringPiece& hash,
347 const StringVector& name_vector,
348 const StringPiece& ext);
353 const StringPiece& path,
354 const StringPiece& filter_id,
355 const StringPiece& hash,
356 const StringPiece& name,
357 const StringPiece& ext) {
358 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
361 const StringPiece& path,
362 const StringPiece& filter_id,
363 const StringPiece& hash,
364 const StringVector& name_vector,
365 const StringPiece& ext);
369 const StringPiece& options);
376 StringPiece old_url,
bool append_new_suffix,
377 StringPiece old_suffix, StringPiece new_suffix);
384 void CallFetcherCallbacks();
385 void OtherCallFetcherCallbacks();
394 bool AddDomain(StringPiece domain);
406 bool AddShard(StringPiece domain, StringPiece shards);
410 const StringPiece& filter_id,
411 const StringPiece& rewritten_ext,
412 const StringPiece& orig_name,
413 const StringPiece& orig_content,
414 const StringPiece& rewritten_name,
415 const StringPiece& rewritten_content);
417 TestRewriteDriverFactory* factory() {
return factory_.get(); }
418 TestRewriteDriverFactory* other_factory() {
return other_factory_.get(); }
420 void UseMd5Hasher() {
422 server_context_->http_cache()->set_hasher(&md5_hasher_);
423 other_server_context_->
set_hasher(&md5_hasher_);
424 other_server_context_->http_cache()->set_hasher(&md5_hasher_);
428 void SetDefaultLongCacheHeaders(
const ContentType* content_type,
429 ResponseHeaders* header) {
431 content_type, StringPiece(), StringPiece(), header);
434 void SetFetchResponse(
const StringPiece& url,
435 const ResponseHeaders& response_header,
436 const StringPiece& response_body) {
437 mock_url_fetcher()->SetResponse(url, response_header, response_body);
442 const ContentType& content_type,
443 const StringPiece& content,
448 bool LoadFile(
const StringPiece& filename, GoogleString* contents);
452 const StringPiece& filename,
453 const ContentType& content_type, int64 ttl_sec);
455 void AddToResponse(
const StringPiece& url,
456 const StringPiece& name,
457 const StringPiece& value) {
461 void SetFetchResponse404(
const StringPiece& url);
463 void SetFetchFailOnUnexpected(
bool fail) {
466 void FetcherUpdateDateHeaders() {
467 mock_url_fetcher()->set_timer(
timer());
470 void ClearFetcherResponses() { mock_url_fetcher()->
Clear(); }
472 virtual void ClearStats();
478 MockUrlFetcher* mock_url_fetcher() {
481 TestDistributedFetcher* test_distributed_fetcher() {
482 return &test_distributed_fetcher_;
484 Hasher* hasher() {
return server_context_->
hasher(); }
485 DelayCache* delay_cache() {
return factory_->delay_cache(); }
486 LRUCache* lru_cache() {
return factory_->lru_cache(); }
487 Statistics* statistics() {
return factory_->statistics(); }
488 MemFileSystem* file_system() {
return factory_->mem_file_system(); }
489 HTTPCache* http_cache() {
return server_context_->http_cache(); }
490 PropertyCache* page_property_cache() {
491 return server_context_->page_property_cache();
493 MockMessageHandler* message_handler() {
494 return factory_->mock_message_handler();
505 RewriteDriver* other_rewrite_driver() {
return other_rewrite_driver_; }
510 int64 start_time_ms()
const {
return factory_->kStartTimeMs; }
512 bool ReadFile(
const char* filename, GoogleString* contents) {
513 return file_system()->ReadFile(filename, contents, message_handler());
515 bool WriteFile(
const char* filename,
const StringPiece& contents) {
516 return file_system()->WriteFile(filename, contents, message_handler());
519 ServerContext* server_context() {
return server_context_; }
520 ServerContext* other_server_context() {
return other_server_context_; }
521 CountingUrlAsyncFetcher* counting_url_async_fetcher() {
522 return factory_->counting_url_async_fetcher();
524 CountingUrlAsyncFetcher* counting_distributed_fetcher() {
525 return factory_->counting_distributed_async_fetcher();
527 void SetMockHashValue(
const GoogleString& value) {
528 factory_->mock_hasher()->set_hash_value(value);
531 void SetCacheDelayUs(int64 delay_us);
537 RewriteDriver*
MakeDriver(ServerContext* server_context,
538 RewriteOptions* options);
548 const StringPiece& filter_id,
549 const StringPiece& ext);
554 const UrlSegmentEncoder*
FindEncoder(
const StringPiece&
id)
const;
580 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
581 ResponseHeaders* headers);
586 const GoogleString& key, HTTPCache* http_cache);
590 const RewriteOptions* options,
591 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
592 ResponseHeaders* headers);
607 StringPiece expected_contents,
608 int64 expected_expiration_ms);
622 const StringPiece& options_signature_hash,
623 UserAgentMatcher::DeviceType device_type) {
625 server_context_->thread_system(),
626 server_context_->page_property_cache(),
628 options_signature_hash,
629 UserAgentMatcher::DeviceTypeSuffix(device_type));
633 return NewMockPage(url,
"hash", UserAgentMatcher::kDesktop);
644 GoogleString GetLazyloadPostscriptHtml();
656 StringPiece url,
bool ignores_metadata_and_pcache);
700 int64 original_content_length,
701 ResponseHeaders* headers);
710 void AdvanceTimeMs(int64 delay_ms) {
AdvanceTimeUs(delay_ms * Timer::kMsUs); }
711 void SetTimeUs(int64 time_us);
712 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
719 RequestContext::TimingInfo* mutable_timing_info();
739 const GoogleString&
id,
741 int rewriter_info_index,
742 int rewriter_info_size,
744 const GoogleString& url);
748 current_user_agent_ = user_agent;
751 GoogleString ExpectedNonce();
766 scoped_ptr<Statistics> statistics_;
774 scoped_ptr<TestRewriteDriverFactory> other_factory_;
779 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
781 bool use_managed_rewrite_drivers_;
782 StringPiece current_user_agent_;
783 MD5Hasher md5_hasher_;
787 UrlSegmentEncoder default_encoder_;
788 ResponseHeaders response_headers_;
790 uint64 expected_nonce_;
void EncodePathAndLeaf(const StringPiece &filter_id, const StringPiece &hash, const StringVector &name_vector, const StringPiece &ext, ResourceNamer *namer)
Encode the given name (path + leaf) using the given pagespeed attributes.
Definition: test_rewrite_driver_factory.h:67
void AddFetchOnlyRewriteFilter(RewriteFilter *filter)
virtual RewriteDriver * html_parse()
Definition: rewrite_test_base.h:212
void SetActiveServer(ActiveServerFlag server_to_use)
void SetCurrentUserAgent(const StringPiece &user_agent)
Sets current_user_agent_.
Definition: rewrite_test_base.h:747
HTTPCache::FindResult HttpBlockingFind(const GoogleString &key, HTTPCache *http_cache, HTTPValue *value_out, ResponseHeaders *headers)
void DefaultResponseHeaders(const ContentType &content_type, int64 ttl_sec, ResponseHeaders *response_headers)
Set default headers for a resource with content_type and Cache ttl_sec.
void CollectCssLinks(const StringPiece &id, const StringPiece &html, StringVector *css_links)
Collects the hrefs for all CSS <link>s on the page.
GoogleString HttpCacheKey(StringPiece url)
Definition: rewrite_test_base.h:755
Use the normal data members.
Definition: rewrite_test_base.h:103
bool FetchResource(const StringPiece &path, const StringPiece &filter_id, const StringPiece &name, const StringPiece &ext, GoogleString *content)
GoogleString Encode(const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Definition: rewrite_test_base.h:323
A vector of CssLink* should know how to accumulate and add.
Definition: rewrite_test_base.h:256
void AdjustTimeUsWithoutWakingAlarms(int64 time_us)
Adjusts time ignoring any scheduler callbacks. Use with caution.
bool DecomposeCombinedUrl(StringPiece base_url, GoogleString *base, StringVector *segments, MessageHandler *handler)
int TimedValue(StringPiece name)
Returns the value of a TimedVariable, specified by name.
static const ProcessContext & process_context()
void CheckFetchFromHttpCache(StringPiece url, StringPiece expected_contents, int64 expected_expiration_ms)
bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain)
void SetDefaultLongCacheHeaders(const ContentType *content_type, StringPiece charset, StringPiece cache_control_suffix, ResponseHeaders *header) const
void SetMimetype(const StringPiece &mimetype)
Sets the response-headers Content-Type as specified.
Definition: property_cache.h:186
Definition: mock_property_page.h:32
RewriteOptions * other_options_
owned by other_rewrite_driver_.
Definition: rewrite_test_base.h:786
void ServeResourceFromNewContext(const GoogleString &resource_url, const StringPiece &expected_content)
void SetDummyRequestHeaders()
Setup dummy empty RequestHeaders object for the driver.
MockPropertyPage * NewMockPage(const StringPiece &url, const StringPiece &options_signature_hash, UserAgentMatcher::DeviceType device_type)
Returns a new mock property page for the page property cache.
Definition: rewrite_test_base.h:621
void DebugWithMessage(StringPiece expected_debug_message)
Definition: rewrite_test_base.h:669
HTTPCache::FindResult HttpBlockingFindStatus(const GoogleString &key, HTTPCache *http_cache)
Adds property-semantics to a raw cache API.
Definition: property_cache.h:180
void AddToResponse(const StringPiece &url, const StringPiece &name, const StringPiece &value)
bool ReadIfCached(const ResourcePtr &resource)
bool LoadFile(const StringPiece &filename, GoogleString *contents)
FindResult
When a lookup is done in the HTTP Cache, it returns one of these values.
Definition: http_cache.h:71
void AppendDefaultHeaders(const ContentType &content_type, GoogleString *text)
Append default headers to the given string.
GoogleString EncodeWithBase(const StringPiece &base, const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Definition: rewrite_test_base.h:352
Definition: rewrite_test_base.h:88
void AddOtherRewriteFilter(RewriteFilter *filter)
RewriteDriver * rewrite_driver()
Definition: rewrite_test_base.h:504
static const char kConfiguredBeaconingKey[]
Beaconing key values used when downstream caching is enabled.
Definition: rewrite_test_base.h:93
void SetCacheInvalidationTimestampForUrl(StringPiece url, bool ignores_metadata_and_pcache)
void VerifyRewriterInfoEntry(AbstractLogRecord *log_record, const GoogleString &id, int url_index, int rewriter_info_index, int rewriter_info_size, int url_list_size, const GoogleString &url)
void SetCacheInvalidationTimestamp()
void SetResponseWithDefaultHeaders(const StringPiece &relative_url, const ContentType &content_type, const StringPiece &content, int64 ttl_sec)
Add content to mock fetcher (with default headers).
void SetRewriteOptions(RewriteOptions *opts)
Set the RewriteOptions to be returned by the RewriteOptionsManager.
GoogleString AppliedRewriterStringFromLog()
static GoogleString ChangeSuffix(StringPiece old_url, bool append_new_suffix, StringPiece old_suffix, StringPiece new_suffix)
void AddOtherFilter(RewriteOptions::Filter filter)
Add a single rewrite filter to other_rewrite_driver_.
Definition: log_record.h:61
const UrlSegmentEncoder * FindEncoder(const StringPiece &id) const
void set_fail_on_unexpected(bool x)
Definition: mock_url_fetcher.h:113
GoogleString debug_message_
Message used by DebugMessage.
Definition: rewrite_test_base.h:792
MockScheduler * mock_scheduler()
The scheduler used by rewrite_driver.
Definition: rewrite_test_base.h:508
RewriteOptions * options_
owned by rewrite_driver_.
Definition: rewrite_test_base.h:785
void SetHtmlMimetype()
Sets the response-headers Content-Type to "text/html".
Definition: rewrite_test_base.h:598
void TestServeFiles(const ContentType *content_type, const StringPiece &filter_id, const StringPiece &rewritten_ext, const StringPiece &orig_name, const StringPiece &orig_content, const StringPiece &rewritten_name, const StringPiece &rewritten_content)
Helper method to test all manner of resource serving from a filter.
void SetMockLogRecord()
Sets MockLogRecord in the driver's request_context.
GoogleString GetLazyloadScriptHtml()
Helper methods to return js/html snippets related to lazyload images.
scoped_ptr< TestRewriteDriverFactory > factory_
Definition: rewrite_test_base.h:773
void AddFileToMockFetcher(const StringPiece &url, const StringPiece &filename, const ContentType &content_type, int64 ttl_sec)
Add the contents of a file to mock fetcher (with default headers).
GoogleString EncodeCssName(const StringPiece &name, bool supports_webp, bool can_inline)
LoggingInfo * logging_info()
void PopulateDefaultHeaders(const ContentType &content_type, int64 original_content_length, ResponseHeaders *headers)
const PropertyCache::Cohort * SetupCohort(PropertyCache *cache, const GoogleString &cohort)
Setup statistics for the given cohort and add it to the give PropertyCache.
Definition: rewrite_test_base.h:611
Definition: test_distributed_fetcher.h:37
void InitiateResourceRead(const ResourcePtr &resource)
HTTPCache::FindResult HttpBlockingFindWithOptions(const RewriteOptions *options, const GoogleString &key, HTTPCache *http_cache, HTTPValue *value_out, ResponseHeaders *headers)
Same as above, but with options (for invalidation checks)
bool AddShard(StringPiece domain, StringPiece shards)
void EnableDebug()
Enables the debug flag, which is often done on a test-by-test basis.
bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain)
Definition: rewrite_filter.h:37
virtual bool AddBody() const
Definition: rewrite_test_base.h:125
Definition: mock_url_fetcher.h:41
void set_update_date_headers(bool x)
Definition: mock_url_fetcher.h:120
Use all the other_ data members.
Definition: rewrite_test_base.h:104
void AddRewriteFilter(RewriteFilter *filter)
Filter
Definition: rewrite_options.h:104
Definition: rewrite_driver.h:98
void ClearRewriteDriver()
Definition: server_context.h:100
void SetXhtmlMimetype()
Sets the response-headers Content-Type to "application/xhtml+xml".
Definition: rewrite_test_base.h:595
MockUrlFetcher mock_url_fetcher_
Definition: rewrite_test_base.h:764
Definition: process_context.h:35
MockLogRecord * mock_log_record()
Returns the MockLogRecord in the driver.
Definition: wait_url_async_fetcher.h:37
void Clear()
Clear all set responses.
GoogleString EncodeNormal(const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Same as Encode but specifically using UrlNamer not TestUrlNamer.
Definition: rewrite_test_base.h:337
const MetadataCacheInfo & metadata_cache_info()
Convenience method to extract read-only metadata_cache_info.
Definition: rewrite_test_base.h:728
virtual TestRewriteDriverFactory * MakeTestFactory()
void SetBaseUrlForFetch(const StringPiece &url)
const GoogleString & CacheFragment() const
bool TryFetchResource(const StringPiece &url)
Just check if we can fetch a resource successfully, ignore response.
void AddRecompressImageFilters()
void SetUseTestUrlNamer(bool use_test_url_namer)
Switch url namers as specified.
Definition: rewrite_options_test_base.h:32
const GoogleString kEtag0
Etag with a 0 hash.
Definition: rewrite_test_base.h:789
Hasher * hasher() const
Definition: server_context.h:177
void SetShouldBeaconHeader(StringPiece rebeaconing_key)
Set ShouldBeacon request header to the specified value.
virtual RequestContextPtr CreateRequestContext()
GoogleString DebugMessage(StringPiece url)
ActiveServerFlag
Definition: rewrite_test_base.h:102
static const char kTestData[]
Testdata directory.
Definition: rewrite_test_base.h:90
void TestRetainExtraHeaders(const StringPiece &name, const StringPiece &filter_id, const StringPiece &ext)
RewriteDriver * MakeDriver(ServerContext *server_context, RewriteOptions *options)
Definition: rewrite_options.h:81
GoogleString AddOptionsToEncodedUrl(const StringPiece &url, const StringPiece &options)
Takes an already-encoded URL and adds options to to it.
void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher *fetcher, RewriteDriver *driver)
void AdvanceTimeUs(int64 delay_ms)
Timer * timer()
Definition: rewrite_test_base.h:183
Representation for a CSS <link> tag.
Definition: rewrite_test_base.h:250
void set_hasher(Hasher *hasher)
Setters should probably only be used in testing.
Definition: server_context.h:332
void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers)
static GoogleString CompositeKey(StringPiece key, StringPiece fragment)
Definition: http_cache.h:365
GoogleString AbsolutifyUrl(const StringPiece &in)
Converts a potentially relative URL off kTestDomain to absolute if needed.
const PropertyCache::Cohort * SetupCohort(PropertyCache *cache, const GoogleString &cohort_name)
Sets up the cohort in the PropertyCache provided.
Definition: resource_namer.h:32
const RequestContext::TimingInfo & timing_info()
Accessor for TimingInfo.
void AddFilter(RewriteOptions::Filter filter)
Add a single rewrite filter to rewrite_driver_.
void SetDownstreamCacheDirectives(StringPiece downstream_cache_purge_method, StringPiece downstream_cache_purge_location_prefix, StringPiece rebeaconing_key)