21 #ifndef NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
22 #define NET_INSTAWEB_REWRITER_PUBLIC_REWRITE_TEST_BASE_H_
28 #include "net/instaweb/http/public/logging_proto.h"
29 #include "net/instaweb/http/public/logging_proto_impl.h"
31 #include "net/instaweb/http/public/request_context.h"
39 #include "net/instaweb/util/public/mock_property_page.h"
40 #include "net/instaweb/util/public/property_cache.h"
41 #include "pagespeed/kernel/base/basictypes.h"
42 #include "pagespeed/kernel/base/md5_hasher.h"
43 #include "pagespeed/kernel/base/mem_file_system.h"
44 #include "pagespeed/kernel/base/mock_hasher.h"
45 #include "pagespeed/kernel/base/mock_message_handler.h"
47 #include "pagespeed/kernel/base/mock_timer.h"
48 #include "pagespeed/kernel/base/scoped_ptr.h"
49 #include "pagespeed/kernel/base/string.h"
50 #include "pagespeed/kernel/base/string_util.h"
51 #include "pagespeed/kernel/base/timer.h"
52 #include "pagespeed/kernel/html/html_parse_test_base.h"
53 #include "pagespeed/kernel/http/content_type.h"
54 #include "pagespeed/kernel/http/response_headers.h"
55 #include "pagespeed/kernel/http/user_agent_matcher.h"
56 #include "pagespeed/kernel/util/url_segment_encoder.h"
59 namespace net_instaweb {
61 class AbstractLogRecord;
62 class CountingUrlAsyncFetcher;
66 class HtmlWriterFilter;
73 class RequestTimingInfo;
77 class WaitUrlAsyncFetcher;
79 class RewriteOptionsTestBase :
public HtmlParseTestBaseNoAlloc {
81 RewriteOptionsTestBase() {
84 ~RewriteOptionsTestBase() {
85 RewriteOptions::Terminate();
95 static const char kWrongBeaconingKey[];
120 virtual void SetUp();
121 virtual void TearDown();
126 virtual bool AddBody()
const {
return false; }
171 StringPiece downstream_cache_purge_method,
172 StringPiece downstream_cache_purge_location_prefix,
173 StringPiece rebeaconing_key);
178 ResourcePtr CreateResource(
const StringPiece& base,
const StringPiece& url);
184 Timer*
timer() {
return factory()->mock_timer(); }
193 int64 original_content_length,
196 void ServeResourceFromManyContexts(
const GoogleString& resource_url,
197 const StringPiece& expected_content);
199 void ServeResourceFromManyContextsWithUA(
200 const GoogleString& resource_url,
201 const StringPiece& expected_content,
202 const StringPiece& user_agent);
207 const GoogleString& resource_url,
208 const StringPiece& expected_content);
217 ResponseHeaders* response_headers);
222 bool FetchResource(
const StringPiece& path,
const StringPiece& filter_id,
223 const StringPiece& name,
const StringPiece& ext,
224 GoogleString* content);
225 bool FetchResource(
const StringPiece& path,
const StringPiece& filter_id,
226 const StringPiece& name,
const StringPiece& ext,
227 GoogleString* content, ResponseHeaders* response);
229 bool FetchResourceUrl(
const StringPiece& url, GoogleString* content,
230 ResponseHeaders* response);
231 bool FetchResourceUrl(
const StringPiece& url,
232 RequestHeaders* request_headers,
233 GoogleString* content,
234 ResponseHeaders* response_headers);
235 bool FetchResourceUrl(
const StringPiece& url, GoogleString* content);
246 GoogleString CssLinkHref(
const StringPiece& url) {
247 return StrCat(
"<link rel=stylesheet href=", url,
">");
253 CssLink(
const StringPiece& url,
const StringPiece& content,
254 const StringPiece& media,
bool supply_mock);
257 class Vector :
public std::vector<CssLink*> {
260 void Add(
const StringPiece& url,
const StringPiece& content,
261 const StringPiece& media,
bool supply_mock);
267 StringVector* segments, MessageHandler* handler);
270 GoogleString content_;
277 StringVector* css_links);
285 const StringPiece& hash,
286 const StringVector& name_vector,
287 const StringPiece& ext,
290 StringVector MultiUrl(
const StringPiece& url1) {
292 v.push_back(url1.as_string());
296 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2) {
298 v.push_back(url1.as_string());
299 v.push_back(url2.as_string());
303 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2,
304 const StringPiece& url3) {
306 v.push_back(url1.as_string());
307 v.push_back(url2.as_string());
308 v.push_back(url3.as_string());
312 StringVector MultiUrl(
const StringPiece& url1,
const StringPiece& url2,
313 const StringPiece& url3,
const StringPiece& url4) {
315 v.push_back(url1.as_string());
316 v.push_back(url2.as_string());
317 v.push_back(url3.as_string());
318 v.push_back(url4.as_string());
324 GoogleString
Encode(
const StringPiece& path,
325 const StringPiece& filter_id,
326 const StringPiece& hash,
327 const StringPiece& name,
328 const StringPiece& ext) {
329 return Encode(path, filter_id, hash, MultiUrl(name), ext);
331 GoogleString
Encode(
const StringPiece& path,
332 const StringPiece& filter_id,
333 const StringPiece& hash,
334 const StringVector& name_vector,
335 const StringPiece& ext);
339 const StringPiece& filter_id,
340 const StringPiece& hash,
341 const StringPiece& name,
342 const StringPiece& ext) {
343 return EncodeNormal(path, filter_id, hash, MultiUrl(name), ext);
346 const StringPiece& filter_id,
347 const StringPiece& hash,
348 const StringVector& name_vector,
349 const StringPiece& ext);
354 const StringPiece& path,
355 const StringPiece& filter_id,
356 const StringPiece& hash,
357 const StringPiece& name,
358 const StringPiece& ext) {
359 return EncodeWithBase(base, path, filter_id, hash, MultiUrl(name), ext);
362 const StringPiece& path,
363 const StringPiece& filter_id,
364 const StringPiece& hash,
365 const StringVector& name_vector,
366 const StringPiece& ext);
370 const StringPiece& options);
377 StringPiece old_url,
bool append_new_suffix,
378 StringPiece old_suffix, StringPiece new_suffix);
385 void CallFetcherCallbacks();
386 void OtherCallFetcherCallbacks();
395 bool AddDomain(StringPiece domain);
407 bool AddShard(StringPiece domain, StringPiece shards);
411 const StringPiece& filter_id,
412 const StringPiece& rewritten_ext,
413 const StringPiece& orig_name,
414 const StringPiece& orig_content,
415 const StringPiece& rewritten_name,
416 const StringPiece& rewritten_content);
418 TestRewriteDriverFactory* factory() {
return factory_.get(); }
419 TestRewriteDriverFactory* other_factory() {
return other_factory_.get(); }
421 void UseMd5Hasher() {
423 server_context_->http_cache()->set_hasher(&md5_hasher_);
424 other_server_context_->
set_hasher(&md5_hasher_);
425 other_server_context_->http_cache()->set_hasher(&md5_hasher_);
429 void SetDefaultLongCacheHeaders(
const ContentType* content_type,
430 ResponseHeaders* header) {
432 content_type, StringPiece(), StringPiece(), header);
435 void SetFetchResponse(
const StringPiece& url,
436 const ResponseHeaders& response_header,
437 const StringPiece& response_body) {
438 mock_url_fetcher()->SetResponse(url, response_header, response_body);
443 const ContentType& content_type,
444 const StringPiece& content,
449 bool LoadFile(
const StringPiece& filename, GoogleString* contents);
453 const StringPiece& filename,
454 const ContentType& content_type, int64 ttl_sec);
456 void AddToResponse(
const StringPiece& url,
457 const StringPiece& name,
458 const StringPiece& value) {
462 void SetFetchResponse404(
const StringPiece& url);
464 void SetFetchFailOnUnexpected(
bool fail) {
467 void FetcherUpdateDateHeaders() {
468 mock_url_fetcher()->set_timer(
timer());
471 void ClearFetcherResponses() { mock_url_fetcher()->
Clear(); }
473 virtual void ClearStats();
479 MockUrlFetcher* mock_url_fetcher() {
482 TestDistributedFetcher* test_distributed_fetcher() {
483 return &test_distributed_fetcher_;
485 Hasher* hasher() {
return server_context_->
hasher(); }
486 DelayCache* delay_cache() {
return factory_->delay_cache(); }
487 LRUCache* lru_cache() {
return factory_->lru_cache(); }
488 Statistics* statistics() {
return factory_->statistics(); }
489 MemFileSystem* file_system() {
return factory_->mem_file_system(); }
490 HTTPCache* http_cache() {
return server_context_->http_cache(); }
491 PropertyCache* page_property_cache() {
492 return server_context_->page_property_cache();
494 MockMessageHandler* message_handler() {
495 return factory_->mock_message_handler();
506 RewriteDriver* other_rewrite_driver() {
return other_rewrite_driver_; }
511 int64 start_time_ms()
const {
return factory_->kStartTimeMs; }
513 bool ReadFile(
const char* filename, GoogleString* contents) {
514 return file_system()->ReadFile(filename, contents, message_handler());
516 bool WriteFile(
const char* filename,
const StringPiece& contents) {
517 return file_system()->WriteFile(filename, contents, message_handler());
520 ServerContext* server_context() {
return server_context_; }
521 ServerContext* other_server_context() {
return other_server_context_; }
522 CountingUrlAsyncFetcher* counting_url_async_fetcher() {
523 return factory_->counting_url_async_fetcher();
525 CountingUrlAsyncFetcher* counting_distributed_fetcher() {
526 return factory_->counting_distributed_async_fetcher();
528 void SetMockHashValue(
const GoogleString& value) {
529 factory_->mock_hasher()->set_hash_value(value);
532 void SetCacheDelayUs(int64 delay_us);
538 RewriteDriver*
MakeDriver(ServerContext* server_context,
539 RewriteOptions* options);
549 const StringPiece& filter_id,
550 const StringPiece& ext);
555 const UrlSegmentEncoder*
FindEncoder(
const StringPiece&
id)
const;
581 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
582 ResponseHeaders* headers);
587 const GoogleString& key, HTTPCache* http_cache);
591 const RewriteOptions* options,
592 const GoogleString& key, HTTPCache* http_cache, HTTPValue* value_out,
593 ResponseHeaders* headers);
608 StringPiece expected_contents,
609 int64 expected_expiration_ms);
613 PropertyCache* cache,
const GoogleString& cohort) {
623 const StringPiece& options_signature_hash,
624 UserAgentMatcher::DeviceType device_type) {
625 return new MockPropertyPage(
626 server_context_->thread_system(),
627 server_context_->page_property_cache(),
629 options_signature_hash,
630 UserAgentMatcher::DeviceTypeSuffix(device_type));
633 MockPropertyPage*
NewMockPage(
const StringPiece& url) {
634 return NewMockPage(url,
"hash", UserAgentMatcher::kDesktop);
645 GoogleString GetLazyloadPostscriptHtml();
657 StringPiece url,
bool ignores_metadata_and_pcache);
701 int64 original_content_length,
702 ResponseHeaders* headers);
711 void AdvanceTimeMs(int64 delay_ms) {
AdvanceTimeUs(delay_ms * Timer::kMsUs); }
712 void SetTimeUs(int64 time_us);
713 void SetTimeMs(int64 time_ms) { SetTimeUs(time_ms * Timer::kMsUs); }
720 RequestTimingInfo* mutable_timing_info();
740 const GoogleString&
id,
742 int rewriter_info_index,
743 int rewriter_info_size,
745 const GoogleString& url);
749 current_user_agent_ = user_agent;
752 GoogleString ExpectedNonce();
767 scoped_ptr<Statistics> statistics_;
775 scoped_ptr<TestRewriteDriverFactory> other_factory_;
780 scoped_ptr<HtmlWriterFilter> other_html_writer_filter_;
782 bool use_managed_rewrite_drivers_;
783 StringPiece current_user_agent_;
784 MD5Hasher md5_hasher_;
788 UrlSegmentEncoder default_encoder_;
789 ResponseHeaders response_headers_;
791 uint64 expected_nonce_;
void EncodePathAndLeaf(const StringPiece &filter_id, const StringPiece &hash, const StringVector &name_vector, const StringPiece &ext, ResourceNamer *namer)
Encode the given name (path + leaf) using the given pagespeed attributes.
Definition: test_rewrite_driver_factory.h:67
void AddFetchOnlyRewriteFilter(RewriteFilter *filter)
virtual RewriteDriver * html_parse()
Definition: rewrite_test_base.h:213
void SetActiveServer(ActiveServerFlag server_to_use)
void SetCurrentUserAgent(const StringPiece &user_agent)
Sets current_user_agent_.
Definition: rewrite_test_base.h:748
HTTPCache::FindResult HttpBlockingFind(const GoogleString &key, HTTPCache *http_cache, HTTPValue *value_out, ResponseHeaders *headers)
void DefaultResponseHeaders(const ContentType &content_type, int64 ttl_sec, ResponseHeaders *response_headers)
Set default headers for a resource with content_type and Cache ttl_sec.
void CollectCssLinks(const StringPiece &id, const StringPiece &html, StringVector *css_links)
Collects the hrefs for all CSS <link>s on the page.
GoogleString HttpCacheKey(StringPiece url)
Definition: rewrite_test_base.h:756
Use the normal data members.
Definition: rewrite_test_base.h:104
bool FetchResource(const StringPiece &path, const StringPiece &filter_id, const StringPiece &name, const StringPiece &ext, GoogleString *content)
GoogleString Encode(const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Definition: rewrite_test_base.h:324
A vector of CssLink* should know how to accumulate and add.
Definition: rewrite_test_base.h:257
void AdjustTimeUsWithoutWakingAlarms(int64 time_us)
Adjusts time ignoring any scheduler callbacks. Use with caution.
bool DecomposeCombinedUrl(StringPiece base_url, GoogleString *base, StringVector *segments, MessageHandler *handler)
int TimedValue(StringPiece name)
Returns the value of a TimedVariable, specified by name.
static const ProcessContext & process_context()
void CheckFetchFromHttpCache(StringPiece url, StringPiece expected_contents, int64 expected_expiration_ms)
bool AddRewriteDomainMapping(StringPiece to_domain, StringPiece from_domain)
void SetDefaultLongCacheHeaders(const ContentType *content_type, StringPiece charset, StringPiece cache_control_suffix, ResponseHeaders *header) const
void SetMimetype(const StringPiece &mimetype)
Sets the response-headers Content-Type as specified.
RewriteOptions * other_options_
owned by other_rewrite_driver_.
Definition: rewrite_test_base.h:787
void ServeResourceFromNewContext(const GoogleString &resource_url, const StringPiece &expected_content)
void SetDummyRequestHeaders()
Setup dummy empty RequestHeaders object for the driver.
MockPropertyPage * NewMockPage(const StringPiece &url, const StringPiece &options_signature_hash, UserAgentMatcher::DeviceType device_type)
Returns a new mock property page for the page property cache.
Definition: rewrite_test_base.h:622
void DebugWithMessage(StringPiece expected_debug_message)
Definition: rewrite_test_base.h:670
HTTPCache::FindResult HttpBlockingFindStatus(const GoogleString &key, HTTPCache *http_cache)
void AddToResponse(const StringPiece &url, const StringPiece &name, const StringPiece &value)
bool ReadIfCached(const ResourcePtr &resource)
bool LoadFile(const StringPiece &filename, GoogleString *contents)
FindResult
When a lookup is done in the HTTP Cache, it returns one of these values.
Definition: http_cache.h:71
void AppendDefaultHeaders(const ContentType &content_type, GoogleString *text)
Append default headers to the given string.
GoogleString EncodeWithBase(const StringPiece &base, const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Definition: rewrite_test_base.h:353
Definition: rewrite_test_base.h:89
void AddOtherRewriteFilter(RewriteFilter *filter)
RewriteDriver * rewrite_driver()
Definition: rewrite_test_base.h:505
static const char kConfiguredBeaconingKey[]
Beaconing key values used when downstream caching is enabled.
Definition: rewrite_test_base.h:94
void SetCacheInvalidationTimestampForUrl(StringPiece url, bool ignores_metadata_and_pcache)
void VerifyRewriterInfoEntry(AbstractLogRecord *log_record, const GoogleString &id, int url_index, int rewriter_info_index, int rewriter_info_size, int url_list_size, const GoogleString &url)
void SetCacheInvalidationTimestamp()
void SetResponseWithDefaultHeaders(const StringPiece &relative_url, const ContentType &content_type, const StringPiece &content, int64 ttl_sec)
Add content to mock fetcher (with default headers).
void SetRewriteOptions(RewriteOptions *opts)
Set the RewriteOptions to be returned by the RewriteOptionsManager.
GoogleString AppliedRewriterStringFromLog()
static GoogleString ChangeSuffix(StringPiece old_url, bool append_new_suffix, StringPiece old_suffix, StringPiece new_suffix)
void AddOtherFilter(RewriteOptions::Filter filter)
Add a single rewrite filter to other_rewrite_driver_.
const UrlSegmentEncoder * FindEncoder(const StringPiece &id) const
void set_fail_on_unexpected(bool x)
Definition: mock_url_fetcher.h:113
GoogleString debug_message_
Message used by DebugMessage.
Definition: rewrite_test_base.h:793
MockScheduler * mock_scheduler()
The scheduler used by rewrite_driver.
Definition: rewrite_test_base.h:509
RewriteOptions * options_
owned by rewrite_driver_.
Definition: rewrite_test_base.h:786
void SetHtmlMimetype()
Sets the response-headers Content-Type to "text/html".
Definition: rewrite_test_base.h:599
void TestServeFiles(const ContentType *content_type, const StringPiece &filter_id, const StringPiece &rewritten_ext, const StringPiece &orig_name, const StringPiece &orig_content, const StringPiece &rewritten_name, const StringPiece &rewritten_content)
Helper method to test all manner of resource serving from a filter.
void SetMockLogRecord()
Sets MockLogRecord in the driver's request_context.
GoogleString GetLazyloadScriptHtml()
Helper methods to return js/html snippets related to lazyload images.
scoped_ptr< TestRewriteDriverFactory > factory_
Definition: rewrite_test_base.h:774
void AddFileToMockFetcher(const StringPiece &url, const StringPiece &filename, const ContentType &content_type, int64 ttl_sec)
Add the contents of a file to mock fetcher (with default headers).
GoogleString EncodeCssName(const StringPiece &name, bool supports_webp, bool can_inline)
LoggingInfo * logging_info()
void PopulateDefaultHeaders(const ContentType &content_type, int64 original_content_length, ResponseHeaders *headers)
const PropertyCache::Cohort * SetupCohort(PropertyCache *cache, const GoogleString &cohort)
Setup statistics for the given cohort and add it to the give PropertyCache.
Definition: rewrite_test_base.h:612
Definition: test_distributed_fetcher.h:37
void InitiateResourceRead(const ResourcePtr &resource)
HTTPCache::FindResult HttpBlockingFindWithOptions(const RewriteOptions *options, const GoogleString &key, HTTPCache *http_cache, HTTPValue *value_out, ResponseHeaders *headers)
Same as above, but with options (for invalidation checks)
bool AddShard(StringPiece domain, StringPiece shards)
void EnableDebug()
Enables the debug flag, which is often done on a test-by-test basis.
bool AddOriginDomainMapping(StringPiece to_domain, StringPiece from_domain)
Definition: rewrite_filter.h:37
virtual bool AddBody() const
Definition: rewrite_test_base.h:126
Definition: mock_url_fetcher.h:41
void set_update_date_headers(bool x)
Definition: mock_url_fetcher.h:120
Use all the other_ data members.
Definition: rewrite_test_base.h:105
void AddRewriteFilter(RewriteFilter *filter)
Filter
Definition: rewrite_options.h:106
Definition: rewrite_driver.h:98
void ClearRewriteDriver()
Definition: server_context.h:101
void SetXhtmlMimetype()
Sets the response-headers Content-Type to "application/xhtml+xml".
Definition: rewrite_test_base.h:596
MockUrlFetcher mock_url_fetcher_
Definition: rewrite_test_base.h:765
Definition: process_context.h:35
MockLogRecord * mock_log_record()
Returns the MockLogRecord in the driver.
Definition: wait_url_async_fetcher.h:37
void Clear()
Clear all set responses.
GoogleString EncodeNormal(const StringPiece &path, const StringPiece &filter_id, const StringPiece &hash, const StringPiece &name, const StringPiece &ext)
Same as Encode but specifically using UrlNamer not TestUrlNamer.
Definition: rewrite_test_base.h:338
const MetadataCacheInfo & metadata_cache_info()
Convenience method to extract read-only metadata_cache_info.
Definition: rewrite_test_base.h:729
virtual TestRewriteDriverFactory * MakeTestFactory()
const RequestTimingInfo & timing_info()
Accessor for TimingInfo.
void SetBaseUrlForFetch(const StringPiece &url)
const GoogleString & CacheFragment() const
bool TryFetchResource(const StringPiece &url)
Just check if we can fetch a resource successfully, ignore response.
void AddRecompressImageFilters()
void SetUseTestUrlNamer(bool use_test_url_namer)
Switch url namers as specified.
Definition: rewrite_options_test_base.h:32
const GoogleString kEtag0
Etag with a 0 hash.
Definition: rewrite_test_base.h:790
Hasher * hasher() const
Definition: server_context.h:178
void SetShouldBeaconHeader(StringPiece rebeaconing_key)
Set ShouldBeacon request header to the specified value.
virtual RequestContextPtr CreateRequestContext()
GoogleString DebugMessage(StringPiece url)
ActiveServerFlag
Definition: rewrite_test_base.h:103
static const char kTestData[]
Testdata directory.
Definition: rewrite_test_base.h:91
void TestRetainExtraHeaders(const StringPiece &name, const StringPiece &filter_id, const StringPiece &ext)
RewriteDriver * MakeDriver(ServerContext *server_context, RewriteOptions *options)
Definition: rewrite_options.h:83
GoogleString AddOptionsToEncodedUrl(const StringPiece &url, const StringPiece &options)
Takes an already-encoded URL and adds options to to it.
void CallFetcherCallbacksForDriver(WaitUrlAsyncFetcher *fetcher, RewriteDriver *driver)
void AdvanceTimeUs(int64 delay_ms)
Timer * timer()
Definition: rewrite_test_base.h:184
Representation for a CSS <link> tag.
Definition: rewrite_test_base.h:251
void set_hasher(Hasher *hasher)
Setters should probably only be used in testing.
Definition: server_context.h:340
void SetUseManagedRewriteDrivers(bool use_managed_rewrite_drivers)
static GoogleString CompositeKey(StringPiece key, StringPiece fragment)
Definition: http_cache.h:365
GoogleString AbsolutifyUrl(const StringPiece &in)
Converts a potentially relative URL off kTestDomain to absolute if needed.
const PropertyCache::Cohort * SetupCohort(PropertyCache *cache, const GoogleString &cohort_name)
Sets up the cohort in the PropertyCache provided.
Definition: resource_namer.h:32
void AddFilter(RewriteOptions::Filter filter)
Add a single rewrite filter to rewrite_driver_.
void SetDownstreamCacheDirectives(StringPiece downstream_cache_purge_method, StringPiece downstream_cache_purge_location_prefix, StringPiece rebeaconing_key)