#include "rewrite_driver.h"

Inheritance diagram for net_instaweb::RewriteDriver:

Public Types
enum	CssResolutionStatus { kWriteFailed, kNoResolutionNeeded, kSuccess }
	Status return-code for ResolveCssUrls.

enum	WaitMode { kNoWait, kWaitForCompletion, kWaitForCachedRender, kWaitForShutDown }
	Mode for BoundedWaitForCompletion. More...

enum	XhtmlStatus { kXhtmlUnknown, kIsXhtml, kIsNotXhtml }

enum	InlineAuthorizationPolicy { kInlineUnauthorizedResources, kInlineOnlyAuthorizedResources }
	See CreateInputResource.

enum	IntendedFor { kIntendedForInlining, kIntendedForGeneral }
	See CreateInputResource.

enum	InputRole { kScript, kStyle, kImg, InputRole::kUnknown, InputRole::kReconstruction }

Public Member Functions
	RewriteDriver (MessageHandler message_handler, FileSystem file_system, UrlAsyncFetcher *url_async_fetcher)

virtual	~RewriteDriver ()

RewriteDriver *	Clone ()

void	Clear ()

void	SetServerContext (ServerContext *server_context)

bool	MayCacheExtendCss () const

bool	MayCacheExtendImages () const

bool	MayCacheExtendPdfs () const

bool	MayCacheExtendScripts () const

const GoogleString &	user_agent () const

const RequestProperties *	request_properties () const

void	ClearRequestProperties ()
	Reinitializes request_properties_, clearing any cached values.

bool	write_property_cache_dom_cohort () const

void	set_write_property_cache_dom_cohort (bool x)

void	PropertyCacheSetupDone ()

RequestContextPtr	request_context ()

void	set_request_context (const RequestContextPtr &x)

RequestTrace *	trace_context ()

void	TracePrintf (const char *fmt,...)

void	TraceLiteral (const char *literal)

void	TraceString (const GoogleString &s)

ResponseHeaders *	mutable_response_headers ()

const ResponseHeaders *	response_headers ()

void	set_response_headers_ptr (ResponseHeaders *headers)

void	SetRequestHeaders (const RequestHeaders &headers)

const RequestHeaders *	request_headers () const

UserAgentMatcher *	user_agent_matcher () const

void	AddFilters ()

void	AddOwnedEarlyPreRenderFilter (HtmlFilter *filter)

void	PrependOwnedPreRenderFilter (HtmlFilter *filter)
	Adds a filter to the beginning of the pre-render chain, taking ownership.

void	AppendOwnedPreRenderFilter (HtmlFilter *filter)
	Adds a filter to the end of the pre-render chain, taking ownership.

void	AppendUnownedPreRenderFilter (HtmlFilter *filter)
	Same, without taking ownership.

void	AddOwnedPostRenderFilter (HtmlFilter *filter)
	Adds a filter to the end of the post-render chain, taking ownership.

void	AddUnownedPostRenderFilter (HtmlFilter *filter)
	Same, without taking ownership.

void	AppendRewriteFilter (RewriteFilter *filter)

void	PrependRewriteFilter (RewriteFilter *filter)

void	AddResourceUrlClaimant (ResourceUrlClaimant *claimant)

void	SetWriter (Writer *writer)

Writer *	writer () const

bool	FetchResource (const StringPiece &url, AsyncFetch *fetch)

void	FetchInPlaceResource (const GoogleUrl &gurl, bool proxy_mode, AsyncFetch *async_fetch)

bool	FetchOutputResource (const OutputResourcePtr &output_resource, RewriteFilter filter, AsyncFetch async_fetch)

OutputResourcePtr	DecodeOutputResource (const GoogleUrl &url, RewriteFilter **filter) const

bool	DecodeOutputResourceName (const GoogleUrl &url, const RewriteOptions options_to_use, const UrlNamer url_namer, ResourceNamer name_out, OutputResourceKind kind_out, RewriteFilter **filter_out) const

bool	LookupMetadataForOutputResource (StringPiece url, GoogleString error_out, RewriteContext::CacheLookupResultCallback callback)

bool	DecodeUrl (const GoogleUrl &url, StringVector *decoded_urls) const
	Decodes the incoming pagespeed url to original url(s).

bool	DecodeUrlGivenOptions (const GoogleUrl &url, const RewriteOptions options, const UrlNamer url_namer, StringVector *decoded_urls) const

FileSystem *	file_system ()

UrlAsyncFetcher *	async_fetcher ()

void	SetSessionFetcher (UrlAsyncFetcher *f)

CacheUrlAsyncFetcher *	CreateCacheFetcher ()

CacheUrlAsyncFetcher *	CreateCacheOnlyFetcher ()
	Returns a cache fetcher that does not fall back to an actual fetcher.

ServerContext *	server_context () const

Statistics *	statistics () const

void	set_custom_options (RewriteOptions *options)
	Takes ownership of 'options'.

void	set_options_for_pool (RewriteDriverPool pool, RewriteOptions options)

RewriteDriverPool *	controlling_pool ()
	Pool in which this driver can be recycled. May be NULL.

const RewriteOptions *	options () const
	Return the options used for this RewriteDriver.

virtual bool	StartParseId (const StringPiece &url, const StringPiece &id, const ContentType &content_type)

virtual void	FinishParse ()

void	FinishParseAsync (Function *callback)

void	InfoAt (const RewriteContext context, const char msg,...) INSTAWEB_PRINTF_FORMAT(3

void bool	GenerateOutputResourceNameAndUrl (const UrlSegmentEncoder encoder, const ResourceContext data, const ResourcePtr &input_resource, GoogleString name, GoogleUrl mapped_gurl, GoogleString *failure_reason)
	Constructs name and URL for the specified input resource and encoder.

OutputResourcePtr	CreateOutputResourceFromResource (const char filter_id, const UrlSegmentEncoder encoder, const ResourceContext data, const ResourcePtr &input_resource, OutputResourceKind kind, GoogleString failure_reason)

OutputResourcePtr	CreateOutputResourceWithPath (const StringPiece &mapped_path, const StringPiece &unmapped_path, const StringPiece &base_url, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)

void	PopulateResourceNamer (const StringPiece &filter_id, const StringPiece &name, ResourceNamer *full_name)

OutputResourcePtr	CreateOutputResourceWithUnmappedUrl (const GoogleUrl &unmapped_gurl, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)

OutputResourcePtr	CreateOutputResourceWithMappedPath (const StringPiece &mapped_path, const StringPiece &unmapped_path, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)

OutputResourcePtr	CreateOutputResourceWithPath (const StringPiece &path, const StringPiece &filter_id, const StringPiece &name, OutputResourceKind kind, GoogleString *failure_reason)

ResourcePtr	CreateInputResource (const GoogleUrl &input_url, InputRole role, bool *is_authorized)

ResourcePtr	CreateInputResource (const GoogleUrl &input_url, InlineAuthorizationPolicy inline_authorization_policy, IntendedFor intended_for, InputRole role, bool *is_authorized)

ResourcePtr	CreateInputResourceAbsoluteUncheckedForTestsOnly (const StringPiece &absolute_url)

bool	IsResourceUrlClaimed (const GoogleUrl &url) const

bool	MatchesBaseUrl (const GoogleUrl &input_url) const

bool	MayRewriteUrl (const GoogleUrl &domain_url, const GoogleUrl &input_url, InlineAuthorizationPolicy inline_authorization_policy, IntendedFor intended_for, bool *is_authorized_domain) const

const GoogleUrl &	base_url () const

StringPiece	fetch_url () const
	The URL that was requested if FetchResource was called.

const GoogleUrl &	decoded_base_url () const

StringPiece	decoded_base () const

bool	IsHttps () const
	Quick way to tell if the document url is https (ie was fetched via https).

const UrlSegmentEncoder *	default_encoder () const

RewriteFilter *	FindFilter (const StringPiece &id) const
	Finds a filter with the given ID, or returns NULL if none found.

bool	refs_before_base () const
	Returns refs_before_base.

bool	other_base_problem () const

void	set_refs_before_base ()

void	set_other_base_problem ()
	Sets if we had other difficulty handling <base> tag.

StringPiece	containing_charset ()

void	set_containing_charset (const StringPiece charset)

HtmlResourceSlotPtr	GetSlot (const ResourcePtr &resource, HtmlElement elt, HtmlElement::Attribute attr)

InlineResourceSlotPtr	GetInlineSlot (const ResourcePtr &resource, HtmlCharactersNode *char_node)

InlineAttributeSlotPtr	GetInlineAttributeSlot (const ResourcePtr &resource, HtmlElement element, HtmlElement::Attribute attribute)

SrcSetSlotCollectionPtr	GetSrcSetSlotCollection (CommonFilter filter, HtmlElement element, HtmlElement::Attribute *attr)

bool	InitiateRewrite (RewriteContext *rewrite_context) LOCKS_EXCLUDED(rewrite_mutex())

void	InitiateFetch (RewriteContext *rewrite_context)

void	RewriteComplete (RewriteContext *rewrite_context, RenderOp permit_render)

void	ReportSlowRewrites (int num)

void	Cleanup ()

void	AddUserReference ()

GoogleString	ToString (bool show_detached_contexts) const LOCKS_EXCLUDED(rewrite_mutex())
	Debugging routines to print out data about the driver.

GoogleString	ToStringLockHeld (bool show_detached_contexts) const EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex())

void	PrintState (bool show_detached_contexts)
	For debugging.

void	PrintStateToErrorLog (bool show_detached_contexts)
	For logs.

void	WaitForCompletion ()

void	WaitForShutDown ()

void	BoundedWaitFor (WaitMode mode, int64 timeout_ms) LOCKS_EXCLUDED(rewrite_mutex())

void	set_fully_rewrite_on_flush (bool x)

bool	fully_rewrite_on_flush () const
	Returns if this response has a blocking rewrite or not.

void	set_fast_blocking_rewrite (bool x)

bool	fast_blocking_rewrite () const

void	EnableBlockingRewrite (RequestHeaders *request_headers)

void	set_externally_managed (bool x)

void	DetachFetch ()

void	DetachedFetchComplete ()

void	FetchComplete ()

void	DeleteRewriteContext (RewriteContext *rewrite_context)

int	rewrite_deadline_ms ()

void	set_max_page_processing_delay_ms (int x)

int	max_page_processing_delay_ms ()

void	set_device_type (UserAgentMatcher::DeviceType x)
	Sets the device type chosen for the current property_page.

UserAgentMatcher::DeviceType	device_type () const

RewriteContext *	RegisterForPartitionKey (const GoogleString &partition_key, RewriteContext *candidate)

void	DeregisterForPartitionKey (const GoogleString &partition_key, RewriteContext *candidate)

void	RequestFlush ()

bool	flush_requested () const

void	ExecuteFlushIfRequested ()

void	ExecuteFlushIfRequestedAsync (Function *callback)

virtual void	Flush ()

void	FlushAsync (Function *done)

void	AddRewriteTask (Function *task)
	Queues up a task to run on the (high-priority) rewrite thread.

void	AddLowPriorityRewriteTask (Function *task)

QueuedWorkerPool::Sequence *	html_worker ()

Sequence *	rewrite_worker ()

Scheduler::Sequence *	scheduler_sequence ()

QueuedWorkerPool::Sequence *	low_priority_rewrite_worker ()

void	RunTasksOnRequestThread ()

void	SwitchToQueuedWorkerPool () EXCLUSIVE_LOCKS_REQUIRED(rewrite_mutex())

Scheduler *	scheduler ()

DomainRewriteFilter *	domain_rewriter ()

UrlLeftTrimFilter *	url_trim_filter ()

CssResolutionStatus	ResolveCssUrls (const GoogleUrl &input_css_base, const StringPiece &output_css_base, const StringPiece &contents, Writer writer, MessageHandler handler)

bool	ShouldAbsolutifyUrl (const GoogleUrl &input_base, const GoogleUrl &output_base, bool *proxy_mode) const

void	UpdatePropertyValueInDomCohort (AbstractPropertyPage *page, StringPiece property_name, StringPiece property_value)

PropertyPage *	property_page () const

FallbackPropertyPage *	fallback_property_page () const

PropertyPage *	origin_property_page () const

void	set_property_page (PropertyPage *page)
	Takes ownership of page.

void	set_fallback_property_page (FallbackPropertyPage *page)
	Takes ownership of page.

void	set_unowned_fallback_property_page (FallbackPropertyPage *page)
	Does not take the ownership of the page.

void	set_origin_property_page (PropertyPage *page)
	Takes ownership of page.

bool	is_critical_images_beacon_enabled ()

CriticalImagesInfo *	critical_images_info () const
	Used by ImageRewriteFilter for identifying critical images.

CriticalSelectorInfo *	critical_selector_info ()

void	set_critical_selector_info (CriticalSelectorInfo *info)

void	set_critical_images_info (CriticalImagesInfo *critical_images_info)

bool	FlattenCssImportsEnabled () const

int	num_inline_preview_images () const

void	increment_num_inline_preview_images ()
	We expect to this method to be called on the HTML parser thread.

void	IncrementAsyncEventsCount ()

void	DecrementAsyncEventsCount ()
	Decrements a reference count bumped up by IncrementAsyncEventsCount()

void	IncrementRenderBlockingAsyncEventsCount ()

void	DecrementRenderBlockingAsyncEventsCount ()

XhtmlStatus	MimeTypeXhtmlStatus ()

void	set_is_lazyload_script_flushed (bool x)

bool	is_lazyload_script_flushed () const

FlushEarlyInfo *	flush_early_info ()
	This method is not thread-safe. Call it only from the html parser thread.

DependencyTracker *	dependency_tracker () const

bool	DebugMode () const

void	InsertDebugComment (StringPiece unescaped_message, HtmlNode *node)

void	InsertDebugComments (const protobuf::RepeatedPtrField< GoogleString > &unescaped_messages, HtmlElement *element)

void	InsertUnauthorizedDomainDebugComment (StringPiece url, InputRole role, HtmlElement *element)

GoogleString	GenerateUnauthorizedDomainDebugComment (const GoogleUrl &gurl, InputRole role)
	Generates an unauthorized domain debug comment. Public for unit tests.

AbstractLogRecord *	log_record ()

DomStatsFilter *	dom_stats_filter () const

bool	can_rewrite_resources () const

bool	is_nested () const
	Determine whether this driver is nested inside another.

bool	Write (const ResourceVector &inputs, const StringPiece &contents, const ContentType type, StringPiece charset, OutputResource output)

void	set_defer_instrumentation_script (bool x)

bool	defer_instrumentation_script () const

void	set_num_initiated_rewrites (int64 x)

int64	num_initiated_rewrites () const

void	set_num_detached_rewrites (int64 x)
	Sets the num_detached_rewrites_. This should only be called from test code.

int64	num_detached_rewrites () const

void	set_pagespeed_query_params (StringPiece x)

StringPiece	pagespeed_query_params () const

void	set_pagespeed_option_cookies (StringPiece x)

StringPiece	pagespeed_option_cookies () const

const GoogleString &	CacheFragment () const

bool	SetOrClearPageSpeedOptionCookies (const GoogleUrl &gurl, ResponseHeaders *response_headers)

bool	Decode (StringPiece leaf, ResourceNamer *resource_namer) const

bool	filters_added () const

bool	has_html_writer_filter () const

void	SetIsAmpDocument (bool is_amp)

bool	is_amp_document () const

const CspContext &	content_security_policy () const

CspContext *	mutable_content_security_policy ()

bool	IsLoadPermittedByCsp (const GoogleUrl &url, InputRole role)

bool	IsLoadPermittedByCsp (const GoogleUrl &url, CspDirective role)

Public Member Functions inherited from net_instaweb::HtmlParse
	HtmlParse (MessageHandler *message_handler)

void	AddFilter (HtmlFilter *filter)
	Application methods for parsing functions and adding filters. More...

bool	StartParse (const StringPiece &url)

bool	StartParseWithType (const StringPiece &url, const ContentType &content_type)

bool	is_url_valid () const
	Returns whether the google_url() URL is valid.

void	SetUrlForTesting (const StringPiece &url)

void	ParseText (const char *content, int size)

void	ParseText (const StringPiece &sp)

HtmlCdataNode *	NewCdataNode (HtmlElement *parent, const StringPiece &contents)
	Utility methods for implementing filters. More...

HtmlCharactersNode *	NewCharactersNode (HtmlElement *parent, const StringPiece &literal)

HtmlCommentNode *	NewCommentNode (HtmlElement *parent, const StringPiece &contents)

HtmlDirectiveNode *	NewDirectiveNode (HtmlElement *parent, const StringPiece &contents)

HtmlIEDirectiveNode *	NewIEDirectiveNode (HtmlElement *parent, const StringPiece &contents)

void	InsertScriptAfterCurrent (StringPiece text, bool external)

void	InsertScriptBeforeCurrent (StringPiece text, bool external)

HtmlElement *	AppendAnchor (StringPiece link, StringPiece text, HtmlElement *parent)

void	InsertNodeBeforeNode (const HtmlNode existing_node, HtmlNode new_node)

void	InsertNodeAfterNode (const HtmlNode existing_node, HtmlNode new_node)

void	InsertElementBeforeElement (const HtmlNode existing_element, HtmlNode new_element)

void	InsertElementAfterElement (const HtmlNode existing_element, HtmlNode new_element)

void	PrependChild (const HtmlElement existing_parent, HtmlNode new_child)

void	AppendChild (const HtmlElement existing_parent, HtmlNode new_child)

void	InsertNodeBeforeCurrent (HtmlNode *new_node)

void	InsertNodeAfterCurrent (HtmlNode *new_node)

bool	AddParentToSequence (HtmlNode first, HtmlNode last, HtmlElement *new_parent)

bool	MoveCurrentInto (HtmlElement *new_parent)

bool	MoveCurrentBefore (HtmlNode *existing_node)

bool	DeleteNode (HtmlNode *node)

bool	DeleteSavingChildren (HtmlElement *element)

bool	MakeElementInvisible (HtmlElement *element)

bool	HasChildrenInFlushWindow (HtmlElement *element)

bool	ReplaceNode (HtmlNode existing_node, HtmlNode new_node)

HtmlElement *	CloneElement (HtmlElement *in_element)

HtmlElement *	NewElement (HtmlElement *parent, const StringPiece &str)

HtmlElement *	NewElement (HtmlElement *parent, HtmlName::Keyword keyword)

HtmlElement *	NewElement (HtmlElement *parent, const HtmlName &name)

void	AddAttribute (HtmlElement *element, HtmlName::Keyword keyword, const StringPiece &value)

void	AddAttribute (HtmlElement *element, StringPiece name, const StringPiece &value)

void	AddEscapedAttribute (HtmlElement *element, HtmlName::Keyword keyword, const StringPiece &escaped_value)

void	SetAttributeName (HtmlElement::Attribute *attribute, HtmlName::Keyword keyword)

HtmlName	MakeName (const StringPiece &str)

HtmlName	MakeName (HtmlName::Keyword keyword)

bool	IsRewritable (const HtmlNode *node) const

bool	CanAppendChild (const HtmlNode *node) const

void	ClearElements ()

void	DebugLogQueue ()
	Log the HtmlEvent queue_ to the message_handler_ for debugging.

void	DebugPrintQueue ()
	Print the HtmlEvent queue_ to stdout for debugging.

bool	IsImplicitlyClosedTag (HtmlName::Keyword keyword) const

bool	IsOptionallyClosedTag (HtmlName::Keyword keyword) const

bool	TagAllowsBriefTermination (HtmlName::Keyword keyword) const
	Determines whether a tag allows brief termination in HTML, e.g. <tag>

MessageHandler *	message_handler () const

const char *	url () const

const GoogleUrl &	google_url () const
	Gets a parsed GoogleUrl& corresponding to url().

const char *	id () const

int	line_number () const

GoogleString	UrlLine () const
	Returns URL (or id) and line number as a string, to be used in messages.

const DocType &	doctype () const

void	Info (const char filename, int line, const char msg,...) INSTAWEB_PRINTF_FORMAT(4
	Interface for any caller to report an error message via the message handler.

void void	Warning (const char filename, int line, const char msg,...) INSTAWEB_PRINTF_FORMAT(4

void void void	Error (const char filename, int line, const char msg,...) INSTAWEB_PRINTF_FORMAT(4

void void void void	FatalError (const char filename, int line, const char msg,...) INSTAWEB_PRINTF_FORMAT(4

void void void void void	InfoV (const char file, int line, const char msg, va_list args)

void	WarningV (const char file, int line, const char msg, va_list args)

void	ErrorV (const char file, int line, const char msg, va_list args)

void	FatalErrorV (const char file, int line, const char msg, va_list args)

void	InfoHere (const char *msg,...) INSTAWEB_PRINTF_FORMAT(2
	Report error message with current parsing filename and linenumber.

void void	WarningHere (const char *msg,...) INSTAWEB_PRINTF_FORMAT(2

void void void	ErrorHere (const char *msg,...) INSTAWEB_PRINTF_FORMAT(2

void void void void	FatalErrorHere (const char *msg,...) INSTAWEB_PRINTF_FORMAT(2

void void void void void	ShowProgress (const char *message)

void	InfoHereV (const char *msg, va_list args)

void	WarningHereV (const char *msg, va_list args)

void	ErrorHereV (const char *msg, va_list args)

void	FatalErrorHereV (const char *msg, va_list args)

void	AddElement (HtmlElement *element, int line_number)

void	CloseElement (HtmlElement *element, HtmlElement::Style style, int line_number)

void	ApplyFilter (HtmlFilter *filter)
	Run a filter on the current queue of parse nodes.

void	set_timer (Timer *timer)

Timer *	timer () const

void	set_log_rewrite_timing (bool x)

void	add_event_listener (HtmlFilter *listener)

bool	InsertComment (StringPiece sp)

void	set_size_limit (int64 x)
	Sets the limit on the maximum number of bytes that should be parsed.

bool	size_limit_exceeded () const
	Returns whether we have exceeded the size limit.

void	SetDynamicallyDisabledFilterList (StringVector *list)

void	DeferCurrentNode ()

void	RestoreDeferredNode (HtmlNode *deferred_node)

bool	can_modify_urls ()
	Returns whether the filter pipeline can rewrite urls.

Static Public Member Functions
static void	InitStats (Statistics *statistics)
	Initialize statistics for all filters that need it.

static void	Initialize ()
	Initialize statics. Initialize/Terminate calls must be paired.

static void	Terminate ()

static GoogleString	DeadlineExceededMessage (StringPiece filter_name)
	Formats a "deadline exceeded" message for a given filter.

static PropertyCache::CohortVector	GetCohortList (const PropertyCache pcache, const RewriteOptions options, const ServerContext *server_context)

Static Public Member Functions inherited from net_instaweb::HtmlParse
static bool	IsLiteralTag (HtmlName::Keyword keyword)

static bool	IsSometimesLiteralTag (HtmlName::Keyword keyword)

Static Public Attributes
static const char	kDomCohort []

static const char	kBeaconCohort []
	The cohort for properties that are written by the beacon handler.

static const char	kDependenciesCohort []

static const char	kLastRequestTimestamp []

static const char	kParseSizeLimitExceeded []
	Tracks if we exceeded the maximum size limit of html which we should parse.

static const char	kSubresourcesPropertyName []
	Flush Subresources Info associted with the HTML page.

static const char	kStatusCodePropertyName []
	Status codes of previous responses.

Protected Member Functions
virtual void	DetermineFiltersBehaviorImpl ()

Protected Member Functions inherited from net_instaweb::HtmlParse
void	BeginFinishParse ()

void	EndFinishParse ()

void	Clear ()

size_t	GetEventQueueSize ()
	Returns the number of events on the event queue.

void	DetermineFiltersBehavior ()
	Calls DetermineFiltersBehaviorImpl in an idempotent way.

void	DetermineFilterListBehavior (const FilterList &list)

void	CheckFilterBehavior (HtmlFilter *filter)

void	set_buffer_events (bool x)

void	DisableFiltersInjectingScripts ()

void	DisableFiltersInjectingScripts (const FilterList &filters)
	Same, but over a passed-in list of filters.

Friends
class	RewriteContext

class	RewriteDriverTest

class	RewriteTestBase

class	ServerContextTest

class	ScanFilter

class	CategorizedRefcount< RewriteDriver, RefCategory >

Additional Inherited Members
Protected Types inherited from net_instaweb::HtmlParse
typedef std::vector< HtmlFilter * >	FilterVector

typedef std::list< HtmlFilter * >	FilterList

typedef std::pair< HtmlNode , HtmlEventList >	DeferredNode

typedef std::map< const HtmlNode , HtmlEventList >	NodeToEventListMap

typedef std::map< HtmlFilter *, DeferredNode >	FilterElementMap

typedef std::set< const HtmlNode * >	NodeSet

Detailed Description

This extends class HtmlParse (which should renamed HtmlContext) by providing context for rewriting resources (css, js, images).

Member Enumeration Documentation

enum net_instaweb::RewriteDriver::InputRole

strong

How the input will be used in the page; relevant for checking against Content-Security-Policy.

Enumerator
kUnknown	Something where we don't know for sure; has to be handled extra-conservatively.
kReconstruction	Special role for resource reconstruction. This will be unchecked since the original resource path should be checked on the web page with appropriate policy.

enum net_instaweb::RewriteDriver::WaitMode

Mode for BoundedWaitForCompletion.

Enumerator
kNoWait	Used internally. Do not pass in.
kWaitForCompletion	wait for everything to complete (up to deadline)
kWaitForCachedRender	wait for at least cached rewrites to complete, and anything else that finishes within deadline.
kWaitForShutDown	Makes sure that all work, including any that's being done in background, finishes.

enum net_instaweb::RewriteDriver::XhtmlStatus

Indicates document's mimetype as XHTML, HTML, or is not known/something else. Note that in Apache we might not know the correct mimetype because a downstream module might change it. It's not clear how likely this is, since mod_rewrite and mod_mime run upstream of mod_pagespeed. However if anyone sets mimetype via "Header Add", it would affect the Browser's view of the document's mimetype (which is what determines the parsing) but mod_pagespeed would not know.

Note that we also have doctype().IsXhtml() but that indicates quirks-mode for CSS, and does not control how the parser parses the document.

Constructor & Destructor Documentation

virtual net_instaweb::RewriteDriver::~RewriteDriver ( )

virtual

Need explicit destructors to allow destruction of scoped_ptr-controlled instances without propagating the include files.

Member Function Documentation

void net_instaweb::RewriteDriver::AddFilters ( )

Adds the filters from the options, specified by name in enabled_filters. This must be called explicitly after object construction to provide an opportunity to programatically add custom filters beyond those defined in RewriteOptions, via AddFilter(HtmlFilter* filter) (below).

void net_instaweb::RewriteDriver::AddLowPriorityRewriteTask ( Function * task )

Queues up a task to run on the low-priority rewrite thread. Such tasks are expected to be safely cancelable.

void net_instaweb::RewriteDriver::AddOwnedEarlyPreRenderFilter ( HtmlFilter * filter )

Adds a filter to the very beginning of the pre-render chain, taking ownership. This should only be used for filters that must run before any filter added via PrependOwnedPreRenderFilter.

void net_instaweb::RewriteDriver::AddResourceUrlClaimant ( ResourceUrlClaimant * claimant )

Tells RewriteDriver that a certain portion of URL namespace should not be handled via usual (HTTP proxy semantics) means. It's up to the filters to actually arrange for that to do something. Takes ownership of the claimant object. Note that it's important for the claims to be disjoint, since the RewriteContext framework needs to be able to assign compatible Resource objects for same URLs/slots among all filters that deal with them.

void net_instaweb::RewriteDriver::AddUserReference ( )

Adds an extra external reference to the object. You should not normally need to call it (NewRewriteDriver does it initially), unless for some reason you want to pin the object (e.g. in tests). Matches up with Cleanup.

void net_instaweb::RewriteDriver::AppendRewriteFilter ( RewriteFilter * filter )

Add a RewriteFilter to the end of the pre-render chain and take ownership of the filter. This differs from AppendOwnedPreRenderFilter in that it adds the filter's ID into a dispatch table for serving rewritten resources. E.g. if your filter->id == "xy" and FetchResource("NAME.pagespeed.xy.HASH.EXT"...) is called, then RewriteDriver will dispatch to filter->Fetch().

This is used when the filter being added is not part of the core set built into RewriteDriver and RewriteOptions, such as platform-specific or server-specific filters, or filters invented for unit-testing the framework.

const GoogleUrl& net_instaweb::RewriteDriver::base_url ( ) const

inline

Returns the appropriate base gurl to be used for resolving hrefs in the document. Note that HtmlParse::google_url() is the URL for the HTML file and is used for printing html syntax errors.

void net_instaweb::RewriteDriver::BoundedWaitFor	(	WaitMode	mode,
		int64	timeout_ms
	)

As above, but with a time bound, and taking a mode parameter to decide between WaitForCompletion or WaitForShutDown behavior. If timeout_ms <= 0, no time bound will be used.

const GoogleString& net_instaweb::RewriteDriver::CacheFragment ( ) const

We fragment the cache based on the hostname we got from the request, unless that was overridden in the options with a cache_fragment.

bool net_instaweb::RewriteDriver::can_rewrite_resources ( ) const

inline

Determines whether the system is healthy enough to rewrite resources. Currently, systems get sick based on the health of the metadata cache.

void net_instaweb::RewriteDriver::Cleanup ( )

If there are not outstanding references to this RewriteDriver, delete it or recycle it to a free pool in the ServerContext. If this is a fetch, calling this also signals to the system that you are no longer interested in its results.

void net_instaweb::RewriteDriver::Clear ( )

Clears the current request cache of resources and base URL. The filter-chain is left intact so that a new request can be issued. Deletes all RewriteContexts.

WaitForCompletion must be called prior to Clear().

RewriteDriver* net_instaweb::RewriteDriver::Clone ( )

Returns a fresh instance using the same options we do, using the same log record. Drivers should only be cloned within the same request.

Clones share the same request_context, which contains bits derived from the request headers, so request_headers_ is also cloned (or shared if we make them shareable).

You must call SetRequestHeaders before calling Clone.

StringPiece net_instaweb::RewriteDriver::containing_charset ( )

inline

Get/set the charset of the containing HTML page. See scan_filter.cc for an explanation of how this is determined, but NOTE that the determined charset can change as more of the HTML is seen, in particular after a meta tag.

CacheUrlAsyncFetcher* net_instaweb::RewriteDriver::CreateCacheFetcher ( )

Creates a cache fetcher that uses the driver's fetcher and its options. Note: this means the driver's fetcher must survive as long as this does.

ResourcePtr net_instaweb::RewriteDriver::CreateInputResource	(	const GoogleUrl &	input_url,
		InputRole	role,
		bool *	is_authorized
	)

Creates an input resource based on input_url. Returns NULL if the input resource url isn't valid or is a data url, or can't legally be rewritten in the context of this page, in which case *is_authorized will be false. Assumes that resources from unauthorized domains may not be rewritten and that the resource is not intended exclusively for inlining.

ResourcePtr net_instaweb::RewriteDriver::CreateInputResource	(	const GoogleUrl &	input_url,
		InlineAuthorizationPolicy	inline_authorization_policy,
		IntendedFor	intended_for,
		InputRole	role,
		bool *	is_authorized
	)

Creates an input resource. Returns NULL if the input resource url isn't valid or is a data url, or can't legally be rewritten in the context of this page (which could mean that it was a resource from an unauthorized domain being processed by a filter that does not allow unauthorized resources, in which case *is_authorized will be false).

There are two "special" options, and if you don't care about them you should just call CreateInputResource(input_url, is_authorized) to use their defaults:

If resources from unauthorized domains may be inlined, set inline_authorization_policy to kInlineUnauthorizedResources, otherwise set it to kInlineOnlyAuthorizedResources.
If this resource will be inlined after fetching, then set intended_for to kIntendedForInlining, otherwise use kIntendedForGeneral. This is to support AllowWhenInlining.

ResourcePtr net_instaweb::RewriteDriver::CreateInputResourceAbsoluteUncheckedForTestsOnly ( const StringPiece & absolute_url )

Creates an input resource from the given absolute url. Requires that the provided url has been checked, and can legally be rewritten in the current page context. Only for use by unit tests.

OutputResourcePtr net_instaweb::RewriteDriver::CreateOutputResourceFromResource	(	const char *	filter_id,
		const UrlSegmentEncoder *	encoder,
		const ResourceContext *	data,
		const ResourcePtr &	input_resource,
		OutputResourceKind	kind,
		GoogleString *	failure_reason
	)

Creates a reference-counted pointer to a new OutputResource object.

The content type is taken from the input_resource, but can be modified with SetType later if that is not correct (e.g. due to image transcoding). Constructs an output resource corresponding to the specified input resource and encoded using the provided encoder. Assumes permissions checking occurred when the input resource was constructed, and does not do it again. To avoid if-chains, tolerates a NULL input_resource (by returning NULL).

Todo:: TODO(jmaessen, jmarantz): Do we want to permit NULL input_resources here? jmarantz has evinced a distaste.

OutputResourcePtr net_instaweb::RewriteDriver::CreateOutputResourceWithMappedPath	(	const StringPiece &	mapped_path,
		const StringPiece &	unmapped_path,
		const StringPiece &	filter_id,
		const StringPiece &	name,
		OutputResourceKind	kind,
		GoogleString *	failure_reason
	)

inline

Version of CreateOutputResourceWithPath where the unmapped and mapped paths are different and the base_url is this driver's base_url.

OutputResourcePtr net_instaweb::RewriteDriver::CreateOutputResourceWithPath	(	const StringPiece &	mapped_path,
		const StringPiece &	unmapped_path,
		const StringPiece &	base_url,
		const StringPiece &	filter_id,
		const StringPiece &	name,
		OutputResourceKind	kind,
		GoogleString *	failure_reason
	)

Creates an output resource where the name is provided. The intent is to be able to derive the content from the name, for example, by encoding URLs and metadata.

This method succeeds unless the filename is too long.

This name is prepended with path for writing hrefs, and the resulting url is encoded and stored at file_prefix when working with the file system. So hrefs are: /usr/lib/gcc-mozilla/bin:/usr/local/bin:/usr/bin:/home/oschaaf/.local/share/umake/bin:/home/oschaaf/bin:/home/oschaaf/code/google/cloud-sdk/google-cloud-sdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/home/oschaaf/bin/depot_tools:/home/oschaaf/bin/depot_tools/.pagespeed[.$EXPERIMENT].. .

EXPERIMENT is set only when there is an active experiment_spec.

Could be private since you should use one of the versions below but put here with the rest like it and for documentation clarity.

OutputResourcePtr net_instaweb::RewriteDriver::CreateOutputResourceWithPath	(	const StringPiece &	path,
		const StringPiece &	filter_id,
		const StringPiece &	name,
		OutputResourceKind	kind,
		GoogleString *	failure_reason
	)

inline

Version of CreateOutputResourceWithPath where the unmapped and mapped paths and the base url are all the same. FOR TESTS ONLY.

OutputResourcePtr net_instaweb::RewriteDriver::CreateOutputResourceWithUnmappedUrl	(	const GoogleUrl &	unmapped_gurl,
		const StringPiece &	filter_id,
		const StringPiece &	name,
		OutputResourceKind	kind,
		GoogleString *	failure_reason
	)

Version of CreateOutputResourceWithPath which first takes only the unmapped path and finds the mapped path using the DomainLawyer and the base_url is this driver's base_url.

CriticalSelectorInfo* net_instaweb::RewriteDriver::critical_selector_info ( )

inline

This should only be called by the CriticalSelectorFinder. Normal users should call CriticalSelectorFinder::IsCriticalImage.

Todo:: TODO(jud): Remove when the finders reside in RewriteDriver and manage their own state.

bool net_instaweb::RewriteDriver::DebugMode ( ) const

inline

Determines whether we are currently in Debug mode; meaning that the site owner or user has enabled filter kDebug.

bool net_instaweb::RewriteDriver::Decode	(	StringPiece	leaf,
		ResourceNamer *	resource_namer
	)		const

Calls the provided ResourceNamer's Decode() function, passing the hash and signature lengths from this RewriteDriver.

const GoogleUrl& net_instaweb::RewriteDriver::decoded_base_url ( ) const

inline

Returns the decoded version of base_gurl() in case it was encoded by a non-default UrlNamer (for the default UrlNamer this returns the same value as base_url()). Required when fetching a resource by its encoded name.

OutputResourcePtr net_instaweb::RewriteDriver::DecodeOutputResource	(	const GoogleUrl &	url,
		RewriteFilter **	filter
	)		const

Attempts to decode an output resource based on the URL pattern without actually rewriting it. No permission checks are performed on the url, though it is parsed to see if it looks like the url of a generated resource (which should mean checking the hash to ensure we generated it ourselves).

Todo:: TODO(jmaessen): add url hash & check thereof.

bool net_instaweb::RewriteDriver::DecodeOutputResourceName	(	const GoogleUrl &	url,
		const RewriteOptions *	options_to_use,
		const UrlNamer *	url_namer,
		ResourceNamer *	name_out,
		OutputResourceKind *	kind_out,
		RewriteFilter **	filter_out
	)		const

As above, but does not actually create a resource object, and instead outputs the decoded information into the various out parameters. Returns whether decoding successful or not. Uses options_to_use rather than this->options() to determine which drivers are forbidden from applying, etc.

bool net_instaweb::RewriteDriver::DecodeUrlGivenOptions	(	const GoogleUrl &	url,
		const RewriteOptions *	options,
		const UrlNamer *	url_namer,
		StringVector *	decoded_urls
	)		const

As above, but lets one specify the options and URL namer to use. Meant for use with the decoding_driver.

void net_instaweb::RewriteDriver::DecrementRenderBlockingAsyncEventsCount ( )

Decrements a reference count bumped up by IncrementRenderBlockingAsyncEventsCount()

void net_instaweb::RewriteDriver::DeleteRewriteContext ( RewriteContext * rewrite_context )

Deletes the specified RewriteContext. If this is the last RewriteContext active on this Driver, and there is no other outstanding activity, then the RewriteDriver itself can be recycled, and WaitForCompletion can return.

We expect to this method to be called on the Rewrite thread.

DependencyTracker* net_instaweb::RewriteDriver::dependency_tracker ( ) const

inline

dependency_tracker()->RegisterDependencyCandidate and ReportDependencyCandidate can be called from any thread.

void net_instaweb::RewriteDriver::DeregisterForPartitionKey	(	const GoogleString &	partition_key,
		RewriteContext *	candidate
	)

Must be called after all other rewrites that are currently relying on this one have had their RepeatedSuccess or RepeatedFailure methods called.

Must only be called from rewrite thread.

void net_instaweb::RewriteDriver::DetachedFetchComplete ( )

Called by RewriteContext when a detached async fetch is complete, allowing the RewriteDriver to be recycled if FetchComplete() got invoked as well.

void net_instaweb::RewriteDriver::DetachFetch ( )

Called by RewriteContext to let RewriteDriver know it will be continuing on the fetch in background, and so it should defer doing full cleanup sequences until DetachedFetchComplete() is called.

virtual void net_instaweb::RewriteDriver::DetermineFiltersBehaviorImpl ( )

protectedvirtual

Call DetermineEnabled() on each filter. Should be called after the property cache lookup has finished since some filters depend on pcache results in their DetermineEnabled implementation. If a subclass has filters that the base HtmlParse doesn't know about, it should override this function and call DetermineEnabled on each of its filters, along with calling the base DetermineEnabledFiltersImpl. For all enabled filters the CanModifyUrl() flag will be aggregated (or'ed) and can be queried on the can_modify_url function.

Reimplemented from net_instaweb::HtmlParse.

DomainRewriteFilter* net_instaweb::RewriteDriver::domain_rewriter ( )

inline

Used by CacheExtender, CssCombineFilter, etc. for rewriting domains of sub-resources in CSS.

void net_instaweb::RewriteDriver::EnableBlockingRewrite ( RequestHeaders * request_headers )

If the value of X-PSA-Blocking-Rewrite request header matches the blocking rewrite key, set fully_rewrite_on_flush flag.

void net_instaweb::RewriteDriver::ExecuteFlushIfRequested ( )

Executes an Flush() if RequestFlush() was called, e.g. from the Listener Filter (see set_event_listener below). Consider an HTML parse driven by a UrlAsyncFetcher. When the UrlAsyncFetcher temporarily runs out of bytes to read, it calls response_writer->Flush(). When that happens, we may want to consider flushing the outstanding HTML events through the system so that the browser can start fetching subresources and rendering. The event_listener (see set_event_listener below) helps determine whether enough "interesting" events have passed in the current flush window so that we should take this incoming network pause as an opportunity.

void net_instaweb::RewriteDriver::ExecuteFlushIfRequestedAsync ( Function * callback )

Asynchronous version of the above. Note that you should not attempt to write out any data until the callback is invoked. (If a flush is not needed, the callback will be invoked immediately).

FallbackPropertyPage* net_instaweb::RewriteDriver::fallback_property_page ( ) const

inline

Returns the property page which contains the cached properties associated with the current URL and fallback URL (i.e. without query params). This should be used where a property is interested in fallback values if actual values are not present.

void net_instaweb::RewriteDriver::FetchComplete ( )

Cleans up the driver and any fetch rewrite contexts, unless the fetch rewrite got detached by a call to DetachFetch(), in which case a call to DetachedFetchComplete() must also be performed.

void net_instaweb::RewriteDriver::FetchInPlaceResource	(	const GoogleUrl &	gurl,
		bool	proxy_mode,
		AsyncFetch *	async_fetch
	)

Initiates an In-Place Resource Optimization (IPRO) fetch (A resource which is served under the original URL, but is still able to be rewritten).

proxy_mode indicates whether we are running as a proxy where users depend on us to send contents. When set true, we will perform HTTP fetches to get contents if not in cache and will ignore kRecentFetchNotCacheable and kRecentFetchFailed since we'll have to fetch the resource for users anyway. Origin implementations (like mod_pagespeed) should set this to false and let the serve serve the resource if it's not in cache.

If proxy_mode is false and the resource could not be found in HTTP cache, async_fetch->Done(false) will be called and async_fetch->status_code() will be CacheUrlAsyncFetcher::kNotInCacheStatus (to distinguish this from a different reason for failure, like kRecentFetchNotCacheable).

Note that if the request headers have not yet been set on the driver then they'll be taken from the fetch.

bool net_instaweb::RewriteDriver::FetchOutputResource	(	const OutputResourcePtr &	output_resource,
		RewriteFilter *	filter,
		AsyncFetch *	async_fetch
	)

See FetchResource. There are two differences:

It takes an OutputResource instead of a URL.
It returns whether a fetch was queued or not. This is safe to ignore because in either case the callback will be called.
If 'filter' is NULL then the request only checks cache and (if enabled) the file system.

bool net_instaweb::RewriteDriver::FetchResource	(	const StringPiece &	url,
		AsyncFetch *	fetch
	)

Initiates an async fetch for a rewritten resource with the specified name. If url matches the pattern of what the driver is authorized to serve, then true is returned and the caller must listen on the callback for the completion of the request.

If the driver is not authorized to serve the resource for any of the following reasons, false is returned and the callback will -not- be called - the request should be passed to another handler.

The URL is invalid or it does not match the general pagespeed pattern.
The filter id in the URL does not map to a known filter.
The filter for the id in the URL doesn't recognize the format of the URL.
The filter for the id in the URL is forbidden.

In other words there are three outcomes for this routine:

the request was handled immediately and the callback called before the method returns. true is returned.
the request looks good but was queued because some other resource fetch is needed to satisfy it. true is returned.
the request does not look like it belongs to Instaweb. The callback will not be called, and false will be returned.

In even other words, if this routine returns 'false' then the callback will not be called. If the callback -is- called, then this should be the 'final word' on this request, whether it was called with success=true or success=false.

Note that if the request headers have not yet been set on the driver then they'll be taken from the fetch.

virtual void net_instaweb::RewriteDriver::FinishParse ( )

virtual

Override HtmlParse's FinishParse to ensure that the request-scoped cache is cleared immediately.

Note that the RewriteDriver can delete itself in this method, if it's not externally managed, and if all RewriteContexts have been completed.

Reimplemented from net_instaweb::HtmlParse.

void net_instaweb::RewriteDriver::FinishParseAsync ( Function * callback )

As above, but asynchronous. Note that the RewriteDriver may already be deleted at the point the callback is invoked. The scheduler lock will not be held when the callback is run.

bool net_instaweb::RewriteDriver::FlattenCssImportsEnabled ( ) const

inline

Return true if we must flatten css imports, either because the filter is enabled explicitly or because it is enabled by PrioritizeCriticalCss.

virtual void net_instaweb::RewriteDriver::Flush ( )

virtual

Overrides HtmlParse::Flush so that it can happen in two phases:

Pre-render chain runs, resulting in async rewrite activity
async rewrite activity ends, calling callback, and post-render filters run. This API is used for unit-tests & Apache (which lacks a useful event model) and results in blocking behavior.

FlushAsync is prefered for event-driven servers.

Reimplemented from net_instaweb::HtmlParse.

void net_instaweb::RewriteDriver::FlushAsync ( Function * done )

Initiates an asynchronous Flush. done->Run() will be called when the flush is complete. Further calls to ParseText should be deferred until the callback is called. Scheduler mutex is not held while done is called.

static PropertyCache::CohortVector net_instaweb::RewriteDriver::GetCohortList	(	const PropertyCache *	pcache,
		const RewriteOptions *	options,
		const ServerContext *	server_context
	)

static

Returns the list of cohorts that should be read in based on our options.

InlineAttributeSlotPtr net_instaweb::RewriteDriver::GetInlineAttributeSlot	(	const ResourcePtr &	resource,
		HtmlElement *	element,
		HtmlElement::Attribute *	attribute
	)

Creates and registers an inline attribute resource slot for rewriting. If this is the first time called for this position, a new slot will be returned. On subsequent calls, the original slot will be returned so that rewrites are propagated between filters.

InlineResourceSlotPtr net_instaweb::RewriteDriver::GetInlineSlot	(	const ResourcePtr &	resource,
		HtmlCharactersNode *	char_node
	)

Creates and registers an inline resource slot for rewriting. If this is the first time called for this position, a new slot will be returned. On subsequent calls, the original slot will be returned so that rewrites are propagated between filters.

HtmlResourceSlotPtr net_instaweb::RewriteDriver::GetSlot	(	const ResourcePtr &	resource,
		HtmlElement *	elt,
		HtmlElement::Attribute *	attr
	)

Creates and registers a HtmlElement slot for rewriting. If this is the first time called for this position, a new slot will be returned. On subsequent calls, the original slot will be returned so that rewrites are propagated between filters.

SrcSetSlotCollectionPtr net_instaweb::RewriteDriver::GetSrcSetSlotCollection	(	CommonFilter *	filter,
		HtmlElement *	element,
		HtmlElement::Attribute *	attr
	)

Create and and registers a source set slot collection for rewriting all the images in the srcset attribute of an . Also creates the neccessary resources using the provided filter's policy.

If this is the first time called for this element + attr, a new collection will be returned. On subsequent calls, the original collection will be returned so that rewrites are propagated between filters. All filters using this call are expected to have the same values for AllowUnauthorizedDomain() and IntendedForInlining().

void net_instaweb::RewriteDriver::IncrementAsyncEventsCount ( )

Increment reference count for misc. async ops that need the RewriteDriver kept alive.

void net_instaweb::RewriteDriver::IncrementRenderBlockingAsyncEventsCount ( )

Increment reference count for misc async ops that should be waited for before doing rendering for current flush window.

void net_instaweb::RewriteDriver::InfoAt	(	const RewriteContext *	context,
		const char *	msg,
			...
	)

Report error message with description of context's location (such as filenames and line numbers). context may be NULL, in which case the current parse position will be used.

bool net_instaweb::RewriteDriver::InitiateRewrite ( RewriteContext * rewrite_context )

Method to start a resource rewrite. This is called by a filter during parsing, although the Rewrite might continue after deadlines expire and the rewritten HTML must be flushed. Returns InitiateRewrite returns false if the system is not healthy enough to support resource rewrites.

void net_instaweb::RewriteDriver::InsertDebugComment	(	StringPiece	unescaped_message,
		HtmlNode *	node
	)

Log the given debug message(s) as HTML comments after the given element, if not NULL, it has not been flushed, and if debug is enabled. The form that takes a repeated field is intended for use by CachedResult, e.g: InsertDebugComment(cached_result.debug_message(), element); Messages are HTML-escaped before being written out to the DOM.

bool net_instaweb::RewriteDriver::is_critical_images_beacon_enabled ( )

The JS to detect above-the-fold images should only be enabled if one of the filters that uses critical image information is enabled, the property cache is enabled (since the critical image information is stored in the property cache), and it is not explicitly disabled through options.

bool net_instaweb::RewriteDriver::IsResourceUrlClaimed ( const GoogleUrl & url ) const

Returns true if some ResourceUrlClaimant has staked a claim on given URL. If this returns true, CreateInputResource will fail, but it's probably not worth logging any debug filter hints about that.

AbstractLogRecord* net_instaweb::RewriteDriver::log_record ( )

log_record() always returns a pointer to a valid AbstractLogRecord, owned by the rewrite_driver's request context.

bool net_instaweb::RewriteDriver::LookupMetadataForOutputResource	(	StringPiece	url,
		GoogleString *	error_out,
		RewriteContext::CacheLookupResultCallback *	callback
	)

Attempts to lookup the metadata cache info that would be used for the output resource at url with the RewriteOptions set on this driver.

If there is a problem with the URL, returns false, and *error_out will contain an error message.

If it can determine the metadata cache key successfully, returns true, and eventually callback will be invoked with the metadata cache key and the decoding results.

After calling this method, the driver should not be used for anything else.

bool net_instaweb::RewriteDriver::MatchesBaseUrl ( const GoogleUrl & input_url ) const

Checks to see if the input_url has the same origin as and the base url, to make sure we're not fetching from another server. Does not consult the domain lawyer, and is not affected by AddDomain(). Precondition: input_url.IsWebValid()

bool net_instaweb::RewriteDriver::MayCacheExtendCss ( ) const

Returns true if we may cache extend Css, Images, PDFs, or Scripts respectively.

bool net_instaweb::RewriteDriver::MayRewriteUrl	(	const GoogleUrl &	domain_url,
		const GoogleUrl &	input_url,
		InlineAuthorizationPolicy	inline_authorization_policy,
		IntendedFor	intended_for,
		bool *	is_authorized_domain
	)		const

Checks to see if we can write the input_url resource in the domain_url taking into account domain authorization, wildcard allow/disallow from RewriteOptions, and the intended use of the url's resource. After the function is executed, is_authorized_domain will indicate whether input_url was found to belong to an authorized domain or not.

XhtmlStatus net_instaweb::RewriteDriver::MimeTypeXhtmlStatus ( )

Determines whether the document's Content-Type has a mimetype indicating that browsers should parse it as XHTML.

ResponseHeaders* net_instaweb::RewriteDriver::mutable_response_headers ( )

inline

Return a mutable pointer to the response headers that filters can update before the first flush. Returns NULL after Flush has occurred.

int net_instaweb::RewriteDriver::num_inline_preview_images ( ) const

inline

We expect to this method to be called on the HTML parser thread. Returns the number of images whose low quality images are inlined in the html page.

PropertyPage* net_instaweb::RewriteDriver::origin_property_page ( ) const

Returns property page which contains cached properties associated with the current origin (host/port/protocol). May be NULL.

void net_instaweb::RewriteDriver::PopulateResourceNamer	(	const StringPiece &	filter_id,
		const StringPiece &	name,
		ResourceNamer *	full_name
	)

Fills in the resource namer based on the give filter_id, name and options stored in the driver.

void net_instaweb::RewriteDriver::PrependRewriteFilter ( RewriteFilter * filter )

Like AppendRewriteFilter, but adds the filter to the beginning of the pre-render chain.

PropertyPage* net_instaweb::RewriteDriver::property_page ( ) const

Returns the property page which contains the cached properties associated with the current URL.

void net_instaweb::RewriteDriver::PropertyCacheSetupDone ( )

Should be called once everything in the property cache has been read, and the pages set on the object.

RewriteContext* net_instaweb::RewriteDriver::RegisterForPartitionKey	(	const GoogleString &	partition_key,
		RewriteContext *	candidate
	)

Tries to register the given rewrite context as working on its partition key. If this context is the first one to try to handle it, returns NULL. Otherwise returns the previous such context.

Must only be called from rewrite thread.

void net_instaweb::RewriteDriver::ReportSlowRewrites ( int num )

Provides a mechanism for a RewriteContext to notify a RewriteDriver that a certain number of rewrites have been discovered to need to take the slow path.

void net_instaweb::RewriteDriver::RequestFlush ( )

inline

Indicates that a Flush through the HTML parser chain should happen soon, e.g. once the network pauses its incoming byte stream.

CssResolutionStatus net_instaweb::RewriteDriver::ResolveCssUrls	(	const GoogleUrl &	input_css_base,
		const StringPiece &	output_css_base,
		const StringPiece &	contents,
		Writer *	writer,
		MessageHandler *	handler
	)

Rewrites CSS content to absolutify any relative embedded URLs, streaming the results to the writer. Returns 'false' if the writer returns false or if the content was not rewritten because the domains of the gurl and resolved_base match.

input_css_base contains the path where the CSS text came from. output_css_base contains the path where the CSS will be written.

const ResponseHeaders* net_instaweb::RewriteDriver::response_headers ( )

inline

Returns a const version of the ResponseHeaders*, indepdendent of whether Flush has occurred. Note that ResponseHeaders* may still be NULL if no one has called set_response_headers_ptr.

Todo:: TODO(jmarantz): Change API to require response_headers in StartParse so we can guarantee this is non-null.

void net_instaweb::RewriteDriver::RewriteComplete	(	RewriteContext *	rewrite_context,
		RenderOp	permit_render
	)

Provides a mechanism for a RewriteContext to notify a RewriteDriver that it is complete, to allow the RewriteDriver to delete itself or return it back to a free pool in the ServerContext.

This will also call back into RewriteContext::Propagate, letting it know whether the context is still attached to the HTML DOM (and hence safe to render), and to do other bookkeeping.

If 'permit_render' is false, no rendering will be asked for even if the context is still attached.

void net_instaweb::RewriteDriver::RunTasksOnRequestThread ( )

Make the rewrite_worker tasks run on the request thread. This must be called immediately after initializing the driver, before it starts processing the request.

void net_instaweb::RewriteDriver::set_critical_images_info ( CriticalImagesInfo * critical_images_info )

inline

Inserts the critical images present on the requested html page. It takes ownership of critical_images_info. This should only be called by the CriticalImagesFinder, normal users should just be using the automatic management of critical_images_info that CriticalImagesFinder provides.

void net_instaweb::RewriteDriver::set_critical_selector_info ( CriticalSelectorInfo * info )

inline

This should only be called by the CriticalSelectorFinder.

Todo:: TODO(jud): Remove when the finders reside in RewriteDriver and manage their own state.

void net_instaweb::RewriteDriver::set_externally_managed ( bool x )

inline

Indicate that this RewriteDriver will be explicitly deleted, and thus should not be auto-deleted at the end of the parse. This is primarily for tests.

Todo:: TODO(jmarantz): Consider phasing this out to make tests behave more like servers.

void net_instaweb::RewriteDriver::set_fast_blocking_rewrite ( bool x )

inline

This is relevant only when fully_rewrite_on_flush is true. When this is set to true, Flush of HTML will not wait for async events while it does wait when it is set to false.

void net_instaweb::RewriteDriver::set_fully_rewrite_on_flush ( bool x )

inline

If this is set to true, during a Flush of HTML the system will wait for results of all rewrites rather than just waiting for cache lookups and a small deadline. Note, however, that in very rare circumstances some rewrites may still be dropped due to excessive load.

Note: reset every time the driver is recycled.

void net_instaweb::RewriteDriver::set_max_page_processing_delay_ms ( int x )

inline

Sets a maximum amount of time to process a page across all flush windows; i.e., the entire lifecycle of this driver during a given pageload. A negative value indicates no limit. Setting fully_rewrite_on_flush() overrides this.

void net_instaweb::RewriteDriver::set_num_initiated_rewrites ( int64 x )

inline

Sets the num_initiated_rewrites_. This should only be called from test code.

void net_instaweb::RewriteDriver::set_options_for_pool	(	RewriteDriverPool *	pool,
		RewriteOptions *	options
	)

inline

Takes ownership of 'options'. pool denotes the pool of rewrite drivers that use these options. May be NULL if using custom options.

void net_instaweb::RewriteDriver::set_refs_before_base ( )

inline

Sets whether or not there were references to urls before the base tag (if there is a base tag). This variable has document-level scope. It is reset at the beginning of every document by ScanFilter.

void net_instaweb::RewriteDriver::set_response_headers_ptr ( ResponseHeaders * headers )

inline

Set the pointer to the response headers that filters can update before the first flush. RewriteDriver does NOT take ownership of this memory.

void net_instaweb::RewriteDriver::SetIsAmpDocument ( bool is_amp )

Declares whether the current document is AMP or not. Prior to calling this, all HTML events are buffered, to avoid waking up filters that inject scripts.

bool net_instaweb::RewriteDriver::SetOrClearPageSpeedOptionCookies	(	const GoogleUrl &	gurl,
		ResponseHeaders *	response_headers
	)

Utility function to set/clear cookies for PageSpeed options. gurl is the URL of the request from which the host is extracted for a cookie attribute.

Todo:: TODO(matterbury): Get the URL from 'this' which we can't do now because it isn't set until we've decided that the content of requested URL is HTML. Returns true if any Set-Cookie headers are added, in which case ComputeCaching has been called on response_headers.

void net_instaweb::RewriteDriver::SetRequestHeaders ( const RequestHeaders & headers )

Reinitializes request_headers_ (a scoped ptr) with a copy of the original request headers. Note that the fetches associated with the driver could be using a modified version of the original request headers. There MUST be exactly 1 call to this method after a rewrite driver object has been constructed or recycled, before the RewriteDriver is used for request processing.

This method also sets up the user-agent and device properties.

void net_instaweb::RewriteDriver::SetServerContext ( ServerContext * server_context )

Sets a server context enabling the rewriting of resources. This will replace any previous server context.

void net_instaweb::RewriteDriver::SetSessionFetcher ( UrlAsyncFetcher * f )

Set a fetcher that will be used by RewriteDriver for current request only (that is, until Clear()). RewriteDriver will take ownership of this fetcher, and will keep it around until Clear(), even if further calls to this method are made.

void net_instaweb::RewriteDriver::SetWriter ( Writer * writer )

Controls how HTML output is written. Be sure to call this last, after all other filters have been established.

Todo:: TODO(jmarantz): fix this in the implementation so that the caller can install filters in any order and the writer will always be last.

bool net_instaweb::RewriteDriver::ShouldAbsolutifyUrl	(	const GoogleUrl &	input_base,
		const GoogleUrl &	output_base,
		bool *	proxy_mode
	)		const

Determines if an URL relative to the given input_base needs to be absolutified given that it will end up under output_base:

If we are proxying and input_base isn't proxy encoded, then yes.
If we aren't proxying and input_base != output_base, then yes.
If we aren't proxying and the domain lawyer will shard or rewrite input_base, then yes. If not NULL also set *proxy_mode to whether proxy mode is active or not.

virtual bool net_instaweb::RewriteDriver::StartParseId	(	const StringPiece &	url,
		const StringPiece &	id,
		const ContentType &	content_type
	)

virtual

Override HtmlParse's StartParseId to propagate any required options. Note that if this (or other variants) returns true you should use FinishParse(), otherwise Cleanup().

Reimplemented from net_instaweb::HtmlParse.

void net_instaweb::RewriteDriver::SwitchToQueuedWorkerPool ( )

Switches the driver back to running rewrite_worker tasks using the QueuedWorkerPool. This can be called when we are retiring a server-request on behalf of the client (e.g. after a deadline was exceeded), but want background optimization to continue. It can no longer continue on the request thread.

RequestTrace* net_instaweb::RewriteDriver::trace_context ( )

Convenience method to return the trace context from the request_context() if both are configured and NULL otherwise.

void net_instaweb::RewriteDriver::TracePrintf	(	const char *	fmt,
			...
	)

Convenience methods to issue a trace annotation if tracing is enabled. If tracing is disabled, these methods are no-ops.

void net_instaweb::RewriteDriver::UpdatePropertyValueInDomCohort	(	AbstractPropertyPage *	page,
		StringPiece	property_name,
		StringPiece	property_value
	)

Update the PropertyValue named 'property_name' in dom cohort with the value 'property_value'. It is the responsibility of the client to ensure that property cache and dom cohort are enabled when this function is called. It is a programming error to call this function when property cache or dom cohort is not available, more so since the value payload has to be serialised before calling this function. Hence this function will DFATAL if property cache or dom cohort is not available.

void net_instaweb::RewriteDriver::WaitForCompletion ( )

Wait for outstanding Rewrite to complete. Once the rewrites are complete they can be rendered.

void net_instaweb::RewriteDriver::WaitForShutDown ( )

Wait for outstanding rewrite to complete, including any background work that may be ongoing even after results were reported.

Note: while this guarantees that the result of the computation is known, the thread that performed it may still be running for a little bit and accessing the driver.

bool net_instaweb::RewriteDriver::Write	(	const ResourceVector &	inputs,
		const StringPiece &	contents,
		const ContentType *	type,
		StringPiece	charset,
		OutputResource *	output
	)

Writes the specified contents into the output resource, and marks it as optimized. 'inputs' described the input resources that were used to construct the output, and is used to determine whether the result can be safely cache extended and be marked publicly cacheable. 'content_type' and 'charset' specify the mimetype and encoding of the contents, and will help form the Content-Type header. 'charset' may be empty when not specified.

Note that this does not escape charset.

Callers should take care that dangerous types like 'text/html' do not sneak into content_type.

Member Data Documentation

const char net_instaweb::RewriteDriver::kDependenciesCohort[]

static

Cohort for dependency information. This is written at different time than kDomCohort, and might not be in use for some requests, depending on settings.

const char net_instaweb::RewriteDriver::kDomCohort[]

static

This string identifies, for the PropertyCache, a group of properties that are computed from the DOM, and thus can, if desired, be rewritten on every HTML request.

const char net_instaweb::RewriteDriver::kLastRequestTimestamp[]

static

Property Names in DomCohort. Tracks the timestamp when we last received a request for this url.

The documentation for this class was generated from the following file:

net/instaweb/rewriter/public/rewrite_driver.h

Public Types

Public Member Functions

Static Public Member Functions

Static Public Attributes

Protected Member Functions

Friends

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

Constructor & Destructor Documentation

Member Function Documentation

Member Data Documentation