html_parse.h

Go to the documentation of this file.

/*
 * Copyright 2010 Google Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http:///www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


#ifndef PAGESPEED_KERNEL_HTML_HTML_PARSE_H_
#define PAGESPEED_KERNEL_HTML_HTML_PARSE_H_

#include <cstdarg>
#include <cstddef>
#include <list>
#include <map>
#include <set>
#include <utility>
#include <vector>

#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/arena.h"
#include "pagespeed/kernel/base/printf_format.h"
#include "pagespeed/kernel/base/scoped_ptr.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/base/symbol_table.h"
#include "pagespeed/kernel/html/html_element.h"
#include "pagespeed/kernel/html/html_name.h"
#include "pagespeed/kernel/html/html_node.h"
#include "pagespeed/kernel/http/content_type.h"
#include "pagespeed/kernel/http/google_url.h"

namespace net_instaweb {

class DocType;
class HtmlEvent;
class HtmlFilter;
class HtmlLexer;
class MessageHandler;
class Timer;

typedef std::set <const HtmlEvent*> ConstHtmlEventSet;

class HtmlParse {
 public:
  explicit HtmlParse(MessageHandler* message_handler);
  virtual ~HtmlParse();


  void AddFilter(HtmlFilter* filter);

  bool StartParse(const StringPiece& url) {
    return StartParseWithType(url, kContentTypeHtml);
  }
  bool StartParseWithType(const StringPiece& url,
                          const ContentType& content_type) {
    return StartParseId(url, url, content_type);
  }

  bool is_url_valid() const { return url_valid_; }

  virtual bool StartParseId(const StringPiece& url, const StringPiece& id,
                            const ContentType& content_type);

  void SetUrlForTesting(const StringPiece& url);

  void ParseText(const char* content, int size) {
    ParseTextInternal(content, size);
  }
  void ParseText(const StringPiece& sp) {
    ParseTextInternal(sp.data(), sp.size());
  }

  virtual void Flush();

  virtual void FinishParse();



  HtmlCdataNode* NewCdataNode(HtmlElement* parent,
                              const StringPiece& contents);
  HtmlCharactersNode* NewCharactersNode(HtmlElement* parent,
                                        const StringPiece& literal);
  HtmlCommentNode* NewCommentNode(HtmlElement* parent,
                                  const StringPiece& contents);
  HtmlDirectiveNode* NewDirectiveNode(HtmlElement* parent,
                                      const StringPiece& contents);
  HtmlIEDirectiveNode* NewIEDirectiveNode(HtmlElement* parent,
                                          const StringPiece& contents);
  void InsertScriptAfterCurrent(StringPiece text, bool external);
  void InsertScriptBeforeCurrent(StringPiece text, bool external);

  HtmlElement* AppendAnchor(StringPiece link, StringPiece text,
                            HtmlElement* parent);



  void InsertNodeBeforeNode(const HtmlNode* existing_node, HtmlNode* new_node);
  void InsertNodeAfterNode(const HtmlNode* existing_node, HtmlNode* new_node);

  void InsertElementBeforeElement(const HtmlNode* existing_element,
                                  HtmlNode* new_element) {
    InsertNodeBeforeNode(existing_element, new_element);
  }

  void InsertElementAfterElement(const HtmlNode* existing_element,
                                 HtmlNode* new_element) {
    InsertNodeAfterNode(existing_element, new_element);
  }

  void PrependChild(const HtmlElement* existing_parent, HtmlNode* new_child);
  void AppendChild(const HtmlElement* existing_parent, HtmlNode* new_child);

  void InsertNodeBeforeCurrent(HtmlNode* new_node);

  void InsertNodeAfterCurrent(HtmlNode* new_node);

  bool AddParentToSequence(HtmlNode* first, HtmlNode* last,
                           HtmlElement* new_parent);

  bool MoveCurrentInto(HtmlElement* new_parent);

  bool MoveCurrentBefore(HtmlNode* existing_node);

  bool DeleteNode(HtmlNode* node);

  bool DeleteSavingChildren(HtmlElement* element);

  bool MakeElementInvisible(HtmlElement* element);

  bool HasChildrenInFlushWindow(HtmlElement* element);

  bool ReplaceNode(HtmlNode* existing_node, HtmlNode* new_node);

  HtmlElement* CloneElement(HtmlElement* in_element);

  HtmlElement* NewElement(HtmlElement* parent, const StringPiece& str) {
    return NewElement(parent, MakeName(str));
  }
  HtmlElement* NewElement(HtmlElement* parent, HtmlName::Keyword keyword) {
    return NewElement(parent, MakeName(keyword));
  }
  HtmlElement* NewElement(HtmlElement* parent, const HtmlName& name);

  void AddAttribute(HtmlElement* element, HtmlName::Keyword keyword,
                    const StringPiece& value) {
    return element->AddAttribute(MakeName(keyword), value,
                                 HtmlElement::DOUBLE_QUOTE);
  }
  void AddAttribute(HtmlElement* element, StringPiece name,
                    const StringPiece& value) {
    return element->AddAttribute(MakeName(name), value,
                                 HtmlElement::DOUBLE_QUOTE);
  }
  void AddEscapedAttribute(HtmlElement* element, HtmlName::Keyword keyword,
                    const StringPiece& escaped_value) {
    return element->AddEscapedAttribute(MakeName(keyword), escaped_value,
                                        HtmlElement::DOUBLE_QUOTE);
  }
  void SetAttributeName(HtmlElement::Attribute* attribute,
                        HtmlName::Keyword keyword) {
    attribute->set_name(MakeName(keyword));
  }

  HtmlName MakeName(const StringPiece& str);
  HtmlName MakeName(HtmlName::Keyword keyword);

  bool IsRewritable(const HtmlNode* node) const;
  bool CanAppendChild(const HtmlNode* node) const;

  void ClearElements();

  void DebugLogQueue();

  void DebugPrintQueue();

  friend class HtmlLexer;

  bool IsImplicitlyClosedTag(HtmlName::Keyword keyword) const;

  static bool IsLiteralTag(HtmlName::Keyword keyword);

  static bool IsSometimesLiteralTag(HtmlName::Keyword keyword);

  bool IsOptionallyClosedTag(HtmlName::Keyword keyword) const;

  bool TagAllowsBriefTermination(HtmlName::Keyword keyword) const;

  MessageHandler* message_handler() const { return message_handler_; }
  const char* url() const { return url_.c_str(); }
  const GoogleUrl& google_url() const { return google_url_; }
  const char* id() const { return id_.c_str(); }
  int line_number() const { return line_number_; }
  GoogleString UrlLine() const {
    return StringPrintf("%s:%d", id(), line_number());
  }

  const DocType& doctype() const;

  void Info(const char* filename, int line, const char* msg, ...)
      INSTAWEB_PRINTF_FORMAT(4, 5);
  void Warning(const char* filename, int line, const char* msg, ...)
      INSTAWEB_PRINTF_FORMAT(4, 5);
  void Error(const char* filename, int line, const char* msg, ...)
      INSTAWEB_PRINTF_FORMAT(4, 5);
  void FatalError(const char* filename, int line, const char* msg, ...)
      INSTAWEB_PRINTF_FORMAT(4, 5);

  void InfoV(const char* file, int line, const char *msg, va_list args);
  void WarningV(const char* file, int line, const char *msg, va_list args);
  void ErrorV(const char* file, int line, const char *msg, va_list args);
  void FatalErrorV(const char* file, int line, const char* msg, va_list args);

  void InfoHere(const char* msg, ...) INSTAWEB_PRINTF_FORMAT(2, 3);
  void WarningHere(const char* msg, ...) INSTAWEB_PRINTF_FORMAT(2, 3);
  void ErrorHere(const char* msg, ...) INSTAWEB_PRINTF_FORMAT(2, 3);
  void FatalErrorHere(const char* msg, ...) INSTAWEB_PRINTF_FORMAT(2, 3);

  void ShowProgress(const char* message);

  void InfoHereV(const char *msg, va_list args) {
    InfoV(id_.c_str(), line_number_, msg, args);
  }
  void WarningHereV(const char *msg, va_list args) {
    WarningV(id_.c_str(), line_number_, msg, args);
  }
  void ErrorHereV(const char *msg, va_list args) {
    ErrorV(id_.c_str(), line_number_, msg, args);
  }
  void FatalErrorHereV(const char* msg, va_list args) {
    FatalErrorV(id_.c_str(), line_number_, msg, args);
  }

  void AddElement(HtmlElement* element, int line_number);
  void CloseElement(HtmlElement* element, HtmlElement::Style style,
                    int line_number);

  void ApplyFilter(HtmlFilter* filter);

  void set_timer(Timer* timer) { timer_ = timer; }
  Timer* timer() const { return timer_; }
  void set_log_rewrite_timing(bool x) { log_rewrite_timing_ = x; }

  void add_event_listener(HtmlFilter* listener);

  bool InsertComment(StringPiece sp);

  void set_size_limit(int64 x);
  bool size_limit_exceeded() const;

  void SetDynamicallyDisabledFilterList(StringVector* list) {
    dynamically_disabled_filter_list_ = list;
  }

  void DeferCurrentNode();

  void RestoreDeferredNode(HtmlNode* deferred_node);

  bool can_modify_urls() {
    return can_modify_urls_;
  }

 protected:
  typedef std::vector<HtmlFilter*> FilterVector;
  typedef std::list<HtmlFilter*> FilterList;
  typedef std::pair<HtmlNode*, HtmlEventList*> DeferredNode;
  typedef std::map<const HtmlNode*, HtmlEventList*> NodeToEventListMap;
  typedef std::map<HtmlFilter*, DeferredNode> FilterElementMap;
  typedef std::set<const HtmlNode*> NodeSet;

  void BeginFinishParse();
  void EndFinishParse();

  void Clear();

  size_t GetEventQueueSize();

  virtual void ParseTextInternal(const char* content, int size);

  void DetermineFiltersBehavior() {
    if (!determine_filter_behavior_called_) {
      determine_filter_behavior_called_ = true;
      can_modify_urls_ = false;
      DetermineFiltersBehaviorImpl();
    }
  }

  void DetermineFilterListBehavior(const FilterList& list) {
    for (FilterList::const_iterator i = list.begin(); i != list.end(); ++i) {
      CheckFilterBehavior(*i);
    }
  }

  void CheckFilterBehavior(HtmlFilter* filter);

  virtual void DetermineFiltersBehaviorImpl();

  void set_buffer_events(bool x) { buffer_events_ = x; }

  void DisableFiltersInjectingScripts();

  void DisableFiltersInjectingScripts(const FilterList& filters);

 private:
  void ApplyFilterHelper(HtmlFilter* filter);
  HtmlEventListIterator Last(); 
  bool IsInEventWindow(const HtmlEventListIterator& iter) const;
  void InsertNodeBeforeEvent(const HtmlEventListIterator& event,
                             HtmlNode* new_node);
  void InsertNodeAfterEvent(const HtmlEventListIterator& event,
                            HtmlNode* new_node);
  bool MoveCurrentBeforeEvent(const HtmlEventListIterator& move_to);
  bool IsDescendantOf(const HtmlNode* possible_child,
                      const HtmlNode* possible_parent);
  void SanityCheck();
  void CheckEventParent(HtmlEvent* event, HtmlElement* expect,
                        HtmlElement* actual);
  void CheckParentFromAddEvent(HtmlEvent* event);
  void FixParents(const HtmlEventListIterator& begin,
                  const HtmlEventListIterator& end_inclusive,
                  HtmlElement* new_parent);
  void CoalesceAdjacentCharactersNodes();
  void ClearEvents();
  void EmitQueue(MessageHandler* handler);
  inline void NextEvent();
  void ClearDeferredNodes();
  inline bool IsRewritableIgnoringDeferral(const HtmlNode* node) const;
  inline bool IsRewritableIgnoringEnd(const HtmlNode* node) const;
  void SetupScript(StringPiece text, bool external, HtmlElement* script);

  friend class HtmlTestingPeer;
  void AddEvent(HtmlEvent* event);
  void SetCurrent(HtmlNode* node);
  void set_coalesce_characters(bool x) { coalesce_characters_ = x; }
  size_t symbol_table_size() const {
    return string_table_.string_bytes_allocated();
  }

  void DelayLiteralTag();

  FilterVector event_listeners_;
  SymbolTableSensitive string_table_;
  FilterList filters_;
  HtmlLexer* lexer_;
  Arena<HtmlNode> nodes_;
  HtmlEventList queue_;
  HtmlEventListIterator current_;
  MessageHandler* message_handler_;
  GoogleString url_;
  GoogleUrl google_url_;
  GoogleString id_; 
  int line_number_;
  bool skip_increment_;
  bool determine_filter_behavior_called_;
  bool can_modify_urls_;
  bool determine_enabled_filters_called_;
  bool need_sanity_check_;
  bool coalesce_characters_;
  bool need_coalesce_characters_;
  bool url_valid_;
  bool log_rewrite_timing_; 
  bool running_filters_;
  bool buffer_events_;
  int64 parse_start_time_us_;
  scoped_ptr<HtmlEvent> delayed_start_literal_;
  Timer* timer_;
  HtmlFilter* current_filter_; 

  FilterElementMap open_deferred_nodes_;

  NodeToEventListMap deferred_nodes_;

  NodeSet deferred_deleted_nodes_;

  StringVector* dynamically_disabled_filter_list_;

  
};

}  

#endif