js_tokenizer.h

// Copyright 2014 Google Inc. All Rights Reserved.

#ifndef PAGESPEED_KERNEL_JS_JS_TOKENIZER_H_
#define PAGESPEED_KERNEL_JS_JS_TOKENIZER_H_

#include <deque>
#include <utility>
#include <vector>

#include "pagespeed/kernel/base/basictypes.h"
#include "pagespeed/kernel/base/string.h"
#include "pagespeed/kernel/base/string_util.h"
#include "pagespeed/kernel/js/js_keywords.h"
#include "pagespeed/kernel/util/re2.h"

namespace pagespeed {

namespace js {

struct JsTokenizerPatterns;

class JsTokenizer {
 public:
  JsTokenizer(const JsTokenizerPatterns* patterns, StringPiece input);

  ~JsTokenizer();

  JsKeywords::Type NextToken(StringPiece* token_out);

  bool has_error() const { return error_; }

  GoogleString ParseStackForTest() const;

 private:
  enum ParseState {
    kStartOfInput,  
    kExpression,
    kOperator,      
    kPeriod,
    kQuestionMark,
    kOpenBrace,
    kOpenBracket,
    kOpenParen,
    kBlockKeyword,  
    kBlockHeader,   
    kReturnThrow,   
    kJumpKeyword,   
    kOtherKeyword,  
  };

  enum JsonStep {
    kJsonStart,
    kJsonOpenBrace,
    kJsonOpenBraceStringLiteral,
    kIsJsonObject,
    kIsNotJsonObject,
  };

  JsKeywords::Type ConsumeOpenBrace(StringPiece* token_out);
  JsKeywords::Type ConsumeCloseBrace(StringPiece* token_out);
  JsKeywords::Type ConsumeOpenBracket(StringPiece* token_out);
  JsKeywords::Type ConsumeCloseBracket(StringPiece* token_out);
  JsKeywords::Type ConsumeOpenParen(StringPiece* token_out);
  JsKeywords::Type ConsumeCloseParen(StringPiece* token_out);
  JsKeywords::Type ConsumeBlockComment(StringPiece* token_out);
  JsKeywords::Type ConsumeLineComment(StringPiece* token_out);
  JsKeywords::Type ConsumeColon(StringPiece* token_out);
  JsKeywords::Type ConsumeComma(StringPiece* token_out);
  JsKeywords::Type ConsumeNumber(StringPiece* token_out);
  JsKeywords::Type ConsumeOperator(StringPiece* token_out);
  JsKeywords::Type ConsumePeriod(StringPiece* token_out);
  JsKeywords::Type ConsumeQuestionMark(StringPiece* token_out);
  JsKeywords::Type ConsumeRegex(StringPiece* token_out);
  JsKeywords::Type ConsumeSemicolon(StringPiece* token_out);
  JsKeywords::Type ConsumeSlash(StringPiece* token_out);
  JsKeywords::Type ConsumeString(StringPiece* token_out);

  bool TryConsumeComment(
      JsKeywords::Type* type_out, StringPiece* token_out);
  bool TryConsumeIdentifierOrKeyword(
      JsKeywords::Type* type_out, StringPiece* token_out);
  bool TryConsumeWhitespace(
      bool allow_semicolon_insertion,
      JsKeywords::Type* type_out, StringPiece* token_out);

  JsKeywords::Type Error(StringPiece* token_out);

  JsKeywords::Type Emit(JsKeywords::Type type, int num_chars,
                        StringPiece* token_out);

  void PushBlockHeader();
  void PushExpression();
  void PushOperator();

  bool TryInsertLinebreakSemicolon();

  static bool CanPreceedObjectLiteral(ParseState state);

  const JsTokenizerPatterns* patterns_;
  std::vector<ParseState> parse_stack_;
  std::deque<std::pair<JsKeywords::Type, StringPiece> > lookahead_queue_;
  StringPiece input_; 
  JsonStep json_step_;
  bool start_of_line_; 
  bool error_;

  
};

struct JsTokenizerPatterns {
 public:
  JsTokenizerPatterns();
  ~JsTokenizerPatterns();

  const RE2 identifier_pattern;
  const RE2 line_comment_pattern;
  const RE2 numeric_literal_pattern;
  const RE2 operator_pattern;
  const RE2 regex_literal_pattern;
  const RE2 string_literal_pattern;
  const RE2 whitespace_pattern;
  const RE2 line_continuation_pattern;

 private:
  
};

}  

}  

#endif