Page Speed Optimization Libraries  1.13.35.1
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
js_tokenizer.h
1 // Copyright 2014 Google Inc. All Rights Reserved.
14 
15 #ifndef PAGESPEED_KERNEL_JS_JS_TOKENIZER_H_
16 #define PAGESPEED_KERNEL_JS_JS_TOKENIZER_H_
17 
18 #include <deque>
19 #include <utility>
20 #include <vector>
21 
27 
28 namespace pagespeed {
29 
30 namespace js {
31 
32 struct JsTokenizerPatterns;
33 
62 class JsTokenizer {
63  public:
66  JsTokenizer(const JsTokenizerPatterns* patterns, StringPiece input);
67 
68  ~JsTokenizer();
69 
75  JsKeywords::Type NextToken(StringPiece* token_out);
76 
79  bool has_error() const { return error_; }
80 
83 
84  private:
91  enum ParseState {
92  kStartOfInput,
93  kExpression,
94  kOperator,
95  kPeriod,
96  kQuestionMark,
97  kOpenBrace,
98  kOpenBracket,
99  kOpenParen,
100  kBlockKeyword,
101  kBlockHeader,
102  kReturnThrow,
103  kJumpKeyword,
104  kOtherKeyword,
105  };
106 
110  enum JsonStep {
111  kJsonStart,
112  kJsonOpenBrace,
113  kJsonOpenBraceStringLiteral,
114  kIsJsonObject,
115  kIsNotJsonObject,
116  };
117 
119  JsKeywords::Type ConsumeOpenBrace(StringPiece* token_out);
120  JsKeywords::Type ConsumeCloseBrace(StringPiece* token_out);
121  JsKeywords::Type ConsumeOpenBracket(StringPiece* token_out);
122  JsKeywords::Type ConsumeCloseBracket(StringPiece* token_out);
123  JsKeywords::Type ConsumeOpenParen(StringPiece* token_out);
124  JsKeywords::Type ConsumeCloseParen(StringPiece* token_out);
125  JsKeywords::Type ConsumeBlockComment(StringPiece* token_out);
126  JsKeywords::Type ConsumeLineComment(StringPiece* token_out);
127  JsKeywords::Type ConsumeColon(StringPiece* token_out);
128  JsKeywords::Type ConsumeComma(StringPiece* token_out);
129  JsKeywords::Type ConsumeNumber(StringPiece* token_out);
130  JsKeywords::Type ConsumeOperator(StringPiece* token_out);
131  JsKeywords::Type ConsumePeriod(StringPiece* token_out);
132  JsKeywords::Type ConsumeQuestionMark(StringPiece* token_out);
133  JsKeywords::Type ConsumeRegex(StringPiece* token_out);
134  JsKeywords::Type ConsumeSemicolon(StringPiece* token_out);
135  JsKeywords::Type ConsumeSlash(StringPiece* token_out);
136  JsKeywords::Type ConsumeString(StringPiece* token_out);
137 
141  bool TryConsumeComment(
142  JsKeywords::Type* type_out, StringPiece* token_out);
143  bool TryConsumeIdentifierOrKeyword(
144  JsKeywords::Type* type_out, StringPiece* token_out);
145  bool TryConsumeWhitespace(
146  bool allow_semicolon_insertion,
147  JsKeywords::Type* type_out, StringPiece* token_out);
148 
150  JsKeywords::Type Error(StringPiece* token_out);
151 
157  JsKeywords::Type Emit(JsKeywords::Type type, int num_chars,
158  StringPiece* token_out);
159 
161  void PushBlockHeader();
162  void PushExpression();
163  void PushOperator();
164 
170  bool TryInsertLinebreakSemicolon();
171 
174  static bool CanPreceedObjectLiteral(ParseState state);
175 
176  const JsTokenizerPatterns* patterns_;
177  std::vector<ParseState> parse_stack_;
178  std::deque<std::pair<JsKeywords::Type, StringPiece> > lookahead_queue_;
179  StringPiece input_;
180  JsonStep json_step_;
181  bool start_of_line_;
182  bool error_;
183 
184 
185 };
186 
195  public:
198 
199  const RE2 identifier_pattern;
200  const RE2 line_comment_pattern;
201  const RE2 numeric_literal_pattern;
202  const RE2 operator_pattern;
203  const RE2 regex_literal_pattern;
204  const RE2 string_literal_pattern;
205  const RE2 whitespace_pattern;
206  const RE2 line_continuation_pattern;
207 
208  private:
209 
210 };
211 
212 }
213 
214 }
215 
216 #endif
GoogleString ParseStackForTest() const
Return a string representing the current parse stack, for testing only.
JsTokenizer(const JsTokenizerPatterns *patterns, StringPiece input)
bool has_error() const
Definition: js_tokenizer.h:79
Type
Definition: js_keywords.h:33
std::string GoogleString
PAGESPEED_KERNEL_BASE_STRING_H_.
Definition: string.h:24
Definition: js_tokenizer.h:194
Definition: js_tokenizer.h:62
JsKeywords::Type NextToken(StringPiece *token_out)