Tokenizer 1059 third_party/protobuf/java/src/main/java/com/google/protobuf/TextFormat.java final Tokenizer tokenizer = new Tokenizer(input); Tokenizer 1070 third_party/protobuf/java/src/main/java/com/google/protobuf/TextFormat.java private static void mergeField(final Tokenizer tokenizer, Tokenizer 134 third_party/protobuf/src/google/protobuf/compiler/importer.cc io::Tokenizer tokenizer(input.get(), &file_error_collector); Tokenizer 114 third_party/protobuf/src/google/protobuf/compiler/parser.cc inline bool Parser::LookingAtType(io::Tokenizer::TokenType token_type) { Tokenizer 119 third_party/protobuf/src/google/protobuf/compiler/parser.cc return LookingAtType(io::Tokenizer::TYPE_END); Tokenizer 150 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 161 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 163 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (!io::Tokenizer::ParseInteger(input_->current().text, Tokenizer 193 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 194 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (!io::Tokenizer::ParseInteger(input_->current().text, max_value, Tokenizer 209 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) { Tokenizer 210 third_party/protobuf/src/google/protobuf/compiler/parser.cc *output = io::Tokenizer::ParseFloat(input_->current().text); Tokenizer 213 third_party/protobuf/src/google/protobuf/compiler/parser.cc } else if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 216 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (!io::Tokenizer::ParseInteger(input_->current().text, Tokenizer 239 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 240 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::ParseString(input_->current().text, output); Tokenizer 243 third_party/protobuf/src/google/protobuf/compiler/parser.cc while (LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 244 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::ParseStringAppend(input_->current().text, output); Tokenizer 341 third_party/protobuf/src/google/protobuf/compiler/parser.cc void Parser::LocationRecorder::StartAt(const io::Tokenizer::Token& token) { Tokenizer 346 third_party/protobuf/src/google/protobuf/compiler/parser.cc void Parser::LocationRecorder::EndAt(const io::Tokenizer::Token& token) { Tokenizer 380 third_party/protobuf/src/google/protobuf/compiler/parser.cc } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) { Tokenizer 398 third_party/protobuf/src/google/protobuf/compiler/parser.cc } else if (LookingAtType(io::Tokenizer::TYPE_SYMBOL)) { Tokenizer 411 third_party/protobuf/src/google/protobuf/compiler/parser.cc bool Parser::Parse(io::Tokenizer* input, FileDescriptorProto* file) { Tokenizer 423 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_START)) { Tokenizer 467 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token syntax_token = input_->current(); Tokenizer 649 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token label_token = input_->current(); Tokenizer 675 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token name_token = input_->current(); Tokenizer 907 third_party/protobuf/src/google/protobuf/compiler/parser.cc if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 1014 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_START: Tokenizer 1018 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_END: Tokenizer 1022 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_IDENTIFIER: { Tokenizer 1035 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_INTEGER: { Tokenizer 1053 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_FLOAT: { Tokenizer 1061 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_STRING: { Tokenizer 1073 third_party/protobuf/src/google/protobuf/compiler/parser.cc case io::Tokenizer::TYPE_SYMBOL: Tokenizer 1109 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token start_token; Tokenizer 1157 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token extendee_start = input_->current(); Tokenizer 1160 third_party/protobuf/src/google/protobuf/compiler/parser.cc io::Tokenizer::Token extendee_end = input_->previous(); Tokenizer 73 third_party/protobuf/src/google/protobuf/compiler/parser.h bool Parse(io::Tokenizer* input, FileDescriptorProto* file); Tokenizer 150 third_party/protobuf/src/google/protobuf/compiler/parser.h inline bool LookingAtType(io::Tokenizer::TokenType token_type); Tokenizer 234 third_party/protobuf/src/google/protobuf/compiler/parser.h void StartAt(const io::Tokenizer::Token& token); Tokenizer 239 third_party/protobuf/src/google/protobuf/compiler/parser.h void EndAt(const io::Tokenizer::Token& token); Tokenizer 420 third_party/protobuf/src/google/protobuf/compiler/parser.h io::Tokenizer* input_; Tokenizer 107 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc input_.reset(new io::Tokenizer(raw_input_.get(), &error_collector_)); Tokenizer 121 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); Tokenizer 141 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); Tokenizer 164 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); Tokenizer 178 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc scoped_ptr<io::Tokenizer> input_; Tokenizer 1300 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); Tokenizer 1347 third_party/protobuf/src/google/protobuf/compiler/parser_unittest.cc EXPECT_EQ(io::Tokenizer::TYPE_END, input_->current().type); Tokenizer 185 third_party/protobuf/src/google/protobuf/io/tokenizer.cc Tokenizer::Tokenizer(ZeroCopyInputStream* input, Tokenizer 208 third_party/protobuf/src/google/protobuf/io/tokenizer.cc Tokenizer::~Tokenizer() { Tokenizer 219 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::NextChar() { Tokenizer 240 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::Refresh() { Tokenizer 270 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::RecordTo(string* target) { Tokenizer 275 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::StopRecording() { Tokenizer 287 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::StartToken() { Tokenizer 295 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::EndToken() { Tokenizer 304 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline bool Tokenizer::LookingAt() { Tokenizer 309 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline bool Tokenizer::TryConsumeOne() { Tokenizer 318 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline bool Tokenizer::TryConsume(char c) { Tokenizer 328 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::ConsumeZeroOrMore() { Tokenizer 335 third_party/protobuf/src/google/protobuf/io/tokenizer.cc inline void Tokenizer::ConsumeOneOrMore(const char* error) { Tokenizer 349 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::ConsumeString(char delimiter) { Tokenizer 411 third_party/protobuf/src/google/protobuf/io/tokenizer.cc Tokenizer::TokenType Tokenizer::ConsumeNumber(bool started_with_zero, Tokenizer 466 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::ConsumeLineComment(string* content) { Tokenizer 477 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::ConsumeBlockComment(string* content) { Tokenizer 527 third_party/protobuf/src/google/protobuf/io/tokenizer.cc Tokenizer::NextCommentStatus Tokenizer::TryConsumeCommentStart() { Tokenizer 551 third_party/protobuf/src/google/protobuf/io/tokenizer.cc bool Tokenizer::Next() { Tokenizer 742 third_party/protobuf/src/google/protobuf/io/tokenizer.cc bool Tokenizer::NextWithComments(string* prev_trailing_comments, Tokenizer 835 third_party/protobuf/src/google/protobuf/io/tokenizer.cc bool Tokenizer::ParseInteger(const string& text, uint64 max_value, Tokenizer 872 third_party/protobuf/src/google/protobuf/io/tokenizer.cc double Tokenizer::ParseFloat(const string& text) { Tokenizer 1011 third_party/protobuf/src/google/protobuf/io/tokenizer.cc void Tokenizer::ParseStringAppend(const string& text, string* output) { Tokenizer 52 third_party/protobuf/src/google/protobuf/io/tokenizer.h class Tokenizer; Tokenizer 82 third_party/protobuf/src/google/protobuf/io/tokenizer.h class LIBPROTOBUF_EXPORT Tokenizer { Tokenizer 87 third_party/protobuf/src/google/protobuf/io/tokenizer.h Tokenizer(ZeroCopyInputStream* input, ErrorCollector* error_collector); Tokenizer 88 third_party/protobuf/src/google/protobuf/io/tokenizer.h ~Tokenizer(); Tokenizer 233 third_party/protobuf/src/google/protobuf/io/tokenizer.h GOOGLE_DISALLOW_EVIL_CONSTRUCTORS(Tokenizer); Tokenizer 367 third_party/protobuf/src/google/protobuf/io/tokenizer.h inline const Tokenizer::Token& Tokenizer::current() { Tokenizer 371 third_party/protobuf/src/google/protobuf/io/tokenizer.h inline const Tokenizer::Token& Tokenizer::previous() { Tokenizer 375 third_party/protobuf/src/google/protobuf/io/tokenizer.h inline void Tokenizer::ParseString(const string& text, string* output) { Tokenizer 183 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_TRUE(Tokenizer::ParseInteger(text, kuint64max, &result)); Tokenizer 198 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::TokenType type; Tokenizer 208 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "hello", Tokenizer::TYPE_IDENTIFIER }, Tokenizer 211 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "123", Tokenizer::TYPE_INTEGER }, Tokenizer 212 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "0xab6", Tokenizer::TYPE_INTEGER }, Tokenizer 213 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "0XAB6", Tokenizer::TYPE_INTEGER }, Tokenizer 214 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "0X1234567", Tokenizer::TYPE_INTEGER }, Tokenizer 215 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "0x89abcdef", Tokenizer::TYPE_INTEGER }, Tokenizer 216 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "0x89ABCDEF", Tokenizer::TYPE_INTEGER }, Tokenizer 217 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "01234567", Tokenizer::TYPE_INTEGER }, Tokenizer 220 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "123.45", Tokenizer::TYPE_FLOAT }, Tokenizer 221 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1.", Tokenizer::TYPE_FLOAT }, Tokenizer 222 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1e3", Tokenizer::TYPE_FLOAT }, Tokenizer 223 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1E3", Tokenizer::TYPE_FLOAT }, Tokenizer 224 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1e-3", Tokenizer::TYPE_FLOAT }, Tokenizer 225 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1e+3", Tokenizer::TYPE_FLOAT }, Tokenizer 226 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1.e3", Tokenizer::TYPE_FLOAT }, Tokenizer 227 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "1.2e3", Tokenizer::TYPE_FLOAT }, Tokenizer 228 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { ".1", Tokenizer::TYPE_FLOAT }, Tokenizer 229 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { ".1e3", Tokenizer::TYPE_FLOAT }, Tokenizer 230 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { ".1e-3", Tokenizer::TYPE_FLOAT }, Tokenizer 231 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { ".1e+3", Tokenizer::TYPE_FLOAT }, Tokenizer 234 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "'hello'", Tokenizer::TYPE_STRING }, Tokenizer 235 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "\"foo\"", Tokenizer::TYPE_STRING }, Tokenizer 236 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "'a\"b'", Tokenizer::TYPE_STRING }, Tokenizer 237 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "\"a'b\"", Tokenizer::TYPE_STRING }, Tokenizer 238 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "'a\\'b'", Tokenizer::TYPE_STRING }, Tokenizer 239 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "\"a\\\"b\"", Tokenizer::TYPE_STRING }, Tokenizer 240 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "'\\xf'", Tokenizer::TYPE_STRING }, Tokenizer 241 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "'\\0'", Tokenizer::TYPE_STRING }, Tokenizer 244 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { "+", Tokenizer::TYPE_SYMBOL }, Tokenizer 245 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { ".", Tokenizer::TYPE_SYMBOL }, Tokenizer 254 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 257 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); Tokenizer 280 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(Tokenizer::TYPE_END, tokenizer.current().type); Tokenizer 298 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 304 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT); Tokenizer 307 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT); Tokenizer 310 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT); Tokenizer 313 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(tokenizer.current().type, Tokenizer::TYPE_FLOAT); Tokenizer 329 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::Token output[10]; // The compiler wants a constant array Tokenizer 343 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 0 }, Tokenizer 348 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo" , 0, 0, 3 }, Tokenizer 349 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_INTEGER , "1" , 0, 4, 5 }, Tokenizer 350 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_FLOAT , "1.2" , 0, 6, 9 }, Tokenizer 351 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "+" , 0, 10, 11 }, Tokenizer 352 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_STRING , "'bar'", 0, 12, 17 }, Tokenizer 353 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 17, 17 }, Tokenizer 358 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "!" , 0, 0, 1 }, Tokenizer 359 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "@" , 0, 1, 2 }, Tokenizer 360 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "+" , 0, 2, 3 }, Tokenizer 361 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "%" , 0, 3, 4 }, Tokenizer 362 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 4, 4 }, Tokenizer 367 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 368 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 4, 7 }, Tokenizer 369 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "rab", 1, 0, 3 }, Tokenizer 370 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "oof", 1, 4, 7 }, Tokenizer 371 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 1, 7, 7 }, Tokenizer 376 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 377 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 8, 11 }, Tokenizer 378 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "baz", 0, 16, 19 }, Tokenizer 379 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 19, 19 }, Tokenizer 384 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_STRING , "\"foo\tbar\"", 0, 0, 12 }, Tokenizer 385 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "baz" , 0, 13, 16 }, Tokenizer 386 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 16, 16 }, Tokenizer 392 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 393 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 0, 3 }, Tokenizer 394 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 1, 30, 30 }, Tokenizer 399 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 400 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 34, 37 }, Tokenizer 401 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 37, 37 }, Tokenizer 407 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 408 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL , "#" , 0, 4, 5 }, Tokenizer 409 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 0, 6, 9 }, Tokenizer 410 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "baz", 1, 0, 3 }, Tokenizer 411 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 1, 3, 3 }, Tokenizer 416 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_SYMBOL, "\300", 0, 0, 1 }, Tokenizer 417 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 0, 1, 1 }, Tokenizer 422 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "foo", 0, 0, 3 }, Tokenizer 423 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_IDENTIFIER, "bar", 1, 11, 14 }, Tokenizer 424 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc { Tokenizer::TYPE_END , "" , 1, 14, 14 }, Tokenizer 434 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 437 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(Tokenizer::TYPE_START, tokenizer.current().type); Tokenizer 445 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::Token token; Tokenizer 451 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::Token previous = tokenizer.current(); Tokenizer 454 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc if (token.type != Tokenizer::TYPE_END) { Tokenizer 474 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc } while (token.type != Tokenizer::TYPE_END); Tokenizer 499 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 500 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc tokenizer.set_comment_style(Tokenizer::SH_COMMENT_STYLE); Tokenizer 686 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 692 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer2(&input2, &error_collector); Tokenizer 746 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("zxy", kuint64max, &i), Tokenizer 748 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("1.2", kuint64max, &i), Tokenizer 750 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("08", kuint64max, &i), Tokenizer 752 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("0xg", kuint64max, &i), Tokenizer 754 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseInteger("-1", kuint64max, &i), Tokenizer 759 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_TRUE (Tokenizer::ParseInteger("0", 0, &i)); Tokenizer 760 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_FALSE(Tokenizer::ParseInteger("1", 0, &i)); Tokenizer 761 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_TRUE (Tokenizer::ParseInteger("1", 1, &i)); Tokenizer 762 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_TRUE (Tokenizer::ParseInteger("12345", 12345, &i)); Tokenizer 763 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_FALSE(Tokenizer::ParseInteger("12346", 12345, &i)); Tokenizer 764 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_TRUE (Tokenizer::ParseInteger("0xFFFFFFFFFFFFFFFF" , kuint64max, &i)); Tokenizer 765 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_FALSE(Tokenizer::ParseInteger("0x10000000000000000", kuint64max, &i)); Tokenizer 769 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1 , Tokenizer::ParseFloat("1.")); Tokenizer 770 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1e3 , Tokenizer::ParseFloat("1e3")); Tokenizer 771 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1e3 , Tokenizer::ParseFloat("1E3")); Tokenizer 772 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1.5e3, Tokenizer::ParseFloat("1.5e3")); Tokenizer 773 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.1 , Tokenizer::ParseFloat(".1")); Tokenizer 774 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.25 , Tokenizer::ParseFloat(".25")); Tokenizer 775 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.1e3 , Tokenizer::ParseFloat(".1e3")); Tokenizer 776 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.25e3, Tokenizer::ParseFloat(".25e3")); Tokenizer 777 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.1e+3, Tokenizer::ParseFloat(".1e+3")); Tokenizer 778 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(.1e-3, Tokenizer::ParseFloat(".1e-3")); Tokenizer 779 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(5 , Tokenizer::ParseFloat("5")); Tokenizer 780 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(6e-12, Tokenizer::ParseFloat("6e-12")); Tokenizer 781 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1.2 , Tokenizer::ParseFloat("1.2")); Tokenizer 782 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1.e2 , Tokenizer::ParseFloat("1.e2")); Tokenizer 785 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e")); Tokenizer 786 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1e-")); Tokenizer 787 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.e")); Tokenizer 790 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1f")); Tokenizer 791 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1.0f")); Tokenizer 792 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DOUBLE_EQ(1, Tokenizer::ParseFloat("1F")); Tokenizer 796 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ( 0.0, Tokenizer::ParseFloat("1e-9999999999999999999999999999")); Tokenizer 797 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_EQ(HUGE_VAL, Tokenizer::ParseFloat("1e+9999999999999999999999999999")); Tokenizer 801 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("zxy"), Tokenizer 803 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("1-e0"), Tokenizer 805 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseFloat("-1.0"), Tokenizer 812 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'hello'", &output); Tokenizer 814 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("\"blah\\nblah2\"", &output); Tokenizer 816 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\1x\\1\\123\\739\\52\\334n\\3'", &output); Tokenizer 818 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\x20\\x4'", &output); Tokenizer 822 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("\"\\a\\l\\v\\t", &output); // \l is invalid Tokenizer 824 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'", &output); Tokenizer 826 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\", &output); Tokenizer 831 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\U00024b62XX'", &output); Tokenizer 834 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\u0024\\u00a2\\u20ac\\ud852\\udf62XX'", &output); Tokenizer 839 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'\\ud852XX'", &output); Tokenizer 842 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("\\u0", &output); Tokenizer 847 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc EXPECT_DEBUG_DEATH(Tokenizer::ParseString("", &output), Tokenizer 855 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseStringAppend("'hello'", &output); Tokenizer 857 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer::ParseString("'hello'", &output); Tokenizer 962 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 988 third_party/protobuf/src/google/protobuf/io/tokenizer_unittest.cc Tokenizer tokenizer(&input, &error_collector); Tokenizer 209 third_party/protobuf/src/google/protobuf/text_format.cc tokenizer_.set_comment_style(io::Tokenizer::SH_COMMENT_STYLE); Tokenizer 223 third_party/protobuf/src/google/protobuf/text_format.cc if (LookingAtType(io::Tokenizer::TYPE_END)) { Tokenizer 238 third_party/protobuf/src/google/protobuf/text_format.cc return suc && LookingAtType(io::Tokenizer::TYPE_END); Tokenizer 591 third_party/protobuf/src/google/protobuf/text_format.cc if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 616 third_party/protobuf/src/google/protobuf/text_format.cc if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 622 third_party/protobuf/src/google/protobuf/text_format.cc LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 654 third_party/protobuf/src/google/protobuf/text_format.cc if (LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 655 third_party/protobuf/src/google/protobuf/text_format.cc while (LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 682 third_party/protobuf/src/google/protobuf/text_format.cc if (!LookingAtType(io::Tokenizer::TYPE_INTEGER) && Tokenizer 683 third_party/protobuf/src/google/protobuf/text_format.cc !LookingAtType(io::Tokenizer::TYPE_FLOAT) && Tokenizer 684 third_party/protobuf/src/google/protobuf/text_format.cc !LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 693 third_party/protobuf/src/google/protobuf/text_format.cc if (has_minus && LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 713 third_party/protobuf/src/google/protobuf/text_format.cc bool LookingAtType(io::Tokenizer::TokenType token_type) { Tokenizer 720 third_party/protobuf/src/google/protobuf/text_format.cc if (!LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 734 third_party/protobuf/src/google/protobuf/text_format.cc if (!LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 740 third_party/protobuf/src/google/protobuf/text_format.cc while (LookingAtType(io::Tokenizer::TYPE_STRING)) { Tokenizer 741 third_party/protobuf/src/google/protobuf/text_format.cc io::Tokenizer::ParseStringAppend(tokenizer_.current().text, text); Tokenizer 752 third_party/protobuf/src/google/protobuf/text_format.cc if (!LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 757 third_party/protobuf/src/google/protobuf/text_format.cc if (!io::Tokenizer::ParseInteger(tokenizer_.current().text, Tokenizer 809 third_party/protobuf/src/google/protobuf/text_format.cc if (LookingAtType(io::Tokenizer::TYPE_INTEGER)) { Tokenizer 815 third_party/protobuf/src/google/protobuf/text_format.cc } else if (LookingAtType(io::Tokenizer::TYPE_FLOAT)) { Tokenizer 817 third_party/protobuf/src/google/protobuf/text_format.cc *value = io::Tokenizer::ParseFloat(tokenizer_.current().text); Tokenizer 821 third_party/protobuf/src/google/protobuf/text_format.cc } else if (LookingAtType(io::Tokenizer::TYPE_IDENTIFIER)) { Tokenizer 901 third_party/protobuf/src/google/protobuf/text_format.cc io::Tokenizer tokenizer_; Tokenizer 33 tools/gn/command_args.cc if (offset == 0 || Tokenizer::IsNewline(data, offset)) Tokenizer 39 tools/gn/command_args.cc if (Tokenizer::IsNewline(data, cur)) Tokenizer 69 tools/gn/command_args.cc Tokenizer::ByteOffsetOfNthLine(data, location.line_number()); Tokenizer 19 tools/gn/err.cc size_t line_off = Tokenizer::ByteOffsetOfNthLine(data, n); Tokenizer 21 tools/gn/err.cc while (end < data.size() && !Tokenizer::IsNewline(data, end)) Tokenizer 58 tools/gn/input_conversion.cc std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, err); Tokenizer 234 tools/gn/input_file_manager.cc std::vector<Token> tokens = Tokenizer::Tokenize(file, err); Tokenizer 79 tools/gn/loader_unittest.cc canned->tokens = Tokenizer::Tokenize(canned->input_file.get(), &err); Tokenizer 18 tools/gn/parser_unittest.cc *result = Tokenizer::Tokenize(input, &err); Tokenizer 63 tools/gn/parser_unittest.cc std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); Tokenizer 81 tools/gn/parser_unittest.cc std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err); Tokenizer 240 tools/gn/setup.cc args_tokens_ = Tokenizer::Tokenize(args_input_file_.get(), &err); Tokenizer 348 tools/gn/setup.cc dotfile_tokens_ = Tokenizer::Tokenize(dotfile_input_file_.get(), &err); Tokenizer 65 tools/gn/string_utils.cc if (!Tokenizer::IsIdentifierFirstChar(input[*i])) { Tokenizer 76 tools/gn/string_utils.cc while (*i < size && Tokenizer::IsIdentifierContinuingChar(input[*i])) Tokenizer 74 tools/gn/tokenizer.cc Tokenizer::Tokenizer(const InputFile* input_file, Err* err) Tokenizer 83 tools/gn/tokenizer.cc Tokenizer::~Tokenizer() { Tokenizer 87 tools/gn/tokenizer.cc std::vector<Token> Tokenizer::Tokenize(const InputFile* input_file, Err* err) { Tokenizer 88 tools/gn/tokenizer.cc Tokenizer t(input_file, err); Tokenizer 92 tools/gn/tokenizer.cc std::vector<Token> Tokenizer::Run() { Tokenizer 140 tools/gn/tokenizer.cc size_t Tokenizer::ByteOffsetOfNthLine(const base::StringPiece& buf, int n) { Tokenizer 161 tools/gn/tokenizer.cc bool Tokenizer::IsNewline(const base::StringPiece& buffer, size_t offset) { Tokenizer 168 tools/gn/tokenizer.cc void Tokenizer::AdvanceToNextToken() { Tokenizer 173 tools/gn/tokenizer.cc Token::Type Tokenizer::ClassifyCurrent() const { Tokenizer 224 tools/gn/tokenizer.cc void Tokenizer::AdvanceToEndOfToken(const Location& location, Tokenizer 310 tools/gn/tokenizer.cc bool Tokenizer::IsCurrentWhitespace() const { Tokenizer 317 tools/gn/tokenizer.cc bool Tokenizer::IsCurrentStringTerminator(char quote_char) const { Tokenizer 333 tools/gn/tokenizer.cc bool Tokenizer::IsCurrentNewline() const { Tokenizer 337 tools/gn/tokenizer.cc void Tokenizer::Advance() { Tokenizer 348 tools/gn/tokenizer.cc Location Tokenizer::GetCurrentLocation() const { Tokenizer 352 tools/gn/tokenizer.cc Err Tokenizer::GetErrorForInvalidToken(const Location& location) const { Tokenizer 17 tools/gn/tokenizer.h class Tokenizer { Tokenizer 45 tools/gn/tokenizer.h explicit Tokenizer(const InputFile* input_file, Err* err); Tokenizer 46 tools/gn/tokenizer.h ~Tokenizer(); Tokenizer 85 tools/gn/tokenizer.h DISALLOW_COPY_AND_ASSIGN(Tokenizer); Tokenizer 23 tools/gn/tokenizer_unittest.cc std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err); Tokenizer 38 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Empty) { Tokenizer 43 tools/gn/tokenizer_unittest.cc std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err); Tokenizer 49 tools/gn/tokenizer_unittest.cc results = Tokenizer::Tokenize(&whitespace_input, &err); Tokenizer 53 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Identifier) { Tokenizer 60 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Integer) { Tokenizer 68 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, IntegerNoSpace) { Tokenizer 76 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, String) { Tokenizer 86 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Operator) { Tokenizer 109 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Scoper) { Tokenizer 121 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, FunctionCall) { Tokenizer 136 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, Locations) { Tokenizer 140 tools/gn/tokenizer_unittest.cc std::vector<Token> results = Tokenizer::Tokenize(&input, &err); Tokenizer 149 tools/gn/tokenizer_unittest.cc TEST(Tokenizer, ByteOffsetOfNthLine) { Tokenizer 150 tools/gn/tokenizer_unittest.cc EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1)); Tokenizer 155 tools/gn/tokenizer_unittest.cc EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]); Tokenizer 156 tools/gn/tokenizer_unittest.cc EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]); Tokenizer 162 tools/gn/tokenizer_unittest.cc EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1)); Tokenizer 163 tools/gn/tokenizer_unittest.cc EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));