diff --git a/src/grammar/RustLexer.g4 b/src/grammar/RustLexer.g4 index 3d8f3aeb28fa7..f062d33f25e25 100644 --- a/src/grammar/RustLexer.g4 +++ b/src/grammar/RustLexer.g4 @@ -8,14 +8,14 @@ lexer grammar RustLexer; tokens { - EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUT, + EQ, LT, LE, EQEQ, NE, GE, GT, ANDAND, OROR, NOT, TILDE, PLUS, MINUS, STAR, SLASH, PERCENT, CARET, AND, OR, SHL, SHR, BINOP, BINOPEQ, AT, DOT, DOTDOT, DOTDOTDOT, COMMA, SEMI, COLON, MOD_SEP, RARROW, FAT_ARROW, LPAREN, RPAREN, LBRACKET, RBRACKET, - LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR, + LBRACE, RBRACE, POUND, DOLLAR, UNDERSCORE, LIT_CHAR, LIT_BYTE, LIT_INTEGER, LIT_FLOAT, LIT_STR, LIT_STR_RAW, LIT_BINARY, - LIT_BINARY_RAW, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT, - COMMENT, SHEBANG + LIT_BINARY_RAW, QUESTION, IDENT, LIFETIME, WHITESPACE, DOC_COMMENT, + COMMENT, SHEBANG, UTF8_BOM } import xidstart , xidcontinue; diff --git a/src/grammar/verify.rs b/src/grammar/verify.rs index dec797747c270..10b8abfc78606 100644 --- a/src/grammar/verify.rs +++ b/src/grammar/verify.rs @@ -111,7 +111,7 @@ fn parse_token_list(file: &str) -> HashMap { "LIT_BINARY_RAW" => token::Literal(token::BinaryRaw(Name(0), 0), None), "QUESTION" => token::Question, "SHEBANG" => token::Shebang(Name(0)), - _ => continue, + _ => panic!("Bad token str `{}`", val), }; res.insert(num.to_string(), tok);