diff --git a/config.yml b/config.yml index 21502ea8ca..cc5eb7e099 100644 --- a/config.yml +++ b/config.yml @@ -377,7 +377,7 @@ tokens: - name: AMPERSAND_EQUAL comment: "&=" - name: BACKTICK - comment: "`" + comment: "` as a method name" - name: BACK_REFERENCE comment: "a back reference" - name: BANG @@ -388,6 +388,10 @@ tokens: comment: "!~" - name: BRACE_LEFT comment: "{" + - name: BRACE_LEFT_ARGUMENT + comment: "{ for a block following a parenthesized argument" + - name: BRACE_LEFT_HASH + comment: "{ for a hash literal" - name: BRACKET_LEFT comment: "[" - name: BRACKET_LEFT_ARRAY @@ -584,6 +588,8 @@ tokens: comment: "a numbered reference to a capture group in the previous regular expression match" - name: PARENTHESIS_LEFT comment: "(" + - name: PARENTHESIS_LEFT_GROUPING + comment: "( scanned at the beginning of an expression" - name: PARENTHESIS_LEFT_PARENTHESES comment: "( for a parentheses node" - name: PERCENT @@ -658,6 +664,8 @@ tokens: comment: "unary **" - name: WORDS_SEP comment: "a separator between words in a list" + - name: XSTRING_BEGIN + comment: "the beginning of an execution string" - name: __END__ comment: "marker for the point in the file at which the parser should stop" flags: diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 4d92842bda..749f11173a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -78,6 +78,8 @@ def deconstruct_keys(keys) # :nodoc: BANG_EQUAL: :on_op, BANG_TILDE: :on_op, BRACE_LEFT: :on_lbrace, + BRACE_LEFT_ARGUMENT: :on_lbrace, + BRACE_LEFT_HASH: :on_lbrace, BRACE_RIGHT: :on_rbrace, BRACKET_LEFT: :on_lbracket, BRACKET_LEFT_ARRAY: :on_lbracket, @@ -191,6 +193,7 @@ def deconstruct_keys(keys) # :nodoc: NEWLINE: :on_nl, NUMBERED_REFERENCE: :on_backref, PARENTHESIS_LEFT: :on_lparen, + PARENTHESIS_LEFT_GROUPING: :on_lparen, PARENTHESIS_LEFT_PARENTHESES: :on_lparen, PARENTHESIS_RIGHT: :on_rparen, PERCENT: :on_op, @@ -233,6 +236,7 @@ def deconstruct_keys(keys) # :nodoc: USTAR: :on_op, USTAR_STAR: :on_op, WORDS_SEP: :on_words_sep, + XSTRING_BEGIN: :on_backtick, "__END__": :on___end__ }.freeze diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 0b2f4b9da7..c26f48bcfb 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -28,11 +28,13 @@ class Lexer # :nodoc: AMPERSAND_DOT: :tANDDOT, AMPERSAND_EQUAL: :tOP_ASGN, BACK_REFERENCE: :tBACK_REF, - BACKTICK: :tXSTRING_BEG, + BACKTICK: :tBACK_REF2, BANG: :tBANG, BANG_EQUAL: :tNEQ, BANG_TILDE: :tNMATCH, BRACE_LEFT: :tLCURLY, + BRACE_LEFT_ARGUMENT: :tLBRACE_ARG, + BRACE_LEFT_HASH: :tLBRACE, BRACE_RIGHT: :tRCURLY, BRACKET_LEFT: :tLBRACK2, BRACKET_LEFT_ARRAY: :tLBRACK, @@ -141,6 +143,7 @@ class Lexer # :nodoc: NEWLINE: :tNL, NUMBERED_REFERENCE: :tNTH_REF, PARENTHESIS_LEFT: :tLPAREN2, + PARENTHESIS_LEFT_GROUPING: :tLPAREN, PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG, PARENTHESIS_RIGHT: :tRPAREN, PERCENT: :tPERCENT, @@ -180,32 +183,10 @@ class Lexer # :nodoc: UPLUS: :tUPLUS, USTAR: :tSTAR, USTAR_STAR: :tDSTAR, - WORDS_SEP: :tSPACE + WORDS_SEP: :tSPACE, + XSTRING_BEGIN: :tXSTRING_BEG } - # These constants represent flags in our lex state. We really, really - # don't want to be using them and we really, really don't want to be - # exposing them as part of our public API. Unfortunately, we don't have - # another way of matching the exact tokens that the parser gem expects - # without them. We should find another way to do this, but in the - # meantime we'll hide them from the documentation and mark them as - # private constants. - EXPR_BEG = 0x1 - EXPR_LABEL = 0x400 - - # The `PARENTHESIS_LEFT` token in Prism is classified as either - # `tLPAREN` or `tLPAREN2` in the Parser gem. The following token types - # are listed as those classified as `tLPAREN`. - LPAREN_CONVERSION_TOKEN_TYPES = Set.new([ - :kAND, :kBEGIN, :kBREAK, :kCASE, :kDO_COND, :kDO_LAMBDA, :kDO, :kELSE, - :kELSIF, :kENSURE, :kFOR, :kIF_MOD, :kIF, :kIN, :kNEXT, :kOR, - :kRESCUE_MOD, :kRESCUE, :kRETURN, :kTHEN, :kUNLESS_MOD, :kUNLESS, - :kUNTIL_MOD, :kUNTIL, :kWHEN, :kWHILE_MOD, :kWHILE, - :tAMPER, :tANDOP, :tBANG, :tCARET, :tCOMMA, :tDIVIDE, :tDOT2, :tDOT3, - :tEQL, :tLCURLY, :tLPAREN_ARG, :tLPAREN, :tLPAREN2, :tLSHFT, :tNL, - :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS - ]) - # Types of tokens that are allowed to continue a method call with comments in-between. # For these, the parser gem doesn't emit a newline token after the last comment. COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT]) @@ -214,7 +195,7 @@ class Lexer # :nodoc: # Heredocs are complex and require us to keep track of a bit of info to refer to later HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true) - private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :LPAREN_CONVERSION_TOKEN_TYPES, :HeredocData + private_constant :TYPES, :HeredocData # The Parser::Source::Buffer that the tokens were lexed from. attr_reader :source_buffer @@ -253,7 +234,7 @@ def to_a comment_newline_location = nil while index < length - token, state = lexed[index] + token, _ = lexed[index] index += 1 next if TYPES_ALWAYS_SKIP.include?(token.type) @@ -324,10 +305,6 @@ def to_a value.chomp!(":") when :tLABEL_END value.chomp!(":") - when :tLCURLY - type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL - when :tLPAREN2 - type = :tLPAREN if tokens.empty? || LPAREN_CONVERSION_TOKEN_TYPES.include?(tokens.dig(-1, 0)) when :tNTH_REF value = parse_integer(value.delete_prefix("$")) when :tOP_ASGN @@ -506,10 +483,6 @@ def to_a type = :tIDENTIFIER end when :tXSTRING_BEG - if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type) - # self.`() - type = :tBACK_REF2 - end quote_stack.push(value) when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP diff --git a/src/prism.c b/src/prism.c index ec5a90752b..566be9e56d 100644 --- a/src/prism.c +++ b/src/prism.c @@ -10477,9 +10477,15 @@ parser_lex(pm_parser_t *parser) { // ( case '(': { + /* A parenthesis scanned at the beginning of an expression + * groups the expression it wraps, while one scanned in + * argument position with a preceding space wraps a command + * argument. Everything else opens an argument list. */ pm_token_type_t type = PM_TOKEN_PARENTHESIS_LEFT; - if (space_seen && (lex_state_arg_p(parser) || parser->lex_state == (PM_LEX_STATE_END | PM_LEX_STATE_LABEL))) { + if (lex_state_beg_p(parser)) { + type = PM_TOKEN_PARENTHESIS_LEFT_GROUPING; + } else if (space_seen && (lex_state_arg_p(parser) || parser->lex_state == (PM_LEX_STATE_END | PM_LEX_STATE_LABEL))) { type = PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES; } @@ -10538,24 +10544,28 @@ parser_lex(pm_parser_t *parser) { pm_token_type_t type = PM_TOKEN_BRACE_LEFT; if (parser->enclosure_nesting == parser->lambda_enclosure_nesting) { - // This { begins a lambda + /* This { begins a lambda */ parser->command_start = true; lex_state_set(parser, PM_LEX_STATE_BEG); type = PM_TOKEN_LAMBDA_BEGIN; } else if (lex_state_p(parser, PM_LEX_STATE_LABELED)) { - // This { begins a hash literal + /* This { begins a hash literal */ lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); + type = PM_TOKEN_BRACE_LEFT_HASH; } else if (lex_state_p(parser, PM_LEX_STATE_ARG_ANY | PM_LEX_STATE_END | PM_LEX_STATE_ENDFN)) { - // This { begins a block + /* This { begins a block */ parser->command_start = true; lex_state_set(parser, PM_LEX_STATE_BEG); } else if (lex_state_p(parser, PM_LEX_STATE_ENDARG)) { - // This { begins a block on a command + /* This { begins a block following a parenthesized + * command argument */ parser->command_start = true; lex_state_set(parser, PM_LEX_STATE_BEG); + type = PM_TOKEN_BRACE_LEFT_ARGUMENT; } else { - // This { begins a hash literal + /* This { begins a hash literal */ lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL); + type = PM_TOKEN_BRACE_LEFT_HASH; } parser->enclosure_nesting++; @@ -10872,7 +10882,7 @@ parser_lex(pm_parser_t *parser) { } lex_mode_push_string(parser, true, false, '\0', '`'); - LEX(PM_TOKEN_BACKTICK); + LEX(PM_TOKEN_XSTRING_BEGIN); } // single-quoted string literal @@ -12735,6 +12745,14 @@ match4(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4); } +/** + * Returns true if the current token is any of the five given types. + */ +static PRISM_INLINE bool +match5(const pm_parser_t *parser, pm_token_type_t type1, pm_token_type_t type2, pm_token_type_t type3, pm_token_type_t type4, pm_token_type_t type5) { + return match1(parser, type1) || match1(parser, type2) || match1(parser, type3) || match1(parser, type4) || match1(parser, type5); +} + /** * Returns true if the current token is any of the six given types. */ @@ -13555,7 +13573,7 @@ parse_targets(pm_parser_t *parser, pm_node_t *first_target, pm_binding_power_t b pm_node_t *splat = UP(pm_splat_node_create(parser, &star_operator, name)); pm_multi_target_node_targets_append(parser, result, splat); has_rest = true; - } else if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { + } else if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) { context_push(parser, PM_CONTEXT_MULTI_TARGET); pm_node_t *target = parse_expression(parser, binding_power, PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_EXPECT_EXPRESSION_AFTER_COMMA, (uint16_t) (depth + 1)); target = parse_target(parser, target, true, false); @@ -13766,7 +13784,7 @@ parse_assocs(pm_parser_t *parser, pm_static_literals_t *literals, pm_node_t *nod pm_token_t operator = parser->previous; pm_node_t *value = NULL; - if (match1(parser, PM_TOKEN_BRACE_LEFT)) { + if (match1(parser, PM_TOKEN_BRACE_LEFT_HASH)) { // If we're about to parse a nested hash that is being // pushed into this hash directly with **, then we want the // inner hash to share the static literals with the outer @@ -14319,7 +14337,7 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for */ static pm_multi_target_node_t * parse_required_destructured_parameter(pm_parser_t *parser) { - expect1(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_ERR_EXPECT_LPAREN_REQ_PARAMETER); + expect1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING, PM_ERR_EXPECT_LPAREN_REQ_PARAMETER); pm_multi_target_node_t *node = pm_multi_target_node_create(parser); pm_multi_target_node_opening_set(parser, node, &parser->previous); @@ -14338,7 +14356,7 @@ parse_required_destructured_parameter(pm_parser_t *parser) { break; } - if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { + if (match1(parser, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) { param = UP(parse_required_destructured_parameter(parser)); } else if (accept1(parser, PM_TOKEN_USTAR)) { pm_token_t star = parser->previous; @@ -14400,7 +14418,7 @@ static pm_parameters_order_t parameters_ordering[PM_TOKEN_MAXIMUM] = { [PM_TOKEN_AMPERSAND] = PM_PARAMETERS_ORDER_NOTHING_AFTER, [PM_TOKEN_UDOT_DOT_DOT] = PM_PARAMETERS_ORDER_NOTHING_AFTER, [PM_TOKEN_IDENTIFIER] = PM_PARAMETERS_ORDER_NAMED, - [PM_TOKEN_PARENTHESIS_LEFT] = PM_PARAMETERS_ORDER_NAMED, + [PM_TOKEN_PARENTHESIS_LEFT_GROUPING] = PM_PARAMETERS_ORDER_NAMED, [PM_TOKEN_EQUAL] = PM_PARAMETERS_ORDER_OPTIONAL, [PM_TOKEN_LABEL] = PM_PARAMETERS_ORDER_KEYWORDS, [PM_TOKEN_USTAR] = PM_PARAMETERS_ORDER_AFTER_OPTIONAL, @@ -14507,7 +14525,7 @@ parse_parameters( bool parsing = true; switch (parser->current.type) { - case PM_TOKEN_PARENTHESIS_LEFT: { + case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: { update_parameter_state(parser, &parser->current, &order); pm_node_t *param = UP(parse_required_destructured_parameter(parser)); @@ -15384,7 +15402,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) { * managed by the lexer. A `do`/`end` block is delimited by keywords, so we * push the frame here (covering the block parameters and body) and pop it * before consuming `end`, mirroring parse.y's `do_body` rule. */ - bool do_block = opening.type != PM_TOKEN_BRACE_LEFT; + bool do_block = opening.type != PM_TOKEN_BRACE_LEFT && opening.type != PM_TOKEN_BRACE_LEFT_ARGUMENT; if (do_block) pm_accepts_block_stack_push(parser, true); pm_parser_scope_push(parser, false); @@ -15409,7 +15427,7 @@ parse_block(pm_parser_t *parser, uint16_t depth) { accept1(parser, PM_TOKEN_NEWLINE); pm_node_t *statements = NULL; - if (opening.type == PM_TOKEN_BRACE_LEFT) { + if (!do_block) { if (!match1(parser, PM_TOKEN_BRACE_RIGHT)) { statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_BRACES, (uint16_t) (depth + 1))); } @@ -15506,7 +15524,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a * it, so pop the delimiter frame, push the command-args frame, and then * restore the delimiter frame on top (the delimiter's closing token * will pop it back off during argument parsing). */ - bool lookahead_delimiter = match4(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES, PM_TOKEN_BRACKET_LEFT, PM_TOKEN_BRACKET_LEFT_ARRAY); + bool lookahead_delimiter = match5(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES, PM_TOKEN_BRACKET_LEFT, PM_TOKEN_BRACKET_LEFT_ARRAY); if (lookahead_delimiter) pm_accepts_block_stack_pop(parser); pm_accepts_block_stack_push(parser, false); if (lookahead_delimiter) pm_accepts_block_stack_push(parser, true); @@ -15528,7 +15546,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a * it, pop the command-args frame beneath it, and restore the block * frame so the block's `}` still pops it. This mirrors the `tLBRACE_ARG` * lookahead handling in parse.y's `command_args` rule. */ - bool lookahead_brace = match1(parser, PM_TOKEN_BRACE_LEFT); + bool lookahead_brace = match2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT); if (lookahead_brace) pm_accepts_block_stack_pop(parser); pm_accepts_block_stack_pop(parser); if (lookahead_brace) pm_accepts_block_stack_push(parser, true); @@ -15540,7 +15558,7 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a if (full_arguments) { pm_block_node_t *block = NULL; - if (accept1(parser, PM_TOKEN_BRACE_LEFT)) { + if (accept2(parser, PM_TOKEN_BRACE_LEFT, PM_TOKEN_BRACE_LEFT_ARGUMENT)) { found |= true; block = parse_block(parser, (uint16_t) (depth + 1)); pm_arguments_validate_block(parser, arguments, block); @@ -16002,7 +16020,7 @@ parse_conditional(pm_parser_t *parser, pm_context_t context, size_t opening_newl #define PM_CASE_PRIMITIVE PM_TOKEN_INTEGER: case PM_TOKEN_INTEGER_IMAGINARY: case PM_TOKEN_INTEGER_RATIONAL: \ case PM_TOKEN_INTEGER_RATIONAL_IMAGINARY: case PM_TOKEN_FLOAT: case PM_TOKEN_FLOAT_IMAGINARY: \ case PM_TOKEN_FLOAT_RATIONAL: case PM_TOKEN_FLOAT_RATIONAL_IMAGINARY: case PM_TOKEN_SYMBOL_BEGIN: \ - case PM_TOKEN_REGEXP_BEGIN: case PM_TOKEN_BACKTICK: case PM_TOKEN_PERCENT_LOWER_X: case PM_TOKEN_PERCENT_LOWER_I: \ + case PM_TOKEN_REGEXP_BEGIN: case PM_TOKEN_XSTRING_BEGIN: case PM_TOKEN_PERCENT_LOWER_X: case PM_TOKEN_PERCENT_LOWER_I: \ case PM_TOKEN_PERCENT_LOWER_W: case PM_TOKEN_PERCENT_UPPER_I: case PM_TOKEN_PERCENT_UPPER_W: \ case PM_TOKEN_STRING_BEGIN: case PM_TOKEN_KEYWORD_NIL: case PM_TOKEN_KEYWORD_SELF: case PM_TOKEN_KEYWORD_TRUE: \ case PM_TOKEN_KEYWORD_FALSE: case PM_TOKEN_KEYWORD___FILE__: case PM_TOKEN_KEYWORD___LINE__: \ @@ -17345,7 +17363,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm pm_array_pattern_node_requireds_append(parser->arena, node, inner); return UP(node); } - case PM_TOKEN_BRACE_LEFT: { + case PM_TOKEN_BRACE_LEFT_HASH: { bool previous_pattern_matching_newlines = parser->pattern_matching_newlines; parser->pattern_matching_newlines = false; @@ -17482,7 +17500,7 @@ parse_pattern_primitive(pm_parser_t *parser, pm_constant_id_list_t *captures, pm return UP(pm_pinned_variable_node_create(parser, &operator, variable)); } - case PM_TOKEN_PARENTHESIS_LEFT: { + case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: { bool previous_pattern_matching_newlines = parser->pattern_matching_newlines; parser->pattern_matching_newlines = false; @@ -17566,7 +17584,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p switch (parser->current.type) { case PM_TOKEN_IDENTIFIER: case PM_TOKEN_BRACKET_LEFT_ARRAY: - case PM_TOKEN_BRACE_LEFT: + case PM_TOKEN_BRACE_LEFT_HASH: case PM_TOKEN_CARET: case PM_TOKEN_CONSTANT: case PM_TOKEN_UCOLON_COLON: @@ -17585,7 +17603,7 @@ parse_pattern_primitives(pm_parser_t *parser, pm_constant_id_list_t *captures, p break; } - case PM_TOKEN_PARENTHESIS_LEFT: + case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES: { pm_token_t operator = parser->previous; pm_token_t opening = parser->current; @@ -19166,6 +19184,13 @@ parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t /* If this is the end of the file or we match a right parenthesis, then we * have an empty parentheses node, and we can immediately return. */ if (match2(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_TOKEN_EOF)) { + /* A command argument group sets EXPR_ENDARG before its ')' is + * consumed, even when the group is empty, so that a following '{' is + * scanned as a block brace. */ + if (match1(parser, PM_TOKEN_PARENTHESIS_RIGHT) && opening.type == PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES) { + lex_state_set(parser, PM_LEX_STATE_ENDARG); + } + expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN); pop_block_exits(parser, previous_block_exits); return UP(pm_parentheses_node_create(parser, &opening, NULL, &parser->previous, paren_flags)); @@ -19486,10 +19511,10 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u return UP(array); } - case PM_TOKEN_PARENTHESIS_LEFT: + case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES: return parse_parentheses(parser, binding_power, flags, depth); - case PM_TOKEN_BRACE_LEFT: { + case PM_TOKEN_BRACE_LEFT_HASH: { // If we were passed a current_hash_keys via the parser, then that // means we're already parsing a hash and we want to share the set // of hash keys with this inner hash we're about to parse for the @@ -20109,7 +20134,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u context_push(parser, PM_CONTEXT_DEFINED); bool newline = accept1(parser, PM_TOKEN_NEWLINE); - if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { + if (accept2(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) { lparen = parser->previous; if (newline && accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { @@ -20278,7 +20303,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u accept1(parser, PM_TOKEN_NEWLINE); - if (accept1(parser, PM_TOKEN_PARENTHESIS_LEFT)) { + if (accept2(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_GROUPING)) { pm_token_t lparen = parser->previous; if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) { @@ -20599,7 +20624,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u pm_interpolated_regular_expression_node_closing_set(parser, interpolated, &closing); return UP(interpolated); } - case PM_TOKEN_BACKTICK: + case PM_TOKEN_XSTRING_BEGIN: case PM_TOKEN_PERCENT_LOWER_X: { parser_lex(parser); pm_token_t opening = parser->previous; diff --git a/templates/src/tokens.c.erb b/templates/src/tokens.c.erb index 472c82ea69..fb71afe217 100644 --- a/templates/src/tokens.c.erb +++ b/templates/src/tokens.c.erb @@ -53,6 +53,10 @@ pm_token_str(pm_token_type_t token_type) { return "'!~'"; case PM_TOKEN_BRACE_LEFT: return "'{'"; + case PM_TOKEN_BRACE_LEFT_ARGUMENT: + return "'{'"; + case PM_TOKEN_BRACE_LEFT_HASH: + return "'{'"; case PM_TOKEN_BRACE_RIGHT: return "'}'"; case PM_TOKEN_BRACKET_LEFT: @@ -275,6 +279,8 @@ pm_token_str(pm_token_type_t token_type) { return "numbered reference"; case PM_TOKEN_PARENTHESIS_LEFT: return "'('"; + case PM_TOKEN_PARENTHESIS_LEFT_GROUPING: + return "'('"; case PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES: return "'('"; case PM_TOKEN_PARENTHESIS_RIGHT: @@ -355,6 +361,8 @@ pm_token_str(pm_token_type_t token_type) { return "**"; case PM_TOKEN_WORDS_SEP: return "string separator"; + case PM_TOKEN_XSTRING_BEGIN: + return "backtick string literal"; case PM_TOKEN___END__: return "'__END__'"; case PM_TOKEN_MAXIMUM: diff --git a/test/prism/errors/command_calls_25.txt b/test/prism/errors/command_calls_25.txt index cf04508f87..c8769c538a 100644 --- a/test/prism/errors/command_calls_25.txt +++ b/test/prism/errors/command_calls_25.txt @@ -3,6 +3,7 @@ ^ expected a `do` keyword or a `{` to open the lambda block ^ unexpected ')', expecting end-of-input ^ unexpected ')', ignoring it - ^ unexpected end-of-input, assuming it is closing the parent top level context + ^ unexpected '{', ignoring it + ^ unexpected '}', ignoring it ^~ expected a lambda block beginning with `do` to end with `end` diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index 856ecedc1d..e44bc20d4d 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -111,12 +111,8 @@ class ParserTest < TestCase skip_tokens = [ "dash_heredocs.txt", "embdoc_no_newline_at_end.txt", - "methods.txt", - "seattlerb/bug169.txt", "seattlerb/case_in.txt", "seattlerb/difficult4__leading_dots2.txt", - "seattlerb/difficult6__7.txt", - "seattlerb/difficult6__8.txt", "seattlerb/heredoc_unicode.txt", "seattlerb/parse_line_heredoc.txt", "seattlerb/pct_w_heredoc_interp_nested.txt", @@ -129,14 +125,10 @@ class ParserTest < TestCase "whitequark/beginless_irange_after_newline.txt", "whitequark/forward_arg_with_open_args.txt", "whitequark/kwarg_no_paren.txt", - "whitequark/lbrace_arg_after_command_args.txt", "whitequark/multiple_pattern_matches.txt", "whitequark/newline_in_hash_argument.txt", "whitequark/pattern_matching_hash.txt", - "whitequark/ruby_bug_14690.txt", - "whitequark/ruby_bug_9669.txt", - "whitequark/space_args_arg_block.txt", - "whitequark/space_args_block.txt" + "whitequark/ruby_bug_9669.txt" ] Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|