Split some tokens

config.yml

@@ -377,7 +377,7 @@ tokens:
   - name: AMPERSAND_EQUAL
     comment: "&="
   - name: BACKTICK
-    comment: "`"
+    comment: "` as a method name"
   - name: BACK_REFERENCE
     comment: "a back reference"
   - name: BANG
@@ -388,6 +388,10 @@ tokens:
     comment: "!~"
   - name: BRACE_LEFT
     comment: "{"
+  - name: BRACE_LEFT_ARGUMENT
+    comment: "{ for a block following a parenthesized argument"
+  - name: BRACE_LEFT_HASH
+    comment: "{ for a hash literal"
   - name: BRACKET_LEFT
     comment: "["
   - name: BRACKET_LEFT_ARRAY
@@ -584,6 +588,8 @@ tokens:
     comment: "a numbered reference to a capture group in the previous regular expression match"
   - name: PARENTHESIS_LEFT
     comment: "("
+  - name: PARENTHESIS_LEFT_GROUPING
+    comment: "( scanned at the beginning of an expression"
   - name: PARENTHESIS_LEFT_PARENTHESES
     comment: "( for a parentheses node"
   - name: PERCENT
@@ -658,6 +664,8 @@ tokens:
     comment: "unary **"
   - name: WORDS_SEP
     comment: "a separator between words in a list"
+  - name: XSTRING_BEGIN
+    comment: "the beginning of an execution string"
   - name: __END__
     comment: "marker for the point in the file at which the parser should stop"
 flags:

lib/prism/lex_compat.rb

@@ -78,6 +78,8 @@ def deconstruct_keys(keys) # :nodoc:
       BANG_EQUAL: :on_op,
       BANG_TILDE: :on_op,
       BRACE_LEFT: :on_lbrace,
+      BRACE_LEFT_ARGUMENT: :on_lbrace,
+      BRACE_LEFT_HASH: :on_lbrace,
       BRACE_RIGHT: :on_rbrace,
       BRACKET_LEFT: :on_lbracket,
       BRACKET_LEFT_ARRAY: :on_lbracket,
@@ -191,6 +193,7 @@ def deconstruct_keys(keys) # :nodoc:
       NEWLINE: :on_nl,
       NUMBERED_REFERENCE: :on_backref,
       PARENTHESIS_LEFT: :on_lparen,
+      PARENTHESIS_LEFT_GROUPING: :on_lparen,
       PARENTHESIS_LEFT_PARENTHESES: :on_lparen,
       PARENTHESIS_RIGHT: :on_rparen,
       PERCENT: :on_op,
@@ -233,6 +236,7 @@ def deconstruct_keys(keys) # :nodoc:
       USTAR: :on_op,
       USTAR_STAR: :on_op,
       WORDS_SEP: :on_words_sep,
+      XSTRING_BEGIN: :on_backtick,
       "__END__": :on___end__
     }.freeze
 

lib/prism/translation/parser/lexer.rb

@@ -28,11 +28,13 @@ class Lexer # :nodoc:
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tXSTRING_BEG,
+          BACKTICK: :tBACK_REF2,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
           BRACE_LEFT: :tLCURLY,
+          BRACE_LEFT_ARGUMENT: :tLBRACE_ARG,
+          BRACE_LEFT_HASH: :tLBRACE,
           BRACE_RIGHT: :tRCURLY,
           BRACKET_LEFT: :tLBRACK2,
           BRACKET_LEFT_ARRAY: :tLBRACK,
@@ -141,6 +143,7 @@ class Lexer # :nodoc:
           NEWLINE: :tNL,
           NUMBERED_REFERENCE: :tNTH_REF,
           PARENTHESIS_LEFT: :tLPAREN2,
+          PARENTHESIS_LEFT_GROUPING: :tLPAREN,
           PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG,
           PARENTHESIS_RIGHT: :tRPAREN,
           PERCENT: :tPERCENT,
@@ -180,32 +183,10 @@ class Lexer # :nodoc:
           UPLUS: :tUPLUS,
           USTAR: :tSTAR,
           USTAR_STAR: :tDSTAR,
-          WORDS_SEP: :tSPACE
+          WORDS_SEP: :tSPACE,
+          XSTRING_BEGIN: :tXSTRING_BEG
         }
 
-        # These constants represent flags in our lex state. We really, really
-        # don't want to be using them and we really, really don't want to be
-        # exposing them as part of our public API. Unfortunately, we don't have
-        # another way of matching the exact tokens that the parser gem expects
-        # without them. We should find another way to do this, but in the
-        # meantime we'll hide them from the documentation and mark them as
-        # private constants.
-        EXPR_BEG = 0x1
-        EXPR_LABEL = 0x400
-
-        # The `PARENTHESIS_LEFT` token in Prism is classified as either
-        # `tLPAREN` or `tLPAREN2` in the Parser gem. The following token types
-        # are listed as those classified as `tLPAREN`.
-        LPAREN_CONVERSION_TOKEN_TYPES = Set.new([
-          :kAND, :kBEGIN, :kBREAK, :kCASE, :kDO_COND, :kDO_LAMBDA, :kDO, :kELSE,
-          :kELSIF, :kENSURE, :kFOR, :kIF_MOD, :kIF, :kIN, :kNEXT, :kOR,
-          :kRESCUE_MOD, :kRESCUE, :kRETURN, :kTHEN, :kUNLESS_MOD, :kUNLESS,
-          :kUNTIL_MOD, :kUNTIL, :kWHEN, :kWHILE_MOD, :kWHILE,
-          :tAMPER, :tANDOP, :tBANG, :tCARET, :tCOMMA, :tDIVIDE, :tDOT2, :tDOT3,
-          :tEQL, :tLCURLY, :tLPAREN_ARG, :tLPAREN, :tLPAREN2, :tLSHFT, :tNL,
-          :tOP_ASGN, :tOROP, :tPIPE, :tSEMI, :tSTRING_DBEG, :tUMINUS, :tUPLUS
-        ])
-
         # Types of tokens that are allowed to continue a method call with comments in-between.
         # For these, the parser gem doesn't emit a newline token after the last comment.
         COMMENT_CONTINUATION_TYPES = Set.new([:COMMENT, :AMPERSAND_DOT, :DOT])
@@ -214,7 +195,7 @@ class Lexer # :nodoc:
         # Heredocs are complex and require us to keep track of a bit of info to refer to later
         HeredocData = Struct.new(:identifier, :common_whitespace, keyword_init: true)
 
-        private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL, :LPAREN_CONVERSION_TOKEN_TYPES, :HeredocData
+        private_constant :TYPES, :HeredocData
 
         # The Parser::Source::Buffer that the tokens were lexed from.
         attr_reader :source_buffer
@@ -253,7 +234,7 @@ def to_a
           comment_newline_location = nil
 
           while index < length
-            token, state = lexed[index]
+            token, _ = lexed[index]
             index += 1
             next if TYPES_ALWAYS_SKIP.include?(token.type)
 
@@ -324,10 +305,6 @@ def to_a
               value.chomp!(":")
             when :tLABEL_END
               value.chomp!(":")
-            when :tLCURLY
-              type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL
-            when :tLPAREN2
-              type = :tLPAREN if tokens.empty? || LPAREN_CONVERSION_TOKEN_TYPES.include?(tokens.dig(-1, 0))
             when :tNTH_REF
               value = parse_integer(value.delete_prefix("$"))
             when :tOP_ASGN
@@ -506,10 +483,6 @@ def to_a
                 type = :tIDENTIFIER
               end
             when :tXSTRING_BEG
-              if (next_token = lexed[index]&.first) && !%i[STRING_CONTENT STRING_END EMBEXPR_BEGIN].include?(next_token.type)
-                # self.`()
-                type = :tBACK_REF2
-              end
               quote_stack.push(value)
             when :tSYMBOLS_BEG, :tQSYMBOLS_BEG, :tWORDS_BEG, :tQWORDS_BEG
               if (next_token = lexed[index]&.first) && next_token.type == :WORDS_SEP