[pull] master from ruby:master

ext/erb/escape/escape.c

@@ -77,7 +77,7 @@ find_next_basic(search_state *search)
 
 #ifdef HAVE_SIMD_SSE2
 
-static inline int trailing_zeros(int input)
+static inline int trailing_zeros32(int input)
 {
     RUBY_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior
 
@@ -97,9 +97,9 @@ static inline int trailing_zeros(int input)
 static inline bool
 find_next_match_sse2(search_state *search)
 {
-    int next_match_offset = trailing_zeros(search->matches_bitmap);
-    search->matches_bitmap >>= (next_match_offset + 1);
-    search->cstr += next_match_offset;
+    uint32_t trailing_zeros = trailing_zeros32(search->matches_bitmap);
+    search->matches_bitmap >>= trailing_zeros;
+    search->cstr += trailing_zeros;
     RUBY_ASSERT(search->cstr <= search->end);
     return true;
 }
@@ -170,14 +170,10 @@ find_next_match_neon(search_state *search)
     uint32_t trailing_zeros = trailing_zeros64(search->matches_bitmap);
 
     // uint64_t >>= 64 is undefined behaviour
-    if (trailing_zeros >= 63) {
-        search->matches_bitmap = 0;
-        search->cstr += 15;
-    }
-    else {
-        search->matches_bitmap >>= (trailing_zeros + 1);
-        search->cstr += trailing_zeros / 4;
-    }
+    RUBY_ASSERT(trailing_zeros < 64);
+    search->matches_bitmap >>= trailing_zeros;
+    search->cstr += trailing_zeros / 4;
+    RUBY_ASSERT(search->cstr <= search->end);
     return true;
 }
 
@@ -223,6 +219,15 @@ find_next_neon(search_state *search)
 #define find_next find_next_neon
 #endif // HAVE_SIMD_NEON
 
+static inline void
+consume_match(search_state *search)
+{
+#ifdef HAVE_SIMD
+    search->matches_bitmap >>= 1;
+#endif
+    search->cstr++;
+}
+
 #ifndef find_next
 #define find_next find_next_basic
 #endif
@@ -242,9 +247,8 @@ optimized_escape_html(VALUE str)
 
     while (find_next(&search)) {
         const unsigned char c = *search.cstr;
-        size_t segment_len = search.cstr - segment_start;
-        search.cstr++;
 
+        size_t segment_len = search.cstr - segment_start;
         if (!buf) {
             buf = ALLOCV_N(char, vbuf, escaped_length(str));
             dest = buf;
@@ -253,7 +257,6 @@ optimized_escape_html(VALUE str)
             memcpy(dest, segment_start, segment_len);
             dest += segment_len;
         }
-        segment_start = search.cstr;
 
         switch(c) {
             #define HTML_ESCAPE(c, str) \
@@ -272,6 +275,8 @@ optimized_escape_html(VALUE str)
 
             #undef HTML_ESCAPE
         }
+        consume_match(&search);
+        segment_start = search.cstr;
     }
 
     VALUE escaped = str;

lib/prism.rb

@@ -112,14 +112,14 @@ def self.find(callable)
   #    def self.parse_success?:      (String source,  ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
   #    def self.parse_failure?:      (String source,  ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
   #    def self.parse_stream:        (_Stream stream, ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
-  #    def self.parse_file:          (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
-  #    def self.profile_file:        (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
-  #    def self.lex_file:            (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
-  #    def self.parse_lex_file:      (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
-  #    def self.dump_file:           (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
-  #    def self.parse_file_comments: (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
-  #    def self.parse_file_success?: (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
-  #    def self.parse_file_failure?: (String filepath,                   ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_file:          (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult
+  #    def self.profile_file:        (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void
+  #    def self.lex_file:            (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult
+  #    def self.parse_lex_file:      (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult
+  #    def self.dump_file:           (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String
+  #    def self.parse_file_comments: (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment]
+  #    def self.parse_file_success?: (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
+  #    def self.parse_file_failure?: (path filepath,                     ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool
 end
 
 require_relative "prism/polyfill/byteindex"

lib/prism/ffi.rb

@@ -214,7 +214,12 @@ def self.with_string(string)
       end
 
       # Yields a PrismSource to the given block, backed by a pm_source_t.
-      def self.with_file(filepath)
+      def self.with_file(filepath, options)
+        unless filepath.is_a?(String)
+          filepath = filepath.to_path if filepath.respond_to?(:to_path)
+          filepath = filepath.to_str if filepath.respond_to?(:to_str)
+        end
+        options[:filepath] = filepath
         raise TypeError unless filepath.is_a?(String)
 
         # On Windows and Mac, it's expected that filepaths will be encoded in
@@ -265,8 +270,7 @@ def dump(source, **options)
 
     # Mirror the Prism.dump_file API by using the serialization API.
     def dump_file(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| dump_common(string, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| dump_common(string, options) }
     end
 
     # Mirror the Prism.lex API by using the serialization API.
@@ -276,8 +280,7 @@ def lex(code, **options)
 
     # Mirror the Prism.lex_file API by using the serialization API.
     def lex_file(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| lex_common(string, string.read, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| lex_common(string, string.read, options) }
     end
 
     # Mirror the Prism.parse API by using the serialization API.
@@ -289,8 +292,7 @@ def parse(code, **options)
     # native strings instead of Ruby strings because it allows us to use mmap
     # when it is available.
     def parse_file(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| parse_common(string, string.read, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_common(string, string.read, options) }
     end
 
     # Mirror the Prism.parse_stream API by using the serialization API.
@@ -349,8 +351,7 @@ def parse_comments(code, **options)
     # API. This uses native strings instead of Ruby strings because it allows us
     # to use mmap when it is available.
     def parse_file_comments(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| parse_comments_common(string, string.read, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_comments_common(string, string.read, options) }
     end
 
     # Mirror the Prism.parse_lex API by using the serialization API.
@@ -360,8 +361,7 @@ def parse_lex(code, **options)
 
     # Mirror the Prism.parse_lex_file API by using the serialization API.
     def parse_lex_file(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| parse_lex_common(string, string.read, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_lex_common(string, string.read, options) }
     end
 
     # Mirror the Prism.parse_success? API by using the serialization API.
@@ -376,8 +376,7 @@ def parse_failure?(code, **options)
 
     # Mirror the Prism.parse_file_success? API by using the serialization API.
     def parse_file_success?(filepath, **options)
-      options[:filepath] = filepath
-      LibRubyParser::PrismSource.with_file(filepath) { |string| parse_file_success_common(string, options) }
+      LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_file_success_common(string, options) }
     end
 
     # Mirror the Prism.parse_file_failure? API by using the serialization API.
@@ -401,9 +400,7 @@ def profile(source, **options)
 
     # Mirror the Prism.profile_file API by using the serialization API.
     def profile_file(filepath, **options)
-      LibRubyParser::PrismSource.with_file(filepath) do |string|
-        options[:filepath] = filepath
-
+      LibRubyParser::PrismSource.with_file(filepath, options) do |string|
         if (format_type = raise_error_format_type(options))
           raise_error(string, options, format_type)
         end

lib/prism/translation/parser/compiler.rb

@@ -165,13 +165,13 @@ def visit_assoc_node(node)
           else
             parts =
               if key.is_a?(SymbolNode)
-                value_loc = key.value_loc
-                if value_loc.nil?
+                value = key.value
+                if value == ""
                   []
-                elsif value_loc.slice.include?("\n")
-                  string_nodes_from_line_continuations(key.unescaped, value_loc.slice, value_loc.start_offset, key.opening)
+                elsif value.include?("\n")
+                  string_nodes_from_line_continuations(key.unescaped, value, key.value_loc.start_offset, key.opening)
                 else
-                  [builder.string_internal([key.unescaped, srange(value_loc)])]
+                  [builder.string_internal([key.unescaped, srange(key.value_loc)])]
                 end
               else
                 visit_all(key.parts)
@@ -1775,7 +1775,7 @@ def visit_symbol_node(node)
             end
           else
             parts =
-              if node.value_loc.nil?
+              if node.value == ""
                 []
               elsif node.value.include?("\n")
                 string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening)

lib/prism/translation/ripper.rb

@@ -3840,12 +3840,14 @@ def visit_super_node(node)
       # ^^^^
       def visit_symbol_node(node)
         with_string_bounds(node) do
-          if node.value_loc.nil?
-            bounds(node.location)
-            on_dyna_symbol(on_string_content)
-          elsif (opening = node.opening)&.match?(/^%s|['"]:?$/)
+          if (opening = node.opening)&.match?(/^%s|['"]:?$/)
             bounds(node.value_loc)
-            content = on_string_add(on_string_content, on_tstring_content(node.value))
+            content = on_string_content
+
+            if !(value = node.value).empty?
+              content = on_string_add(content, on_tstring_content(value))
+            end
+
             bounds(node.location)
             on_dyna_symbol(content)
           elsif (closing = node.closing) == ":"

prism/config.yml

@@ -4553,7 +4553,7 @@ nodes:
       - name: opening_loc
         type: location?
       - name: value_loc
-        type: location?
+        type: location
       - name: closing_loc
         type: location?
       - name: unescaped

prism/extension.c

@@ -393,9 +393,11 @@ file_options(int argc, VALUE *argv, pm_options_t *options, VALUE *encoded_filepa
     VALUE keywords;
     rb_scan_args(argc, argv, "1:", &filepath, &keywords);
 
-    if (!RB_TYPE_P(filepath, T_STRING)) {
+    int state = 0;
+    filepath = rb_protect(rb_get_path, filepath, &state);
+    if (state != 0) {
         pm_options_free(options);
-        rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(filepath));
+        rb_jump_tag(state);
     }
 
     *encoded_filepath = rb_str_encode_ospath(filepath);

test/erb/test_erb_escape.rb

@@ -58,6 +58,85 @@ def test_simd_coverage
     assert_equal '&lt;' * 32, h('<' * 32)
   end
 
+  def test_html_escape_simd_block_boundary
+    # Ensure we only escape the characters that need to be escaped.
+    (0...128).each do |pos|
+      s = "a" * 128
+      s[pos] = "<"
+      expected = "a" * pos + "&lt;" + "a" * (128 - pos - 1)
+      assert_equal(expected, ERB::Util.html_escape(s), "escape at position #{pos}")
+    end
+  end
+
+  HTML_ESCAPE_ENTITIES = {"'" => "&#39;", '"' => "&quot;", "&" => "&amp;", "<" => "&lt;", ">" => "&gt;"}
+
+  def test_html_escape_simd_multiple_matches_per_block
+    chars = ["'", '"', '&', '<', '>']
+    (0..15).each do |a|
+      (0..15).each do |b|
+        next if a == b
+        s = "a" * 32
+        s[a] = chars[a % chars.size]
+        s[b] = chars[b % chars.size]
+        expected = Array.new(32, "a")
+        expected[a] = HTML_ESCAPE_ENTITIES[chars[a % chars.size]]
+        expected[b] = HTML_ESCAPE_ENTITIES[chars[b % chars.size]]
+        assert_equal(expected.join, ERB::Util.html_escape(s), "positions #{a}, #{b}")
+      end
+    end
+  end
+
+  def test_html_escape_simd_tail_lengths
+    (1..128).each do |len|
+      (0...len).each do |pos|
+        s = "a" * len
+        s[pos] = ">"
+        expected = "a" * pos + "&gt;" + "a" * (len - pos - 1)
+        assert_equal(expected, ERB::Util.html_escape(s), "len=#{len} pos=#{pos}")
+      end
+    end
+  end
+
+  def test_html_escape_simd_wide_block_boundary
+    # Ensure a 64-byte-wide SIMD fast path correctly locates a match
+    # at every byte position, including the last byte of the block
+    # (which is special-cased in find_next_match_neon).
+    (0...128).each do |pos|
+      s = "a" * 128
+      s[pos] = "<"
+      expected = "a" * pos + "&lt;" + "a" * (128 - pos - 1)
+      assert_equal(expected, ERB::Util.html_escape(s), "escape at position #{pos}")
+    end
+  end
+
+  def test_html_escape_simd_wide_block_multiple_matches
+    chars = ["'", '"', '&', '<', '>']
+    boundary_positions = [0, 1, 15, 16, 17, 31, 32, 33, 47, 48, 49, 62, 63]
+    boundary_positions.each do |a|
+      boundary_positions.each do |b|
+        next if a == b
+        s = "a" * 64
+        s[a] = chars[a % chars.size]
+        s[b] = chars[b % chars.size]
+        expected = Array.new(64, "a")
+        expected[a] = HTML_ESCAPE_ENTITIES[chars[a % chars.size]]
+        expected[b] = HTML_ESCAPE_ENTITIES[chars[b % chars.size]]
+        assert_equal(expected.join, ERB::Util.html_escape(s), "positions #{a}, #{b}")
+      end
+    end
+  end
+
+  def test_html_escape_simd_wide_block_tail_lengths
+    ([*56..72] + [*120..136]).each do |len|
+      (0...len).each do |pos|
+        s = "a" * len
+        s[pos] = ">"
+        expected = "a" * pos + "&gt;" + "a" * (len - pos - 1)
+        assert_equal(expected, ERB::Util.html_escape(s), "len=#{len} pos=#{pos}")
+      end
+    end
+  end
+
   private
 
   def h(...)

test/prism/api/parse_test.rb

@@ -69,6 +69,12 @@ def test_parse_tempfile
       end
     end
 
+    def test_parse_pathname
+      pathname = Pathname.new(__FILE__)
+      node = Prism.parse_file(pathname).value
+      assert_kind_of ProgramNode, node
+    end
+
     if RUBY_ENGINE != "truffleruby"
       def test_parse_nonascii
         Dir.mktmpdir do |dir|

vm_trace.c

@@ -92,6 +92,9 @@ static void clean_hooks(rb_hook_list_t *list);
 void
 rb_hook_list_free(rb_hook_list_t *hooks)
 {
+    for (rb_event_hook_t *hook = hooks->hooks; hook; hook = hook->next) {
+        hook->hook_flags |= RUBY_EVENT_HOOK_FLAG_DELETED;
+    }
     hooks->need_clean = true;
 
     if (hooks->running == 0) {