From f791417e718460b84b46c3b8cba9a4158d5b308c Mon Sep 17 00:00:00 2001 From: nozomemein Date: Fri, 24 Jul 2026 07:41:22 +0900 Subject: [PATCH 1/8] ZJIT: Specialize caller splats with monomorphic lengths Use the profiled splat array length to expand caller splats into fixed positional arguments for SendDirect. Guard the runtime length and fall back to the original Send when it changes or ruby2_keywords semantics apply. Validate direct sends using the expanded argument count, then apply the existing keyword setup and rest parameter packing. Move the ruby2_keywords splat predicate into shared JIT code so both YJIT and ZJIT can use it. --- jit.c | 13 + yjit.c | 13 - yjit/bindgen/src/main.rs | 2 +- yjit/src/codegen.rs | 4 +- yjit/src/cruby_bindings.inc.rs | 2 +- zjit.rb | 1 + zjit/bindgen/src/main.rs | 1 + zjit/src/codegen_tests.rs | 90 +++++- zjit/src/cruby.rs | 1 + zjit/src/cruby_bindings.inc.rs | 1 + zjit/src/hir.rs | 284 +++++++++++++++++-- zjit/src/hir/opt_tests.rs | 485 ++++++++++++++++++++++++++++++++- zjit/src/stats.rs | 3 + 13 files changed, 842 insertions(+), 58 deletions(-) diff --git a/jit.c b/jit.c index 21a16a156e488c..484c7a63a9e35c 100644 --- a/jit.c +++ b/jit.c @@ -562,6 +562,19 @@ rb_jit_array_len(VALUE a) return rb_array_len(a); } +// Return non-zero when `obj` is an array and its last item is a +// `ruby2_keywords` hash. The JITs don't support this kind of splat. +size_t +rb_jit_ruby2_keywords_splat_p(VALUE obj) +{ + if (!RB_TYPE_P(obj, T_ARRAY)) return 0; + long len = RARRAY_LEN(obj); + if (len == 0) return 0; + VALUE last = RARRAY_AREF(obj, len - 1); + if (!RB_TYPE_P(last, T_HASH)) return 0; + return FL_TEST_RAW(last, RHASH_PASS_AS_KEYWORDS); +} + void rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) { diff --git a/yjit.c b/yjit.c index 2b6f1110275362..d59bfaa38108ee 100644 --- a/yjit.c +++ b/yjit.c @@ -242,19 +242,6 @@ rb_yjit_rb_ary_subseq_length(VALUE ary, long beg) return rb_ary_subseq(ary, beg, len); } -// Return non-zero when `obj` is an array and its last item is a -// `ruby2_keywords` hash. We don't support this kind of splat. -size_t -rb_yjit_ruby2_keywords_splat_p(VALUE obj) -{ - if (!RB_TYPE_P(obj, T_ARRAY)) return 0; - long len = RARRAY_LEN(obj); - if (len == 0) return 0; - VALUE last = RARRAY_AREF(obj, len - 1); - if (!RB_TYPE_P(last, T_HASH)) return 0; - return FL_TEST_RAW(last, RHASH_PASS_AS_KEYWORDS); -} - // Checks to establish preconditions for rb_yjit_splat_varg_cfunc() VALUE rb_yjit_splat_varg_checks(VALUE *sp, VALUE splat_array, rb_control_frame_t *cfp) diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 28afb79144f7fc..ff9e587484a1e3 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -371,7 +371,7 @@ fn main() { .allowlist_function("rb_yarv_str_eql_internal") .allowlist_function("rb_str_neq_internal") .allowlist_function("rb_yarv_ary_entry_internal") - .allowlist_function("rb_yjit_ruby2_keywords_splat_p") + .allowlist_function("rb_jit_ruby2_keywords_splat_p") .allowlist_function("rb_jit_fix_div_fix") .allowlist_function("rb_jit_fix_mod_fix") .allowlist_function("rb_FL_TEST") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index dffb4593c09202..a1ee87e2f29e9e 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -7031,7 +7031,7 @@ fn gen_send_cfunc( if variable_splat { let splat_array_idx = i32::from(kw_splat) + i32::from(block_arg); let comptime_splat_array = jit.peek_at_stack(&asm.ctx, splat_array_idx as isize); - if unsafe { rb_yjit_ruby2_keywords_splat_p(comptime_splat_array) } != 0 { + if unsafe { rb_jit_ruby2_keywords_splat_p(comptime_splat_array) } != 0 { gen_counter_incr(jit, asm, Counter::send_cfunc_splat_varg_ruby2_keywords); return None; } @@ -7932,7 +7932,7 @@ fn gen_send_iseq( // All splats need to guard for ruby2_keywords hash. Check with a function call when // splatting into a rest param since the index for the last item in the array is dynamic. asm_comment!(asm, "guard no ruby2_keywords hash in splat"); - let bad_splat = asm.ccall(rb_yjit_ruby2_keywords_splat_p as _, vec![asm.stack_opnd(splat_pos)]); + let bad_splat = asm.ccall(rb_jit_ruby2_keywords_splat_p as _, vec![asm.stack_opnd(splat_pos)]); asm.cmp(bad_splat, 0.into()); asm.jnz(Target::side_exit(Counter::guard_send_splatarray_last_ruby2_keywords)); } diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 143eef16e28ed8..a5302a9a56b17d 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1260,7 +1260,6 @@ extern "C" { pub fn rb_str_neq_internal(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_ary_unshift_m(argc: ::std::os::raw::c_int, argv: *mut VALUE, ary: VALUE) -> VALUE; pub fn rb_yjit_rb_ary_subseq_length(ary: VALUE, beg: ::std::os::raw::c_long) -> VALUE; - pub fn rb_yjit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_yjit_splat_varg_checks( sp: *mut VALUE, splat_array: VALUE, @@ -1386,6 +1385,7 @@ extern "C" { pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_jit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; diff --git a/zjit.rb b/zjit.rb index 3b52211eef3735..f2f2742e5ae234 100644 --- a/zjit.rb +++ b/zjit.rb @@ -137,6 +137,7 @@ def stats_string :empty_inline_frame_count, :non_variadic_cfunc_optimized_send_count, :variadic_cfunc_optimized_send_count, + :caller_splat_optimized, ], buf:, stats:, right_align: true, base: :send_count) print_counters([ :dynamic_setivar_count, diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index fe25d56f081a0a..00d169f924fb4e 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -313,6 +313,7 @@ fn main() { .allowlist_function("rb_jit_mark_unused") .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_jit_array_len") + .allowlist_function("rb_jit_ruby2_keywords_splat_p") .allowlist_function("rb_jit_fix_div_fix") .allowlist_function("rb_jit_iseq_builtin_attrs") .allowlist_function("rb_jit_str_concat_codepoint") diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 0fd00630c1952a..50990cb14be938 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -7123,13 +7123,95 @@ fn test_send_on_heap_object_in_spilled_arg() { } #[test] -fn test_send_splat() { - assert_snapshot!(inspect(" +fn test_send_caller_splat_arguments() { + eval(" def test(a, b) = [a, b] - def entry(arr) = test(*arr) + def entry(args) = test(*args) entry([1, 2]) + "); + assert_snapshot!(assert_compiles("entry([1, 2])"), @"[1, 2]"); +} + +#[test] +fn test_send_empty_caller_splat_arguments() { + eval(" + def test(a = 1) = a + def entry(args) = test(*args) + entry([]) + "); + assert_snapshot!(assert_compiles("entry([])"), @"1"); +} + +#[test] +fn test_send_caller_splat_arguments_with_positional_prefix() { + eval(" + def test(a, b, c) = [a, b, c] + def entry(args) = test(1, *args) + entry([2, 3]) + "); + assert_snapshot!(assert_compiles("entry([2, 3])"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_many_caller_splat_arguments_to_rest_parameter() { + eval(" + def test(*args) = args.length + def entry(args) = test(*args) + entry([1, 2, 3, 4, 5, 6, 7]) + "); + assert_snapshot!(assert_compiles("entry([1, 2, 3, 4, 5, 6, 7])"), @"7"); +} + +#[test] +fn test_send_caller_splat_arguments_to_complex_parameters() { + eval(" + def test(a, b = 2, *rest, z, k: 40) = [a, b, rest, z, k] + def entry(args) = test(1, *args) + entry([3, 4, 5]) + "); + assert_snapshot!(assert_compiles("entry([3, 4, 5])"), @"[1, 3, [4], 5, 40]"); +} + +#[test] +fn test_send_caller_splat_arguments_with_required_keyword() { + eval(" + def test(*args, k:) = [args, k] + def entry(args) = test(*args, k: 40) entry([1, 2]) - "), @"[1, 2]"); + "); + assert_snapshot!(assert_compiles("entry([1, 2])"), @"[[1, 2], 40]"); +} + +#[test] +fn test_send_caller_splat_arguments_with_block_literal() { + eval(" + def test(*args) = yield args.length + def entry(args) = test(*args) { |n| n + 4 } + entry([1, 2, 3]) + "); + assert_snapshot!(assert_compiles("entry([1, 2, 3])"), @"7"); +} + +#[test] +fn test_send_caller_splat_length_mismatch_falls_back() { + eval(" + def test(*args) = args + def entry(args) = test(*args) + entry([1, 2]) + "); + assert_snapshot!(assert_compiles("entry([1, 2, 3])"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_caller_splat_with_ruby2_keywords_hash_falls_back() { + eval(" + def capture(*args) = args + ruby2_keywords(:capture) + def test(arg = :default, k: nil) = [arg, k] + def entry(args) = test(*args) + entry(capture(k: 1)) + "); + assert_snapshot!(assert_compiles("entry(capture(k: 1))"), @"[:default, 1]"); } #[test] diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index d4e6955bededdd..e144f21a7a17d9 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1736,6 +1736,7 @@ pub(crate) mod ids { name: aref content: b"[]" name: rb_obj_is_proc name: rb_ivar_get_at_no_ractor_check + name: rb_jit_ruby2_keywords_splat_p name: RUBY_FL_FREEZE name: RUBY_ELTS_SHARED name: RubyVM diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 4bb6e5bc8a38ab..fd4f085023f10e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2566,6 +2566,7 @@ unsafe extern "C" { pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_jit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 44c5da93f37dab..14f7c936ede873 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -15,7 +15,7 @@ use std::{ use crate::hir_type::{Type, types}; use crate::hir_effect::{Effect, abstract_heaps, effects}; use crate::bitset::BitSet; -use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::profile::{ProfiledType, SplatLength, SplatLengthDistributionSummary, TypeDistributionSummary}; use crate::stats::{Counter, incr_counter}; use SendFallbackReason::*; @@ -2631,13 +2631,13 @@ pub enum ValidationError { } /// Check if we can emit SendDirect to the given ISEQ with the given arguments. -fn can_direct_send(iseq: *const rb_iseq_t, ci: *const rb_callinfo, args: &[InsnId], has_block: bool) -> Result<(), SendDirectFailure> { +fn can_direct_send(iseq: *const rb_iseq_t, caller_args: &CallerArguments, has_block: bool, caller_splat: Option) -> Result<(), SendDirectFailure> { let mut complex_arg_counters = vec![]; let mut count_failure = |counter| complex_arg_counters.push(counter); let params = unsafe { iseq.params() }; let callee_has_block_param = 0 != params.flags.has_block(); - let caller_passes_block_arg = has_block && (unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) != 0; + let caller_passes_block_arg = has_block && (caller_args.flags & VM_CALL_ARGS_BLOCKARG) != 0; use Counter::*; if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } @@ -2668,15 +2668,20 @@ fn can_direct_send(iseq: *const rb_iseq_t, ci: *const rb_callinfo, args: &[InsnI let keyword = params.keyword; let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; - let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_kw_count = caller_args.kwarg_count; let has_rest = 0 != params.flags.has_rest(); - let caller_positional = match args.len().checked_sub(caller_kw_count) { + let caller_positional = match caller_args.original.len().checked_sub(caller_kw_count) { Some(count) => count, None => { return Err(SendDirectFailure::new(ArgcParamMismatch)); } }; + // A caller splat occupies one argument slot before expansion. Replace that + // slot with its profiled length to get the effective positional argument count. + let caller_positional = match caller_splat { + None => caller_positional, + Some(splat) => caller_positional - 1 + splat.length as usize, + }; // Match vm_args.c's setup_parameters_complex via args_kw_argv_to_hash: // keywords passed to a method with no keyword parameters can become one @@ -2848,10 +2853,64 @@ struct SendDirectArgs { jit_entry_idx: u16, } +/// Caller Arguments as they appear on the original Send instruction. +struct CallerArguments<'a> { + /// Argument values in the order stored by the original Send. + original: &'a [InsnId], + /// Call-site flags from the Send's callinfo. + flags: u32, + /// Explicit keyword metadata, or null when the caller has no keywords. + kwarg: *const rb_callinfo_kwarg, + /// Number of explicit keyword values at the end of `original`. + kwarg_count: usize, + /// Index of the caller splat array, when VM_CALL_ARGS_SPLAT is set. + splat_arg_idx: Option, +} + +impl<'a> CallerArguments<'a> { + /// Decode callinfo metadata and locate the splat in the original Send arguments. + /// Do this once per Send so builds for different splat lengths share the same layout. + fn new(original: &'a [InsnId], ci: *const rb_callinfo) -> Self { + let flags = unsafe { rb_vm_ci_flag(ci) }; + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let kwarg_count = if kwarg.is_null() { + 0 + } else { + (unsafe { get_cikw_keyword_len(kwarg) }) as usize + }; + let splat_arg_idx = if flags & VM_CALL_ARGS_SPLAT != 0 { + // The splat array is the final positional operand, before explicit keyword values. + Some(original.len() - kwarg_count - 1) + } else { + None + }; + + Self { original, flags, kwarg, kwarg_count, splat_arg_idx } + } +} + +/// Caller splat expansion selected for one SendDirect path. +#[derive(Clone, Copy)] +struct CallerSplat { + /// Index of the splat array in the original Send argument vector. + arg_idx: usize, + /// HIR value that produces the splat array at runtime. + array: InsnId, + /// Profiled array length handled by this path. + length: SplatLength, +} + /// One SendDirect argument before its HIR value is materialized. enum SendDirectArg { /// A HIR value already present in the original Send argument vector. Existing(InsnId), + /// An element to load from the caller splat array on the selected path. + SplatElement { + /// HIR value that produces the splat array. + array: InsnId, + /// Zero-based index of the element to load. + index: SplatLength, + }, /// A Ruby value to materialize as a Const instruction on the selected path. Constant(VALUE), /// Explicit caller keywords to materialize as one positional Hash. @@ -3828,11 +3887,19 @@ impl Function { } } + /// Return the caller splat length profile at the given Snapshot, if available. + /// These are historical observations, so specialized paths must still check + /// the runtime array length before expanding the splat. + fn profiled_splat_length_summary_at(&self, state: InsnId) -> Option { + let state = self.frame_state(state); + get_or_create_iseq_payload(state.iseq).profile.get_splat_length_summary(state.insn_idx) + } + /// Validate and normalize SendDirect arguments without emitting HIR. - fn build_send_direct_args(&self, args: &[InsnId], ci: *const rb_callinfo, iseq: IseqPtr, has_block: bool) -> Result { - can_direct_send(iseq, ci, args, has_block)?; - let args = args.iter().copied().map(SendDirectArg::Existing).collect(); - let (args, kw_bits) = Self::plan_send_direct_keyword_arguments(args, ci, iseq) + fn build_send_direct_args(&self, caller_args: &CallerArguments, caller_splat: Option, iseq: IseqPtr, has_block: bool) -> Result { + can_direct_send(iseq, caller_args, has_block, caller_splat)?; + let args = Self::expand_caller_splat_args(caller_args, caller_splat); + let (args, kw_bits) = Self::plan_send_direct_keyword_arguments(args, caller_args, iseq) .map_err(SendDirectFailure::new)?; let (args, jit_entry_idx) = Self::plan_send_direct_rest_parameter(args, iseq) .map_err(SendDirectFailure::new)?; @@ -3871,6 +3938,10 @@ impl Function { fn emit_send_direct_arg(&mut self, block: BlockId, arg: SendDirectArg, state: InsnId) -> InsnId { match arg { SendDirectArg::Existing(value) => value, + SendDirectArg::SplatElement { array, index } => { + let index = self.push_insn(block, Insn::Const { val: Const::CInt64(i64::from(index)) }); + self.push_insn(block, Insn::ArrayAref { array, index }) + } SendDirectArg::Constant(value) => { self.push_insn(block, Insn::Const { val: Const::Value(value) }) } @@ -3891,6 +3962,94 @@ impl Function { } } + /// Expand the caller splat for the selected length without emitting ArrayAref. + /// Match vm_args.c's setup_parameters_complex: VM_CALL_ARGS_SPLAT stores the + /// array separately and argument setup consumes its elements as positional args. + fn expand_caller_splat_args(caller_args: &CallerArguments, caller_splat: Option) -> Vec { + let Some(splat) = caller_splat else { + return caller_args.original.iter().copied().map(SendDirectArg::Existing).collect(); + }; + + let mut args = Vec::with_capacity(caller_args.original.len() - 1 + splat.length as usize); + args.extend(caller_args.original[..splat.arg_idx].iter().copied().map(SendDirectArg::Existing)); + args.extend((0..splat.length).map(|index| SendDirectArg::SplatElement { array: splat.array, index })); + args.extend(caller_args.original[splat.arg_idx + 1..].iter().copied().map(SendDirectArg::Existing)); + args + } + + /// Dispatch a caller splat to a fixed-length direct path, falling back to the + /// original Send when its length differs or ruby2_keywords semantics apply. + fn dispatch_caller_splat( + &mut self, + block: BlockId, + caller_splat: CallerSplat, + optimized_block: BlockId, + optimized_result: InsnId, + send: &Insn, + state: InsnId, + ) -> (BlockId, InsnId) { + // The fixed-length direct path and VM fallback produce the result of the + // same Send, so route them through one join block. + let insn_idx = self.frame_state(state).insn_idx() as u32; + let fallback_block = self.new_block(insn_idx); + let join_block = self.new_block(insn_idx); + let join_param = self.push_insn(join_block, Insn::Param); + let edge = |target| BranchEdge { target, args: vec![] }; + + // Compare the runtime array length with the profiled length before + // entering the path that expands the array for SendDirect. + let length = self.push_insn(block, Insn::ArrayLength { array: caller_splat.array }); + let expected = self.push_insn(block, Insn::Const { val: Const::CInt64(i64::from(caller_splat.length)) }); + let length_matches = self.push_insn(block, Insn::IsBitEqual { left: length, right: expected }); + + // An empty splat cannot end in a ruby2_keywords hash, so skip + // that runtime check when the profiled length is zero. + if caller_splat.length == 0 { + self.push_insn(block, Insn::CondBranch { + val: length_matches, + if_true: edge(optimized_block), + if_false: edge(fallback_block), + }); + } else { + let ruby2_keywords_block = self.new_block(insn_idx); + self.push_insn(block, Insn::CondBranch { + val: length_matches, + if_true: edge(ruby2_keywords_block), + if_false: edge(fallback_block), + }); + + // A ruby2_keywords hash changes how the VM interprets the final splat + // element, so only expand arrays that preserve positional semantics. + let ruby2_keywords_splat = self.push_insn(ruby2_keywords_block, Insn::CCall { + cfunc: rb_jit_ruby2_keywords_splat_p as *const u8, + recv: caller_splat.array, + args: vec![], + name: ID!(rb_jit_ruby2_keywords_splat_p), + owner: Qnil, + return_type: types::CInt64, + elidable: false, + }); + let zero = self.push_insn(ruby2_keywords_block, Insn::Const { val: Const::CInt64(0) }); + let is_positional = self.push_insn(ruby2_keywords_block, Insn::IsBitEqual { left: ruby2_keywords_splat, right: zero }); + self.push_insn(ruby2_keywords_block, Insn::CondBranch { + val: is_positional, + if_true: edge(optimized_block), + if_false: edge(fallback_block), + }); + } + + // Preserve the original splat Send on the fallback path so VM argument + // setup handles lengths and keyword conversion that SendDirect cannot. + self.count(fallback_block, Counter::complex_arg_pass_caller_splat); + let fallback_result = self.push_insn(fallback_block, send.clone()); + self.set_dynamic_send_reason(fallback_result, ComplexArgPass); + self.push_insn(fallback_block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); + + self.push_insn(optimized_block, Insn::Jump(BranchEdge { target: join_block, args: vec![optimized_result] })); + + (join_block, join_param) + } + /// Reorder keyword arguments to match the callee's expected order, and synthesize /// default values for any optional keywords not provided by the caller. /// @@ -3902,10 +4061,10 @@ impl Function { /// (used by checkkeyword to determine if non-constant defaults need evaluation) fn plan_send_direct_keyword_arguments( args: Vec, - ci: *const rb_callinfo, + caller_args: &CallerArguments, iseq: IseqPtr, ) -> Result<(Vec, u32), SendFallbackReason> { - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let kwarg = caller_args.kwarg; let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; if callee_keyword.is_null() { if kwarg.is_null() { @@ -3914,8 +4073,7 @@ impl Function { } let params = unsafe { iseq.params() }; - let ci_flags = unsafe { rb_vm_ci_flag(ci) }; - if ci_flags & VM_CALL_KW_SPLAT != 0 { + if caller_args.flags & VM_CALL_KW_SPLAT != 0 { // Caller **kw is one runtime Hash, not explicit keyword slots, so // there is no static key/value list to repack here. return Err(SendDirectKeywordMismatch); @@ -4477,9 +4635,10 @@ impl Function { /// opens the door for inlining. /// Also try and inline constant caches, specialize object allocations, and more. fn type_specialize(&mut self) { - for block in self.reverse_post_order() { - let old_insns = std::mem::take(&mut self.blocks[block].insns); - assert!(self.blocks[block].insns.is_empty()); + for original_block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[original_block].insns); + assert!(self.blocks[original_block].insns.is_empty()); + let mut block = original_block; for insn_id in old_insns { let resolved = self.resolve(insn_id); match resolved.insn(self) { @@ -4487,7 +4646,8 @@ impl Function { self.try_rewrite_freeze(block, insn_id, recv, state), &Insn::Send { recv, block: None, ref args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => self.try_rewrite_uminus(block, insn_id, recv, state), - &Insn::Send { mut recv, cd, state, block: send_block, .. } => { + &Insn::Send { mut recv, cd, state, block: send_block, reason, .. } => { + let send = resolved.insn(self).clone(); let mut has_block = send_block.is_some(); let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), state) { ReceiverTypeResolution::StaticallyKnown { class } => (class, None), @@ -4598,7 +4758,12 @@ impl Function { // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendDirect`. // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) and C methods handle their own argument constraints (e.g., kw_splat for Proc call). // Mask out ARGS_BLOCKARG only if we've already handled the nil block arg case above. - let flags_for_check = if stripped_nil_block { flags & !VM_CALL_ARGS_BLOCKARG } else { flags }; + let mut flags_for_check = if stripped_nil_block { flags & !VM_CALL_ARGS_BLOCKARG } else { flags }; + if def_type == VM_METHOD_TYPE_ISEQ { + // Caller splat specialization currently only supports ISEQ callees, so + // skip the generic splat rejection here and validate its profile below. + flags_for_check &= !VM_CALL_ARGS_SPLAT; + } if def_type != VM_METHOD_TYPE_OPTIMIZED && def_type != VM_METHOD_TYPE_CFUNC && unspecializable_call_type(flags_for_check) { self.count_complex_call_features(block, flags, state); self.set_dynamic_send_reason(insn_id, ComplexArgPass); @@ -4610,30 +4775,91 @@ impl Function { // Only specialize positional-positional calls // TODO(max): Handle other kinds of parameter passing let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - let Ok(call) = self.build_send_direct_args(&args, ci, iseq, has_block) + let caller_args = CallerArguments::new(&args, ci); + let caller_splat = if let Some(arg_idx) = caller_args.splat_arg_idx { + // A classified caller-splat Send is a dynamic fallback retained from + // an earlier specialization pass. Do not dispatch it again. + if !matches!(reason, Uncategorized(_)) { + self.push_insn_id(block, insn_id); continue; + } + // Count the profile shape for every caller-splat execution; + // complex_arg_pass_caller_splat separately tracks fallbacks. + self.count_caller_splat_profile(block, state); + // Expand a caller splat only when profiling observed one stable + // array length; otherwise keep the original dynamic Send. + // TODO: Support polymorphic caller-splat length profiles. + let profiled_length = match self.profiled_splat_length_summary_at(state) { + Some(summary) if summary.is_monomorphic() => summary.bucket(0), + Some(_) | None => None, + }; + let Some(length) = profiled_length else { + self.count(block, Counter::complex_arg_pass_caller_splat); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + }; + Some(CallerSplat { + arg_idx, + array: caller_args.original[arg_idx], + length, + }) + } else { + None + }; + let Ok(call) = self.build_send_direct_args(&caller_args, caller_splat, iseq, has_block) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Send)) else { self.push_insn_id(block, insn_id); continue; }; + // Keep splat expansion and optimized-send guards on the path selected by + // the runtime length check. Calls without a splat use the current block. + let optimized_block = if caller_splat.is_some() { + let insn_idx = self.frame_state(state).insn_idx() as u32; + self.new_block(insn_idx) + } else { + block + }; + // Check singleton class assumption first, before emitting other patchpoints - if !self.assume_no_singleton_classes(block, klass, state) { + if !self.assume_no_singleton_classes(optimized_block, klass, state) { + if caller_splat.is_some() { + self.remove_block(optimized_block); + } self.set_dynamic_send_reason(insn_id, SingletonClassSeen); self.push_insn_id(block, insn_id); continue; } + if caller_splat.is_some() { + // Count caller-splat executions that take this optimized path. + // This is a feature-specific counter, not part of optimized_send_count. + self.count(optimized_block, Counter::caller_splat_optimized); + } + // Add PatchPoint for method redefinition - self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + self.push_insn(optimized_block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); // Add GuardType for profiled receiver if let Some(profiled_type) = profiled_type { - recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); + recv = self.push_insn(optimized_block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); self.insn_types[recv] = self.infer_type(recv); } let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = - self.emit_send_direct_args(block, call, &args, send_frame_state); - let replacement = self.try_inline_send_direct(block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))); - self.make_equal_to(insn_id, replacement); + self.emit_send_direct_args(optimized_block, call, &args, send_frame_state); + let replacement = self.try_inline_send_direct(optimized_block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))); + if let Some(caller_splat) = caller_splat { + let (join_block, join_param) = self.dispatch_caller_splat( + block, + caller_splat, + optimized_block, + replacement, + &send, + state, + ); + self.make_equal_to(insn_id, join_param); + block = join_block; + } else { + self.make_equal_to(insn_id, replacement); + } } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; let proc = unsafe { rb_jit_get_proc_ptr(procv) }; @@ -4647,7 +4873,8 @@ impl Function { let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - let Ok(call) = self.build_send_direct_args(&args, ci, iseq, has_block) + let caller_args = CallerArguments::new(&args, ci); + let Ok(call) = self.build_send_direct_args(&caller_args, None, iseq, has_block) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Send)) else { self.push_insn_id(block, insn_id); continue; }; @@ -5195,7 +5422,8 @@ impl Function { // If not, we can't do direct dispatch. let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; // TODO: pass Option to build_send_direct_args when we start specializing `super { ... }`. - let Ok(call) = self.build_send_direct_args(&args, ci, super_iseq, false) + let caller_args = CallerArguments::new(&args, ci); + let Ok(call) = self.build_send_direct_args(&caller_args, None, super_iseq, false) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Super)) else { self.push_insn_id(block, insn_id); continue; }; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index d2adf0478684c6..6db62787eb7dcd 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -14670,8 +14670,23 @@ mod hir_opt_tests { v49:NilClass = Const Value(nil) v13:ArrayExact = NewArray v19:ArrayExact = ToArray v13 - v21:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing - v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v54:CInt64 = ArrayLength v19 + v55:CInt64[0] = Const CInt64(0) + v56:CBool = IsBitEqual v54, v55 + CondBranch v56, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v50:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + PushInlineFrame :foo, v50 (0x1038), num_args=0 + PatchPoint MethodRedefined(Object@0x1000, itself@0x1060, cme:0x1068) + CheckInterrupts + PopInlineFrame + Jump bb6(v50) + bb5(): + v58:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing + Jump bb6(v58) + bb6(v53:BasicObject): + v25:StringExact[VALUE(0x1090)] = Const Value(VALUE(0x1090)) v26:StringExact = StringCopy v25 PatchPoint NoEPEscape(test) v31:ArrayExact = ToArray v13 @@ -14685,13 +14700,13 @@ mod hir_opt_tests { } #[test] - fn dont_specialize_call_to_iseq_with_monomorphic_caller_splat() { + fn inline_call_to_iseq_with_monomorphic_caller_splat() { enable_zjit_stats(); eval(" - def foo(*args) = args + def foo(a, b) = [a, b] def test(args) = foo(*args) - test([1]) - test([2]) + test([1, 2]) + test([3, 4]) "); assert_snapshot!(hir_string("test"), @" fn test@:3: @@ -14714,9 +14729,461 @@ mod hir_opt_tests { IncrCounter zjit_insn_count v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_monomorphic + v41:CInt64 = ArrayLength v20 + v42:CInt64[2] = Const CInt64(2) + v43:CBool = IsBitEqual v41, v42 + CondBranch v43, bb7(), bb5() + bb7(): + v45:CInt64 = CCall v20, :rb_jit_ruby2_keywords_splat_p@0x1001 + v46:CInt64[0] = Const CInt64(0) + v47:CBool = IsBitEqual v45, v46 + CondBranch v47, bb4(), bb5() + bb4(): + IncrCounter caller_splat_optimized + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v33:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v10, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v34:CInt64[0] = Const CInt64(0) + v35:BasicObject = ArrayAref v20, v34 + v36:CInt64[1] = Const CInt64(1) + v37:BasicObject = ArrayAref v20, v36 + PushInlineFrame :foo, v33 (0x1040), num_args=2 + IncrCounter inline_iseq_optimized_send_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v65:ArrayExact = NewArray v35, v37 + IncrCounter zjit_insn_count + CheckInterrupts + PopInlineFrame + Jump bb6(v65) + bb5(): + IncrCounter complex_arg_pass_caller_splat + v50:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing + Jump bb6(v50) + bb6(v40:BasicObject): + IncrCounter zjit_insn_count + CheckInterrupts + Return v40 + "); + } + + #[test] + fn specialize_call_to_iseq_with_monomorphic_caller_splat() { + eval(" + def foo(arg) = arg + 1 + def test(args) = foo(*args) + test([1]) + test([2]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v31:CInt64 = ArrayLength v16 + v32:CInt64[1] = Const CInt64(1) + v33:CBool = IsBitEqual v31, v32 + CondBranch v33, bb7(), bb5() + bb7(): + v35:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v36:CInt64[0] = Const CInt64(0) + v37:CBool = IsBitEqual v35, v36 + CondBranch v37, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v26:CInt64[0] = Const CInt64(0) + v27:BasicObject = ArrayAref v16, v26 + PushInlineFrame :foo, v25 (0x1040), num_args=1 + v49:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) + v63:Fixnum = GuardType v27, Fixnum recompile + v64:Fixnum = FixnumAdd v63, v49 + CheckInterrupts + PopInlineFrame + Jump bb6(v64) + bb5(): + v39:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v39) + bb6(v30:BasicObject): + CheckInterrupts + Return v30 + "); + } + + #[test] + fn specialize_call_to_iseq_with_empty_caller_splat() { + eval(" + def foo(arg = 1) = arg + def test(args) = foo(*args) + test([]) + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v29:CInt64 = ArrayLength v16 + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v29, v30 + CondBranch v31, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + PushInlineFrame :foo, v25 (0x1040), num_args=0 + v41:Fixnum[1] = Const Value(1) + CheckInterrupts + PopInlineFrame + Jump bb6(v41) + bb5(): + v33:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v33) + bb6(v28:BasicObject): + CheckInterrupts + Return v28 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_positional_prefix() { + eval(" + def foo(a, b, c) = [a, b, c] + def test(args) = foo(1, *args) + test([2, 3]) + test([4, 5]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:ArrayExact = ToArray v10 + v35:CInt64 = ArrayLength v18 + v36:CInt64[2] = Const CInt64(2) + v37:CBool = IsBitEqual v35, v36 + CondBranch v37, bb7(), bb5() + bb7(): + v39:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v40:CInt64[0] = Const CInt64(0) + v41:CBool = IsBitEqual v39, v40 + CondBranch v41, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v28:CInt64[0] = Const CInt64(0) + v29:BasicObject = ArrayAref v18, v28 + v30:CInt64[1] = Const CInt64(1) + v31:BasicObject = ArrayAref v18, v30 + PushInlineFrame :foo, v27 (0x1040), num_args=3 + v57:ArrayExact = NewArray v15, v29, v31 + CheckInterrupts + PopInlineFrame + Jump bb6(v57) + bb5(): + v43:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing + Jump bb6(v43) + bb6(v34:BasicObject): + CheckInterrupts + Return v34 + "); + } + + #[test] + fn specialize_call_to_iseq_with_many_caller_splat_arguments_and_rest_parameter() { + eval(" + def foo(*args) = args.length + def test(args) = foo(*args) + test([1, 2, 3, 4, 5, 6, 7]) + test([8, 9, 10, 11, 12, 13, 14]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v44:CInt64 = ArrayLength v16 + v45:CInt64[7] = Const CInt64(7) + v46:CBool = IsBitEqual v44, v45 + CondBranch v46, bb7(), bb5() + bb7(): + v48:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v48, v49 + CondBranch v50, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v26:CInt64[0] = Const CInt64(0) + v27:BasicObject = ArrayAref v16, v26 + v28:CInt64[1] = Const CInt64(1) + v29:BasicObject = ArrayAref v16, v28 + v30:CInt64[2] = Const CInt64(2) + v31:BasicObject = ArrayAref v16, v30 + v32:CInt64[3] = Const CInt64(3) + v33:BasicObject = ArrayAref v16, v32 + v34:CInt64[4] = Const CInt64(4) + v35:BasicObject = ArrayAref v16, v34 + v36:CInt64[5] = Const CInt64(5) + v37:BasicObject = ArrayAref v16, v36 + v38:CInt64[6] = Const CInt64(6) + v39:BasicObject = ArrayAref v16, v38 + v40:ArrayExact = NewArray v27, v29, v31, v33, v35, v37, v39 + PushInlineFrame :foo, v25 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1068) + PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) + v76:CInt64 = ArrayLength v40 + v77:Fixnum = BoxFixnum v76 + CheckInterrupts + PopInlineFrame + Jump bb6(v77) + bb5(): + v52:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v52) + bb6(v43:BasicObject): + CheckInterrupts + Return v43 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_complex_parameters() { + eval(" + def foo(a, b = 2, *rest, z, k: 40) = [a, b, rest, z, k] + def test(args) = foo(1, *args) + test([3, 4, 5]) + test([6, 7, 8]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:ArrayExact = ToArray v10 + v39:CInt64 = ArrayLength v18 + v40:CInt64[3] = Const CInt64(3) + v41:CBool = IsBitEqual v39, v40 + CondBranch v41, bb7(), bb5() + bb7(): + v43:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v44:CInt64[0] = Const CInt64(0) + v45:CBool = IsBitEqual v43, v44 + CondBranch v45, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v28:CInt64[0] = Const CInt64(0) + v29:BasicObject = ArrayAref v18, v28 + v30:CInt64[1] = Const CInt64(1) + v31:BasicObject = ArrayAref v18, v30 + v32:ArrayExact = NewArray v31 + v33:CInt64[2] = Const CInt64(2) + v34:BasicObject = ArrayAref v18, v33 + v35:Fixnum[40] = Const Value(40) + v71:Fixnum[0] = Const Value(0) + PushInlineFrame :foo, v27 (0x1040), num_args=5 + v66:ArrayExact = NewArray v15, v29, v32, v34, v35 + CheckInterrupts + PopInlineFrame + Jump bb6(v66) + bb5(): + v47:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing + Jump bb6(v47) + bb6(v38:BasicObject): + CheckInterrupts + Return v38 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_caller_splat_and_required_keyword() { + enable_zjit_stats(); + eval(" + def foo(*args, k:) = [args, k] + def test(args) = foo(*args, k: 40) + test([1, 2]) + test([3, 4]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v6, v7) + bb3(v10:BasicObject, v11:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v20:ArrayExact = ToArray v11 + IncrCounter zjit_insn_count + v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_monomorphic - v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter complex_arg_pass_caller_kw_splat + v26:BasicObject = Send v10, :foo, v20, v23 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v26 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_block_literal() { + eval(" + def foo(*args) = yield args.length + def test(args) = foo(*args) { |n| n + 4 } + test([1, 2, 3]) + test([4, 5, 6]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v38:CInt64 = ArrayLength v16 + v39:CInt64[3] = Const CInt64(3) + v40:CBool = IsBitEqual v38, v39 + CondBranch v40, bb7(), bb5() + bb7(): + v42:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v43:CInt64[0] = Const CInt64(0) + v44:CBool = IsBitEqual v42, v43 + CondBranch v44, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v28:CInt64[0] = Const CInt64(0) + v29:BasicObject = ArrayAref v16, v28 + v30:CInt64[1] = Const CInt64(1) + v31:BasicObject = ArrayAref v16, v30 + v32:CInt64[2] = Const CInt64(2) + v33:BasicObject = ArrayAref v16, v32 + v34:ArrayExact = NewArray v29, v31, v33 + PushInlineFrame :foo, v27 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1068) + PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) + v76:CInt64 = ArrayLength v34 + v77:Fixnum = BoxFixnum v76 + v59:CPtr = GetEP 0 + v60:CInt64 = LoadField v59, :VM_ENV_DATA_INDEX_SPECVAL@0x10a0 + v61:CInt64[-4] = Const CInt64(-4) + v62:CInt64 = IntAnd v60, v61 + v63:BasicObject = InvokeBlockIseqDirect (0x10a8), v62, v77 + CheckInterrupts + PopInlineFrame + Jump bb6(v63) + bb5(): + v46:BasicObject = Send v9, 0x10a8, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v46) + bb6(v37:BasicObject): + PatchPoint NoEPEscape(test) + CheckInterrupts + Return v37 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_monomorphic_caller_splat_argc_mismatch() { + enable_zjit_stats(); + eval(" + def foo(a, b) = [a, b] + def test(args) = foo(*args) + test([1]) rescue nil + test([2]) rescue nil + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v6, v7) + bb3(v10:BasicObject, v11:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v20:ArrayExact = ToArray v11 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_monomorphic + IncrCounter send_direct_fallback_context_send + v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Argument count does not match parameter count IncrCounter zjit_insn_count CheckInterrupts Return v24 @@ -14755,9 +15222,9 @@ mod hir_opt_tests { IncrCounter zjit_insn_count v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count - IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_polymorphic - v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter complex_arg_pass_caller_splat + v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts Return v24 diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index c4476db2d9cc40..31f15d8e0f8632 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -442,6 +442,9 @@ make_counters! { caller_splat_profile_megamorphic, caller_splat_profile_skewed_megamorphic, + // Caller splat specialization + caller_splat_optimized, + // Contexts in which SendDirect argument planning failed. These are kept // outside dynamic_send because the detailed fallback reason is also counted. send_direct_fallback_context_send, From ba2a5dc2ff1f095177d830309c5b3cd47f92a92c Mon Sep 17 00:00:00 2001 From: nozomemein Date: Mon, 24 Aug 2026 09:40:24 +0900 Subject: [PATCH 2/8] ZJIT: Fix type inference panic after caller-splat dispatch Caller-splat dispatch creates a join result during type_specialize. A following CFunc inline may use it before infer_types runs again, causing ZJIT to infer the type of an untyped Param and panic. Mark the join result as BasicObject and add regression tests for using it with Hash#[]=. --- zjit/src/codegen_tests.rs | 13 ++++++++ zjit/src/hir.rs | 3 ++ zjit/src/hir/opt_tests.rs | 64 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 50990cb14be938..42769ab6c00c95 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -7214,6 +7214,19 @@ fn test_send_caller_splat_with_ruby2_keywords_hash_falls_back() { assert_snapshot!(assert_compiles("entry(capture(k: 1))"), @"[:default, 1]"); } +#[test] +fn test_send_caller_splat_result_used_by_hash_aset() { + eval(" + def test(value) = value + def entry(args) + hash = {} + hash[:value] = test(*args) + end + entry([1]) + "); + assert_snapshot!(assert_compiles("entry([2])"), @"2"); +} + #[test] fn test_send_kwarg() { assert_snapshot!(inspect(" diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 14f7c936ede873..46455fa963f555 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -3994,6 +3994,9 @@ impl Function { let fallback_block = self.new_block(insn_idx); let join_block = self.new_block(insn_idx); let join_param = self.push_insn(join_block, Insn::Param); + // The join result may be used later in this type_specialize pass, before + // infer_types runs again. Both branches produce a Ruby value. + self.insn_types[join_param.to_usize()] = types::BasicObject; let edge = |target| BranchEdge { target, args: vec![] }; // Compare the runtime array length with the profiled length before diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 6db62787eb7dcd..e9d479caa3040d 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -15231,6 +15231,70 @@ mod hir_opt_tests { "); } + #[test] + fn specialize_call_to_iseq_with_caller_splat_result_used_by_hash_aset() { + // Hash#[]= returns its value argument from its CFunc inline. Ensure it can + // consume the caller-splat join result before infer_types runs again. + eval(" + def target(value) = value + def test(args) + hash = {} + hash[:value] = target(*args) + end + test([1]) + test([2]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + v4:NilClass = Const Value(nil) + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + v9:NilClass = Const Value(nil) + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + v17:HashExact = NewHash + PatchPoint NoEPEscape(test) + v23:NilClass = Const Value(nil) + v26:StaticSymbol[:value] = Const Value(VALUE(0x1008)) + v30:ArrayExact = ToArray v12 + v50:CInt64 = ArrayLength v30 + v51:CInt64[1] = Const CInt64(1) + v52:CBool = IsBitEqual v50, v51 + CondBranch v52, bb7(), bb5() + bb7(): + v54:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 + v55:CInt64[0] = Const CInt64(0) + v56:CBool = IsBitEqual v54, v55 + CondBranch v56, bb4(), bb5() + bb4(): + PatchPoint MethodRedefined(Object@0x1018, target@0x1020, cme:0x1028) + v44:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile + v45:CInt64[0] = Const CInt64(0) + v46:BasicObject = ArrayAref v30, v45 + PushInlineFrame :target, v44 (0x1050), num_args=1 + CheckInterrupts + PopInlineFrame + Jump bb6(v46) + bb5(): + v58:BasicObject = Send v11, :target, v30 # SendFallbackReason: Complex argument passing + Jump bb6(v58) + bb6(v49:BasicObject): + PatchPoint NoSingletonClass(Hash@0x1078) + PatchPoint MethodRedefined(Hash@0x1078, []=@0x1080, cme:0x1088) + HashAset v17, v26, v49 + CheckInterrupts + Return v49 + "); + } + #[test] fn test_inline_symbol_to_sym() { eval(r#" From c04e133d6dda765b0465c67995f8ad9d3256b1e2 Mon Sep 17 00:00:00 2001 From: nozomemein Date: Tue, 25 Aug 2026 08:44:46 +0900 Subject: [PATCH 3/8] ZJIT: Build caller-splat direct paths during dispatch Move receiver guards, argument emission, and SendDirect generation into a callback invoked by dispatch_caller_splat. This keeps caller-splat CFG construction together and allows adding one optimized path per profiled length for polymorphic dispatch. --- zjit/src/hir.rs | 72 ++++---- zjit/src/hir/opt_tests.rs | 336 +++++++++++++++++++------------------- 2 files changed, 206 insertions(+), 202 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 46455fa963f555..8943101a3ecc17 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -3977,16 +3977,17 @@ impl Function { args } - /// Dispatch a caller splat to a fixed-length direct path, falling back to the - /// original Send when its length differs or ruby2_keywords semantics apply. + /// Dispatch a Send with a caller splat between a fixed-length direct path and + /// the original Send fallback. The callback generates the direct path after + /// the runtime length and ruby2_keywords checks have been connected. fn dispatch_caller_splat( &mut self, block: BlockId, caller_splat: CallerSplat, optimized_block: BlockId, - optimized_result: InsnId, send: &Insn, state: InsnId, + emit_optimized: impl FnOnce(&mut Function, BlockId) -> InsnId, ) -> (BlockId, InsnId) { // The fixed-length direct path and VM fallback produce the result of the // same Send, so route them through one join block. @@ -4041,6 +4042,10 @@ impl Function { }); } + // Generate the direct path only in the block selected by the checks above. + let optimized_result = emit_optimized(self, optimized_block); + self.push_insn(optimized_block, Insn::Jump(BranchEdge { target: join_block, args: vec![optimized_result] })); + // Preserve the original splat Send on the fallback path so VM argument // setup handles lengths and keyword conversion that SendDirect cannot. self.count(fallback_block, Counter::complex_arg_pass_caller_splat); @@ -4048,8 +4053,6 @@ impl Function { self.set_dynamic_send_reason(fallback_result, ComplexArgPass); self.push_insn(fallback_block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); - self.push_insn(optimized_block, Insn::Jump(BranchEdge { target: join_block, args: vec![optimized_result] })); - (join_block, join_param) } @@ -4813,8 +4816,9 @@ impl Function { self.push_insn_id(block, insn_id); continue; }; - // Keep splat expansion and optimized-send guards on the path selected by - // the runtime length check. Calls without a splat use the current block. + // Start the direct path in a detached block so a failed singleton-class + // assumption can discard it before caller-splat dispatch connects the CFG. + // Calls without a splat use the current block directly. let optimized_block = if caller_splat.is_some() { let insn_idx = self.frame_state(state).insn_idx() as u32; self.new_block(insn_idx) @@ -4831,38 +4835,38 @@ impl Function { self.push_insn_id(block, insn_id); continue; } - if caller_splat.is_some() { - // Count caller-splat executions that take this optimized path. - // This is a feature-specific counter, not part of optimized_send_count. - self.count(optimized_block, Counter::caller_splat_optimized); - } + let emit_optimized = move |function: &mut Function, optimized_block: BlockId| { + if caller_splat.is_some() { + // Count caller-splat executions that take this optimized path. + // This is a feature-specific counter, not part of optimized_send_count. + function.count(optimized_block, Counter::caller_splat_optimized); + } - // Add PatchPoint for method redefinition - self.push_insn(optimized_block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + // Add PatchPoint for method redefinition + function.push_insn(optimized_block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); - // Add GuardType for profiled receiver - if let Some(profiled_type) = profiled_type { - recv = self.push_insn(optimized_block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); - self.insn_types[recv] = self.infer_type(recv); - } + // Add GuardType for profiled receiver + let recv = if let Some(profiled_type) = profiled_type { + let recv = function.push_insn(optimized_block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); + function.insn_types[recv] = function.infer_type(recv); + recv + } else { + recv + }; - let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = - self.emit_send_direct_args(optimized_block, call, &args, send_frame_state); - let replacement = self.try_inline_send_direct(optimized_block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))); - if let Some(caller_splat) = caller_splat { - let (join_block, join_param) = self.dispatch_caller_splat( - block, - caller_splat, - optimized_block, - replacement, - &send, - state, - ); - self.make_equal_to(insn_id, join_param); + let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = + function.emit_send_direct_args(optimized_block, call, &args, send_frame_state); + function.try_inline_send_direct(optimized_block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))) + }; + + let replacement = if let Some(caller_splat) = caller_splat { + let (join_block, join_param) = self.dispatch_caller_splat(block, caller_splat, optimized_block, &send, state, emit_optimized); block = join_block; + join_param } else { - self.make_equal_to(insn_id, replacement); - } + emit_optimized(self, optimized_block) + }; + self.make_equal_to(insn_id, replacement); } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; let proc = unsafe { rb_jit_get_proc_ptr(procv) }; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index e9d479caa3040d..d37d3b6835ea51 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -14670,22 +14670,22 @@ mod hir_opt_tests { v49:NilClass = Const Value(nil) v13:ArrayExact = NewArray v19:ArrayExact = ToArray v13 - v54:CInt64 = ArrayLength v19 - v55:CInt64[0] = Const CInt64(0) - v56:CBool = IsBitEqual v54, v55 - CondBranch v56, bb4(), bb5() + v50:CInt64 = ArrayLength v19 + v51:CInt64[0] = Const CInt64(0) + v52:CBool = IsBitEqual v50, v51 + CondBranch v52, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v50:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - PushInlineFrame :foo, v50 (0x1038), num_args=0 + v55:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + PushInlineFrame :foo, v55 (0x1038), num_args=0 PatchPoint MethodRedefined(Object@0x1000, itself@0x1060, cme:0x1068) CheckInterrupts PopInlineFrame - Jump bb6(v50) + Jump bb6(v55) bb5(): - v58:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing - Jump bb6(v58) - bb6(v53:BasicObject): + v59:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing + Jump bb6(v59) + bb6(v49:BasicObject): v25:StringExact[VALUE(0x1090)] = Const Value(VALUE(0x1090)) v26:StringExact = StringCopy v25 PatchPoint NoEPEscape(test) @@ -14730,41 +14730,41 @@ mod hir_opt_tests { v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count IncrCounter caller_splat_profile_monomorphic - v41:CInt64 = ArrayLength v20 - v42:CInt64[2] = Const CInt64(2) - v43:CBool = IsBitEqual v41, v42 - CondBranch v43, bb7(), bb5() + v32:CInt64 = ArrayLength v20 + v33:CInt64[2] = Const CInt64(2) + v34:CBool = IsBitEqual v32, v33 + CondBranch v34, bb7(), bb5() bb7(): - v45:CInt64 = CCall v20, :rb_jit_ruby2_keywords_splat_p@0x1001 - v46:CInt64[0] = Const CInt64(0) - v47:CBool = IsBitEqual v45, v46 - CondBranch v47, bb4(), bb5() + v36:CInt64 = CCall v20, :rb_jit_ruby2_keywords_splat_p@0x1001 + v37:CInt64[0] = Const CInt64(0) + v38:CBool = IsBitEqual v36, v37 + CondBranch v38, bb4(), bb5() bb4(): IncrCounter caller_splat_optimized PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v33:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v10, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v34:CInt64[0] = Const CInt64(0) - v35:BasicObject = ArrayAref v20, v34 - v36:CInt64[1] = Const CInt64(1) - v37:BasicObject = ArrayAref v20, v36 - PushInlineFrame :foo, v33 (0x1040), num_args=2 + v42:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v10, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v43:CInt64[0] = Const CInt64(0) + v44:BasicObject = ArrayAref v20, v43 + v45:CInt64[1] = Const CInt64(1) + v46:BasicObject = ArrayAref v20, v45 + PushInlineFrame :foo, v42 (0x1040), num_args=2 IncrCounter inline_iseq_optimized_send_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v65:ArrayExact = NewArray v35, v37 + v65:ArrayExact = NewArray v44, v46 IncrCounter zjit_insn_count CheckInterrupts PopInlineFrame Jump bb6(v65) bb5(): IncrCounter complex_arg_pass_caller_splat - v50:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing - Jump bb6(v50) - bb6(v40:BasicObject): + v51:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing + Jump bb6(v51) + bb6(v31:BasicObject): IncrCounter zjit_insn_count CheckInterrupts - Return v40 + Return v31 "); } @@ -14791,34 +14791,34 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v31:CInt64 = ArrayLength v16 - v32:CInt64[1] = Const CInt64(1) - v33:CBool = IsBitEqual v31, v32 - CondBranch v33, bb7(), bb5() + v25:CInt64 = ArrayLength v16 + v26:CInt64[1] = Const CInt64(1) + v27:CBool = IsBitEqual v25, v26 + CondBranch v27, bb7(), bb5() bb7(): - v35:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v36:CInt64[0] = Const CInt64(0) - v37:CBool = IsBitEqual v35, v36 - CondBranch v37, bb4(), bb5() + v29:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v29, v30 + CondBranch v31, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v26:CInt64[0] = Const CInt64(0) - v27:BasicObject = ArrayAref v16, v26 - PushInlineFrame :foo, v25 (0x1040), num_args=1 + v34:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v35:CInt64[0] = Const CInt64(0) + v36:BasicObject = ArrayAref v16, v35 + PushInlineFrame :foo, v34 (0x1040), num_args=1 v49:Fixnum[1] = Const Value(1) PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) - v63:Fixnum = GuardType v27, Fixnum recompile + v63:Fixnum = GuardType v36, Fixnum recompile v64:Fixnum = FixnumAdd v63, v49 CheckInterrupts PopInlineFrame Jump bb6(v64) bb5(): - v39:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v39) - bb6(v30:BasicObject): + v40:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v40) + bb6(v24:BasicObject): CheckInterrupts - Return v30 + Return v24 "); } @@ -14845,24 +14845,24 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v29:CInt64 = ArrayLength v16 - v30:CInt64[0] = Const CInt64(0) - v31:CBool = IsBitEqual v29, v30 - CondBranch v31, bb4(), bb5() + v25:CInt64 = ArrayLength v16 + v26:CInt64[0] = Const CInt64(0) + v27:CBool = IsBitEqual v25, v26 + CondBranch v27, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - PushInlineFrame :foo, v25 (0x1040), num_args=0 + v30:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + PushInlineFrame :foo, v30 (0x1040), num_args=0 v41:Fixnum[1] = Const Value(1) CheckInterrupts PopInlineFrame Jump bb6(v41) bb5(): - v33:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v33) - bb6(v28:BasicObject): + v34:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v34) + bb6(v24:BasicObject): CheckInterrupts - Return v28 + Return v24 "); } @@ -14890,33 +14890,33 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) v18:ArrayExact = ToArray v10 - v35:CInt64 = ArrayLength v18 - v36:CInt64[2] = Const CInt64(2) - v37:CBool = IsBitEqual v35, v36 - CondBranch v37, bb7(), bb5() + v27:CInt64 = ArrayLength v18 + v28:CInt64[2] = Const CInt64(2) + v29:CBool = IsBitEqual v27, v28 + CondBranch v29, bb7(), bb5() bb7(): - v39:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 - v40:CInt64[0] = Const CInt64(0) - v41:CBool = IsBitEqual v39, v40 - CondBranch v41, bb4(), bb5() + v31:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v32:CInt64[0] = Const CInt64(0) + v33:CBool = IsBitEqual v31, v32 + CondBranch v33, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v28:CInt64[0] = Const CInt64(0) - v29:BasicObject = ArrayAref v18, v28 - v30:CInt64[1] = Const CInt64(1) - v31:BasicObject = ArrayAref v18, v30 - PushInlineFrame :foo, v27 (0x1040), num_args=3 - v57:ArrayExact = NewArray v15, v29, v31 + v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v37:CInt64[0] = Const CInt64(0) + v38:BasicObject = ArrayAref v18, v37 + v39:CInt64[1] = Const CInt64(1) + v40:BasicObject = ArrayAref v18, v39 + PushInlineFrame :foo, v36 (0x1040), num_args=3 + v57:ArrayExact = NewArray v15, v38, v40 CheckInterrupts PopInlineFrame Jump bb6(v57) bb5(): - v43:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing - Jump bb6(v43) - bb6(v34:BasicObject): + v44:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing + Jump bb6(v44) + bb6(v26:BasicObject): CheckInterrupts - Return v34 + Return v26 "); } @@ -14943,47 +14943,47 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v44:CInt64 = ArrayLength v16 - v45:CInt64[7] = Const CInt64(7) - v46:CBool = IsBitEqual v44, v45 - CondBranch v46, bb7(), bb5() + v25:CInt64 = ArrayLength v16 + v26:CInt64[7] = Const CInt64(7) + v27:CBool = IsBitEqual v25, v26 + CondBranch v27, bb7(), bb5() bb7(): - v48:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v49:CInt64[0] = Const CInt64(0) - v50:CBool = IsBitEqual v48, v49 - CondBranch v50, bb4(), bb5() + v29:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v30:CInt64[0] = Const CInt64(0) + v31:CBool = IsBitEqual v29, v30 + CondBranch v31, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v25:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v26:CInt64[0] = Const CInt64(0) - v27:BasicObject = ArrayAref v16, v26 - v28:CInt64[1] = Const CInt64(1) - v29:BasicObject = ArrayAref v16, v28 - v30:CInt64[2] = Const CInt64(2) - v31:BasicObject = ArrayAref v16, v30 - v32:CInt64[3] = Const CInt64(3) - v33:BasicObject = ArrayAref v16, v32 - v34:CInt64[4] = Const CInt64(4) - v35:BasicObject = ArrayAref v16, v34 - v36:CInt64[5] = Const CInt64(5) - v37:BasicObject = ArrayAref v16, v36 - v38:CInt64[6] = Const CInt64(6) - v39:BasicObject = ArrayAref v16, v38 - v40:ArrayExact = NewArray v27, v29, v31, v33, v35, v37, v39 - PushInlineFrame :foo, v25 (0x1040), num_args=1 + v34:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v35:CInt64[0] = Const CInt64(0) + v36:BasicObject = ArrayAref v16, v35 + v37:CInt64[1] = Const CInt64(1) + v38:BasicObject = ArrayAref v16, v37 + v39:CInt64[2] = Const CInt64(2) + v40:BasicObject = ArrayAref v16, v39 + v41:CInt64[3] = Const CInt64(3) + v42:BasicObject = ArrayAref v16, v41 + v43:CInt64[4] = Const CInt64(4) + v44:BasicObject = ArrayAref v16, v43 + v45:CInt64[5] = Const CInt64(5) + v46:BasicObject = ArrayAref v16, v45 + v47:CInt64[6] = Const CInt64(6) + v48:BasicObject = ArrayAref v16, v47 + v49:ArrayExact = NewArray v36, v38, v40, v42, v44, v46, v48 + PushInlineFrame :foo, v34 (0x1040), num_args=1 PatchPoint NoSingletonClass(Array@0x1068) PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) - v76:CInt64 = ArrayLength v40 + v76:CInt64 = ArrayLength v49 v77:Fixnum = BoxFixnum v76 CheckInterrupts PopInlineFrame Jump bb6(v77) bb5(): - v52:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v52) - bb6(v43:BasicObject): + v53:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v53) + bb6(v24:BasicObject): CheckInterrupts - Return v43 + Return v24 "); } @@ -15011,38 +15011,38 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) v18:ArrayExact = ToArray v10 - v39:CInt64 = ArrayLength v18 - v40:CInt64[3] = Const CInt64(3) - v41:CBool = IsBitEqual v39, v40 - CondBranch v41, bb7(), bb5() + v27:CInt64 = ArrayLength v18 + v28:CInt64[3] = Const CInt64(3) + v29:CBool = IsBitEqual v27, v28 + CondBranch v29, bb7(), bb5() bb7(): - v43:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 - v44:CInt64[0] = Const CInt64(0) - v45:CBool = IsBitEqual v43, v44 - CondBranch v45, bb4(), bb5() + v31:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v32:CInt64[0] = Const CInt64(0) + v33:CBool = IsBitEqual v31, v32 + CondBranch v33, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v28:CInt64[0] = Const CInt64(0) - v29:BasicObject = ArrayAref v18, v28 - v30:CInt64[1] = Const CInt64(1) - v31:BasicObject = ArrayAref v18, v30 - v32:ArrayExact = NewArray v31 - v33:CInt64[2] = Const CInt64(2) - v34:BasicObject = ArrayAref v18, v33 - v35:Fixnum[40] = Const Value(40) + v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v37:CInt64[0] = Const CInt64(0) + v38:BasicObject = ArrayAref v18, v37 + v39:CInt64[1] = Const CInt64(1) + v40:BasicObject = ArrayAref v18, v39 + v41:ArrayExact = NewArray v40 + v42:CInt64[2] = Const CInt64(2) + v43:BasicObject = ArrayAref v18, v42 + v44:Fixnum[40] = Const Value(40) v71:Fixnum[0] = Const Value(0) - PushInlineFrame :foo, v27 (0x1040), num_args=5 - v66:ArrayExact = NewArray v15, v29, v32, v34, v35 + PushInlineFrame :foo, v36 (0x1040), num_args=5 + v66:ArrayExact = NewArray v15, v38, v41, v43, v44 CheckInterrupts PopInlineFrame Jump bb6(v66) bb5(): - v47:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing - Jump bb6(v47) - bb6(v38:BasicObject): + v48:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing + Jump bb6(v48) + bb6(v26:BasicObject): CheckInterrupts - Return v38 + Return v26 "); } @@ -15110,29 +15110,29 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v38:CInt64 = ArrayLength v16 - v39:CInt64[3] = Const CInt64(3) - v40:CBool = IsBitEqual v38, v39 - CondBranch v40, bb7(), bb5() + v27:CInt64 = ArrayLength v16 + v28:CInt64[3] = Const CInt64(3) + v29:CBool = IsBitEqual v27, v28 + CondBranch v29, bb7(), bb5() bb7(): - v42:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v43:CInt64[0] = Const CInt64(0) - v44:CBool = IsBitEqual v42, v43 - CondBranch v44, bb4(), bb5() + v31:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v32:CInt64[0] = Const CInt64(0) + v33:CBool = IsBitEqual v31, v32 + CondBranch v33, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v28:CInt64[0] = Const CInt64(0) - v29:BasicObject = ArrayAref v16, v28 - v30:CInt64[1] = Const CInt64(1) - v31:BasicObject = ArrayAref v16, v30 - v32:CInt64[2] = Const CInt64(2) - v33:BasicObject = ArrayAref v16, v32 - v34:ArrayExact = NewArray v29, v31, v33 - PushInlineFrame :foo, v27 (0x1040), num_args=1 + v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v37:CInt64[0] = Const CInt64(0) + v38:BasicObject = ArrayAref v16, v37 + v39:CInt64[1] = Const CInt64(1) + v40:BasicObject = ArrayAref v16, v39 + v41:CInt64[2] = Const CInt64(2) + v42:BasicObject = ArrayAref v16, v41 + v43:ArrayExact = NewArray v38, v40, v42 + PushInlineFrame :foo, v36 (0x1040), num_args=1 PatchPoint NoSingletonClass(Array@0x1068) PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) - v76:CInt64 = ArrayLength v34 + v76:CInt64 = ArrayLength v43 v77:Fixnum = BoxFixnum v76 v59:CPtr = GetEP 0 v60:CInt64 = LoadField v59, :VM_ENV_DATA_INDEX_SPECVAL@0x10a0 @@ -15143,12 +15143,12 @@ mod hir_opt_tests { PopInlineFrame Jump bb6(v63) bb5(): - v46:BasicObject = Send v9, 0x10a8, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v46) - bb6(v37:BasicObject): + v47:BasicObject = Send v9, 0x10a8, :foo, v16 # SendFallbackReason: Complex argument passing + Jump bb6(v47) + bb6(v26:BasicObject): PatchPoint NoEPEscape(test) CheckInterrupts - Return v37 + Return v26 "); } @@ -15265,33 +15265,33 @@ mod hir_opt_tests { v23:NilClass = Const Value(nil) v26:StaticSymbol[:value] = Const Value(VALUE(0x1008)) v30:ArrayExact = ToArray v12 - v50:CInt64 = ArrayLength v30 - v51:CInt64[1] = Const CInt64(1) - v52:CBool = IsBitEqual v50, v51 - CondBranch v52, bb7(), bb5() + v44:CInt64 = ArrayLength v30 + v45:CInt64[1] = Const CInt64(1) + v46:CBool = IsBitEqual v44, v45 + CondBranch v46, bb7(), bb5() bb7(): - v54:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 - v55:CInt64[0] = Const CInt64(0) - v56:CBool = IsBitEqual v54, v55 - CondBranch v56, bb4(), bb5() + v48:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 + v49:CInt64[0] = Const CInt64(0) + v50:CBool = IsBitEqual v48, v49 + CondBranch v50, bb4(), bb5() bb4(): PatchPoint MethodRedefined(Object@0x1018, target@0x1020, cme:0x1028) - v44:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile - v45:CInt64[0] = Const CInt64(0) - v46:BasicObject = ArrayAref v30, v45 - PushInlineFrame :target, v44 (0x1050), num_args=1 + v53:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile + v54:CInt64[0] = Const CInt64(0) + v55:BasicObject = ArrayAref v30, v54 + PushInlineFrame :target, v53 (0x1050), num_args=1 CheckInterrupts PopInlineFrame - Jump bb6(v46) + Jump bb6(v55) bb5(): - v58:BasicObject = Send v11, :target, v30 # SendFallbackReason: Complex argument passing - Jump bb6(v58) - bb6(v49:BasicObject): + v59:BasicObject = Send v11, :target, v30 # SendFallbackReason: Complex argument passing + Jump bb6(v59) + bb6(v43:BasicObject): PatchPoint NoSingletonClass(Hash@0x1078) PatchPoint MethodRedefined(Hash@0x1078, []=@0x1080, cme:0x1088) - HashAset v17, v26, v49 + HashAset v17, v26, v43 CheckInterrupts - Return v49 + Return v43 "); } From 76d4288406b55b2f01a7c31b7825ab691c0915f2 Mon Sep 17 00:00:00 2001 From: nozomemein Date: Thu, 27 Aug 2026 12:41:24 +0900 Subject: [PATCH 4/8] ZJIT: Side-exit on caller-splat guard misses Use guards instead of a dynamic Send fallback and join block for monomorphic caller splats. Recompile on length mismatches so the call site can collect a new profile, and side-exit without recompiling for ruby2_keywords hashes. Skip caller-splat specialization for final no-side-exit ISEQ versions. --- zjit/src/codegen_tests.rs | 8 +- zjit/src/hir.rs | 134 ++++-------- zjit/src/hir/opt_tests.rs | 443 +++++++++++++++++--------------------- zjit/src/stats.rs | 4 + 4 files changed, 245 insertions(+), 344 deletions(-) diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 42769ab6c00c95..a7aeb434c18352 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -7193,17 +7193,17 @@ fn test_send_caller_splat_arguments_with_block_literal() { } #[test] -fn test_send_caller_splat_length_mismatch_falls_back() { +fn test_send_caller_splat_length_mismatch_side_exits() { eval(" def test(*args) = args def entry(args) = test(*args) entry([1, 2]) "); - assert_snapshot!(assert_compiles("entry([1, 2, 3])"), @"[1, 2, 3]"); + assert_snapshot!(assert_compiles_allowing_exits("entry([1, 2, 3])"), @"[1, 2, 3]"); } #[test] -fn test_send_caller_splat_with_ruby2_keywords_hash_falls_back() { +fn test_send_caller_splat_with_ruby2_keywords_hash_side_exits() { eval(" def capture(*args) = args ruby2_keywords(:capture) @@ -7211,7 +7211,7 @@ fn test_send_caller_splat_with_ruby2_keywords_hash_falls_back() { def entry(args) = test(*args) entry(capture(k: 1)) "); - assert_snapshot!(assert_compiles("entry(capture(k: 1))"), @"[:default, 1]"); + assert_snapshot!(assert_compiles_allowing_exits("entry(capture(k: 1))"), @"[:default, 1]"); } #[test] diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 8943101a3ecc17..ea0fe799aa1297 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -665,6 +665,8 @@ pub enum SideExitReason { SplatKwNotNilOrHash, SplatKwPolymorphic, SplatKwNotProfiled, + CallerSplatLengthMismatch, + CallerSplatRuby2Keywords, DirectiveInduced, SendWhileTracing, NoProfileSend, @@ -3977,54 +3979,32 @@ impl Function { args } - /// Dispatch a Send with a caller splat between a fixed-length direct path and - /// the original Send fallback. The callback generates the direct path after - /// the runtime length and ruby2_keywords checks have been connected. + /// Guard a monomorphic caller splat and generate its direct path in place. fn dispatch_caller_splat( &mut self, block: BlockId, caller_splat: CallerSplat, - optimized_block: BlockId, - send: &Insn, state: InsnId, emit_optimized: impl FnOnce(&mut Function, BlockId) -> InsnId, - ) -> (BlockId, InsnId) { - // The fixed-length direct path and VM fallback produce the result of the - // same Send, so route them through one join block. - let insn_idx = self.frame_state(state).insn_idx() as u32; - let fallback_block = self.new_block(insn_idx); - let join_block = self.new_block(insn_idx); - let join_param = self.push_insn(join_block, Insn::Param); - // The join result may be used later in this type_specialize pass, before - // infer_types runs again. Both branches produce a Ruby value. - self.insn_types[join_param.to_usize()] = types::BasicObject; - let edge = |target| BranchEdge { target, args: vec![] }; - - // Compare the runtime array length with the profiled length before - // entering the path that expands the array for SendDirect. + ) -> InsnId { + // Recompile when the runtime length changes so the call site can collect + // a new length profile instead of repeatedly taking the same side exit. let length = self.push_insn(block, Insn::ArrayLength { array: caller_splat.array }); - let expected = self.push_insn(block, Insn::Const { val: Const::CInt64(i64::from(caller_splat.length)) }); - let length_matches = self.push_insn(block, Insn::IsBitEqual { left: length, right: expected }); + self.push_insn(block, Insn::GuardBitEquals { + val: length, + expected: Const::CInt64(i64::from(caller_splat.length)), + reason: Box::new(SideExitReason::CallerSplatLengthMismatch), + state, + recompile: Some(Recompile), + }); // An empty splat cannot end in a ruby2_keywords hash, so skip // that runtime check when the profiled length is zero. - if caller_splat.length == 0 { - self.push_insn(block, Insn::CondBranch { - val: length_matches, - if_true: edge(optimized_block), - if_false: edge(fallback_block), - }); - } else { - let ruby2_keywords_block = self.new_block(insn_idx); - self.push_insn(block, Insn::CondBranch { - val: length_matches, - if_true: edge(ruby2_keywords_block), - if_false: edge(fallback_block), - }); - + if caller_splat.length != 0 { // A ruby2_keywords hash changes how the VM interprets the final splat - // element, so only expand arrays that preserve positional semantics. - let ruby2_keywords_splat = self.push_insn(ruby2_keywords_block, Insn::CCall { + // element. Recompilation would produce the same length-based plan, so + // side-exit without recompiling when one is present. + let ruby2_keywords_splat = self.push_insn(block, Insn::CCall { cfunc: rb_jit_ruby2_keywords_splat_p as *const u8, recv: caller_splat.array, args: vec![], @@ -4033,27 +4013,16 @@ impl Function { return_type: types::CInt64, elidable: false, }); - let zero = self.push_insn(ruby2_keywords_block, Insn::Const { val: Const::CInt64(0) }); - let is_positional = self.push_insn(ruby2_keywords_block, Insn::IsBitEqual { left: ruby2_keywords_splat, right: zero }); - self.push_insn(ruby2_keywords_block, Insn::CondBranch { - val: is_positional, - if_true: edge(optimized_block), - if_false: edge(fallback_block), + self.push_insn(block, Insn::GuardBitEquals { + val: ruby2_keywords_splat, + expected: Const::CInt64(0), + reason: Box::new(SideExitReason::CallerSplatRuby2Keywords), + state, + recompile: None, }); } - // Generate the direct path only in the block selected by the checks above. - let optimized_result = emit_optimized(self, optimized_block); - self.push_insn(optimized_block, Insn::Jump(BranchEdge { target: join_block, args: vec![optimized_result] })); - - // Preserve the original splat Send on the fallback path so VM argument - // setup handles lengths and keyword conversion that SendDirect cannot. - self.count(fallback_block, Counter::complex_arg_pass_caller_splat); - let fallback_result = self.push_insn(fallback_block, send.clone()); - self.set_dynamic_send_reason(fallback_result, ComplexArgPass); - self.push_insn(fallback_block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); - - (join_block, join_param) + emit_optimized(self, block) } /// Reorder keyword arguments to match the callee's expected order, and synthesize @@ -4641,10 +4610,9 @@ impl Function { /// opens the door for inlining. /// Also try and inline constant caches, specialize object allocations, and more. fn type_specialize(&mut self) { - for original_block in self.reverse_post_order() { - let old_insns = std::mem::take(&mut self.blocks[original_block].insns); - assert!(self.blocks[original_block].insns.is_empty()); - let mut block = original_block; + for block in self.reverse_post_order() { + let old_insns = std::mem::take(&mut self.blocks[block].insns); + assert!(self.blocks[block].insns.is_empty()); for insn_id in old_insns { let resolved = self.resolve(insn_id); match resolved.insn(self) { @@ -4652,8 +4620,7 @@ impl Function { self.try_rewrite_freeze(block, insn_id, recv, state), &Insn::Send { recv, block: None, ref args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => self.try_rewrite_uminus(block, insn_id, recv, state), - &Insn::Send { mut recv, cd, state, block: send_block, reason, .. } => { - let send = resolved.insn(self).clone(); + &Insn::Send { mut recv, cd, state, block: send_block, .. } => { let mut has_block = send_block.is_some(); let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), state) { ReceiverTypeResolution::StaticallyKnown { class } => (class, None), @@ -4783,14 +4750,16 @@ impl Function { let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; let caller_args = CallerArguments::new(&args, ci); let caller_splat = if let Some(arg_idx) = caller_args.splat_arg_idx { - // A classified caller-splat Send is a dynamic fallback retained from - // an earlier specialization pass. Do not dispatch it again. - if !matches!(reason, Uncategorized(_)) { - self.push_insn_id(block, insn_id); continue; - } // Count the profile shape for every caller-splat execution; // complex_arg_pass_caller_splat separately tracks fallbacks. self.count_caller_splat_profile(block, state); + // The final ISEQ version cannot recover from guard exits by + // recompiling, so keep the dynamic Send instead of specializing. + if self.policy.no_side_exits { + self.count(block, Counter::complex_arg_pass_caller_splat); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + } // Expand a caller splat only when profiling observed one stable // array length; otherwise keep the original dynamic Send. // TODO: Support polymorphic caller-splat length profiles. @@ -4816,38 +4785,25 @@ impl Function { self.push_insn_id(block, insn_id); continue; }; - // Start the direct path in a detached block so a failed singleton-class - // assumption can discard it before caller-splat dispatch connects the CFG. - // Calls without a splat use the current block directly. - let optimized_block = if caller_splat.is_some() { - let insn_idx = self.frame_state(state).insn_idx() as u32; - self.new_block(insn_idx) - } else { - block - }; - // Check singleton class assumption first, before emitting other patchpoints - if !self.assume_no_singleton_classes(optimized_block, klass, state) { - if caller_splat.is_some() { - self.remove_block(optimized_block); - } + if !self.assume_no_singleton_classes(block, klass, state) { self.set_dynamic_send_reason(insn_id, SingletonClassSeen); self.push_insn_id(block, insn_id); continue; } - let emit_optimized = move |function: &mut Function, optimized_block: BlockId| { + let emit_optimized = move |function: &mut Function, block: BlockId| { if caller_splat.is_some() { // Count caller-splat executions that take this optimized path. // This is a feature-specific counter, not part of optimized_send_count. - function.count(optimized_block, Counter::caller_splat_optimized); + function.count(block, Counter::caller_splat_optimized); } // Add PatchPoint for method redefinition - function.push_insn(optimized_block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + function.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); // Add GuardType for profiled receiver let recv = if let Some(profiled_type) = profiled_type { - let recv = function.push_insn(optimized_block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); + let recv = function.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); function.insn_types[recv] = function.infer_type(recv); recv } else { @@ -4855,16 +4811,14 @@ impl Function { }; let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = - function.emit_send_direct_args(optimized_block, call, &args, send_frame_state); - function.try_inline_send_direct(optimized_block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))) + function.emit_send_direct_args(block, call, &args, send_frame_state); + function.try_inline_send_direct(block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))) }; let replacement = if let Some(caller_splat) = caller_splat { - let (join_block, join_param) = self.dispatch_caller_splat(block, caller_splat, optimized_block, &send, state, emit_optimized); - block = join_block; - join_param + self.dispatch_caller_splat(block, caller_splat, state, emit_optimized) } else { - emit_optimized(self, optimized_block) + emit_optimized(self, block) }; self.make_equal_to(insn_id, replacement); } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index d37d3b6835ea51..86984403081e64 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -14667,33 +14667,25 @@ mod hir_opt_tests { v5:BasicObject = LoadArg :self@0 Jump bb3(v5) bb3(v8:BasicObject): - v49:NilClass = Const Value(nil) + v70:NilClass = Const Value(nil) v13:ArrayExact = NewArray v19:ArrayExact = ToArray v13 - v50:CInt64 = ArrayLength v19 - v51:CInt64[0] = Const CInt64(0) - v52:CBool = IsBitEqual v50, v51 - CondBranch v52, bb4(), bb5() - bb4(): + v49:CInt64 = ArrayLength v19 + v50:CInt64[0] = GuardBitEquals v49, CInt64(0) recompile PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) - v55:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile - PushInlineFrame :foo, v55 (0x1038), num_args=0 - PatchPoint MethodRedefined(Object@0x1000, itself@0x1060, cme:0x1068) + v52:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + PushInlineFrame :foo, v52 (0x1038), num_args=0 + PatchPoint MethodRedefined(Object@0x1000, itself@0x1058, cme:0x1060) CheckInterrupts PopInlineFrame - Jump bb6(v55) - bb5(): - v59:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing - Jump bb6(v59) - bb6(v49:BasicObject): - v25:StringExact[VALUE(0x1090)] = Const Value(VALUE(0x1090)) + v25:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) v26:StringExact = StringCopy v25 PatchPoint NoEPEscape(test) v31:ArrayExact = ToArray v13 v33:BasicObject = Send v26, :display, v31 # SendFallbackReason: Complex argument passing PatchPoint NoEPEscape(test) v41:ArrayExact = ToArray v13 - v43:BasicObject = Send v8, :itself, v41 # SendFallbackReason: Complex argument passing + v43:BasicObject = Send v52, :itself, v41 # SendFallbackReason: Complex argument passing CheckInterrupts Return v43 "); @@ -14730,41 +14722,28 @@ mod hir_opt_tests { v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count IncrCounter caller_splat_profile_monomorphic - v32:CInt64 = ArrayLength v20 - v33:CInt64[2] = Const CInt64(2) - v34:CBool = IsBitEqual v32, v33 - CondBranch v34, bb7(), bb5() - bb7(): - v36:CInt64 = CCall v20, :rb_jit_ruby2_keywords_splat_p@0x1001 - v37:CInt64[0] = Const CInt64(0) - v38:CBool = IsBitEqual v36, v37 - CondBranch v38, bb4(), bb5() - bb4(): + v32:CInt64 = ArrayLength v21 + v33:CInt64[2] = GuardBitEquals v32, CInt64(2) recompile + v34:CInt64 = CCall v21, :rb_jit_ruby2_keywords_splat_p@0x1001 + v35:CInt64[0] = GuardBitEquals v34, CInt64(0) IncrCounter caller_splat_optimized PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v42:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v10, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v43:CInt64[0] = Const CInt64(0) - v44:BasicObject = ArrayAref v20, v43 - v45:CInt64[1] = Const CInt64(1) - v46:BasicObject = ArrayAref v20, v45 - PushInlineFrame :foo, v42 (0x1040), num_args=2 + v38:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v39:CInt64[0] = Const CInt64(0) + v40:BasicObject = ArrayAref v21, v39 + v41:CInt64[1] = Const CInt64(1) + v42:BasicObject = ArrayAref v21, v41 + PushInlineFrame :foo, v38 (0x1040), num_args=2 IncrCounter inline_iseq_optimized_send_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v65:ArrayExact = NewArray v44, v46 + v57:ArrayExact = NewArray v40, v42 IncrCounter zjit_insn_count CheckInterrupts PopInlineFrame - Jump bb6(v65) - bb5(): - IncrCounter complex_arg_pass_caller_splat - v51:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing - Jump bb6(v51) - bb6(v31:BasicObject): IncrCounter zjit_insn_count - CheckInterrupts - Return v31 + Return v57 "); } @@ -14791,34 +14770,22 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v25:CInt64 = ArrayLength v16 - v26:CInt64[1] = Const CInt64(1) - v27:CBool = IsBitEqual v25, v26 - CondBranch v27, bb7(), bb5() - bb7(): - v29:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v30:CInt64[0] = Const CInt64(0) - v31:CBool = IsBitEqual v29, v30 - CondBranch v31, bb4(), bb5() - bb4(): + v24:CInt64 = ArrayLength v16 + v25:CInt64[1] = GuardBitEquals v24, CInt64(1) recompile + v26:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v27:CInt64[0] = GuardBitEquals v26, CInt64(0) PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v34:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v35:CInt64[0] = Const CInt64(0) - v36:BasicObject = ArrayAref v16, v35 - PushInlineFrame :foo, v34 (0x1040), num_args=1 - v49:Fixnum[1] = Const Value(1) - PatchPoint MethodRedefined(Integer@0x1068, +@0x1070, cme:0x1078) - v63:Fixnum = GuardType v36, Fixnum recompile - v64:Fixnum = FixnumAdd v63, v49 + v29:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v30:CInt64[0] = Const CInt64(0) + v31:BasicObject = ArrayAref v16, v30 + PushInlineFrame :foo, v29 (0x1040), num_args=1 + v41:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + v55:Fixnum = GuardType v31, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v41 CheckInterrupts PopInlineFrame - Jump bb6(v64) - bb5(): - v40:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v40) - bb6(v24:BasicObject): - CheckInterrupts - Return v24 + Return v56 "); } @@ -14845,24 +14812,15 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v25:CInt64 = ArrayLength v16 - v26:CInt64[0] = Const CInt64(0) - v27:CBool = IsBitEqual v25, v26 - CondBranch v27, bb4(), bb5() - bb4(): + v24:CInt64 = ArrayLength v16 + v25:CInt64[0] = GuardBitEquals v24, CInt64(0) recompile PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v30:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - PushInlineFrame :foo, v30 (0x1040), num_args=0 - v41:Fixnum[1] = Const Value(1) + v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + PushInlineFrame :foo, v27 (0x1040), num_args=0 + v35:Fixnum[1] = Const Value(1) CheckInterrupts PopInlineFrame - Jump bb6(v41) - bb5(): - v34:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v34) - bb6(v24:BasicObject): - CheckInterrupts - Return v24 + Return v35 "); } @@ -14890,33 +14848,21 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) v18:ArrayExact = ToArray v10 - v27:CInt64 = ArrayLength v18 - v28:CInt64[2] = Const CInt64(2) - v29:CBool = IsBitEqual v27, v28 - CondBranch v29, bb7(), bb5() - bb7(): - v31:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 - v32:CInt64[0] = Const CInt64(0) - v33:CBool = IsBitEqual v31, v32 - CondBranch v33, bb4(), bb5() - bb4(): + v26:CInt64 = ArrayLength v18 + v27:CInt64[2] = GuardBitEquals v26, CInt64(2) recompile + v28:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v37:CInt64[0] = Const CInt64(0) - v38:BasicObject = ArrayAref v18, v37 - v39:CInt64[1] = Const CInt64(1) - v40:BasicObject = ArrayAref v18, v39 - PushInlineFrame :foo, v36 (0x1040), num_args=3 - v57:ArrayExact = NewArray v15, v38, v40 + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v18, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v18, v34 + PushInlineFrame :foo, v31 (0x1040), num_args=3 + v49:ArrayExact = NewArray v15, v33, v35 CheckInterrupts PopInlineFrame - Jump bb6(v57) - bb5(): - v44:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing - Jump bb6(v44) - bb6(v26:BasicObject): - CheckInterrupts - Return v26 + Return v49 "); } @@ -14943,47 +14889,35 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v25:CInt64 = ArrayLength v16 - v26:CInt64[7] = Const CInt64(7) - v27:CBool = IsBitEqual v25, v26 - CondBranch v27, bb7(), bb5() - bb7(): - v29:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v30:CInt64[0] = Const CInt64(0) - v31:CBool = IsBitEqual v29, v30 - CondBranch v31, bb4(), bb5() - bb4(): + v24:CInt64 = ArrayLength v16 + v25:CInt64[7] = GuardBitEquals v24, CInt64(7) recompile + v26:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v27:CInt64[0] = GuardBitEquals v26, CInt64(0) PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v34:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v35:CInt64[0] = Const CInt64(0) - v36:BasicObject = ArrayAref v16, v35 - v37:CInt64[1] = Const CInt64(1) - v38:BasicObject = ArrayAref v16, v37 - v39:CInt64[2] = Const CInt64(2) - v40:BasicObject = ArrayAref v16, v39 - v41:CInt64[3] = Const CInt64(3) - v42:BasicObject = ArrayAref v16, v41 - v43:CInt64[4] = Const CInt64(4) - v44:BasicObject = ArrayAref v16, v43 - v45:CInt64[5] = Const CInt64(5) - v46:BasicObject = ArrayAref v16, v45 - v47:CInt64[6] = Const CInt64(6) - v48:BasicObject = ArrayAref v16, v47 - v49:ArrayExact = NewArray v36, v38, v40, v42, v44, v46, v48 - PushInlineFrame :foo, v34 (0x1040), num_args=1 - PatchPoint NoSingletonClass(Array@0x1068) - PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) - v76:CInt64 = ArrayLength v49 - v77:Fixnum = BoxFixnum v76 + v29:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v30:CInt64[0] = Const CInt64(0) + v31:BasicObject = ArrayAref v16, v30 + v32:CInt64[1] = Const CInt64(1) + v33:BasicObject = ArrayAref v16, v32 + v34:CInt64[2] = Const CInt64(2) + v35:BasicObject = ArrayAref v16, v34 + v36:CInt64[3] = Const CInt64(3) + v37:BasicObject = ArrayAref v16, v36 + v38:CInt64[4] = Const CInt64(4) + v39:BasicObject = ArrayAref v16, v38 + v40:CInt64[5] = Const CInt64(5) + v41:BasicObject = ArrayAref v16, v40 + v42:CInt64[6] = Const CInt64(6) + v43:BasicObject = ArrayAref v16, v42 + v44:ArrayExact = NewArray v31, v33, v35, v37, v39, v41, v43 + PushInlineFrame :foo, v29 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1060) + PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + v68:CInt64 = ArrayLength v44 + v69:Fixnum = BoxFixnum v68 CheckInterrupts PopInlineFrame - Jump bb6(v77) - bb5(): - v53:BasicObject = Send v9, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v53) - bb6(v24:BasicObject): - CheckInterrupts - Return v24 + Return v69 "); } @@ -15011,38 +14945,26 @@ mod hir_opt_tests { bb3(v9:BasicObject, v10:BasicObject): v15:Fixnum[1] = Const Value(1) v18:ArrayExact = ToArray v10 - v27:CInt64 = ArrayLength v18 - v28:CInt64[3] = Const CInt64(3) - v29:CBool = IsBitEqual v27, v28 - CondBranch v29, bb7(), bb5() - bb7(): - v31:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 - v32:CInt64[0] = Const CInt64(0) - v33:CBool = IsBitEqual v31, v32 - CondBranch v33, bb4(), bb5() - bb4(): + v26:CInt64 = ArrayLength v18 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) recompile + v28:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v37:CInt64[0] = Const CInt64(0) + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v18, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v18, v34 + v36:ArrayExact = NewArray v35 + v37:CInt64[2] = Const CInt64(2) v38:BasicObject = ArrayAref v18, v37 - v39:CInt64[1] = Const CInt64(1) - v40:BasicObject = ArrayAref v18, v39 - v41:ArrayExact = NewArray v40 - v42:CInt64[2] = Const CInt64(2) - v43:BasicObject = ArrayAref v18, v42 - v44:Fixnum[40] = Const Value(40) - v71:Fixnum[0] = Const Value(0) - PushInlineFrame :foo, v36 (0x1040), num_args=5 - v66:ArrayExact = NewArray v15, v38, v41, v43, v44 + v39:Fixnum[40] = Const Value(40) + v63:Fixnum[0] = Const Value(0) + PushInlineFrame :foo, v31 (0x1040), num_args=5 + v58:ArrayExact = NewArray v15, v33, v36, v38, v39 CheckInterrupts PopInlineFrame - Jump bb6(v66) - bb5(): - v48:BasicObject = Send v9, :foo, v15, v18 # SendFallbackReason: Complex argument passing - Jump bb6(v48) - bb6(v26:BasicObject): - CheckInterrupts - Return v26 + Return v58 "); } @@ -15062,28 +14984,29 @@ mod hir_opt_tests { v1:BasicObject = LoadSelf v2:CPtr = LoadSP v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr Jump bb3(v1, v3) bb2(): EntryPoint JIT(0) - v6:BasicObject = LoadArg :self@0 - v7:BasicObject = LoadArg :args@1 + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 IncrCounterPtr - Jump bb3(v6, v7) - bb3(v10:BasicObject, v11:BasicObject): + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v20:ArrayExact = ToArray v11 + v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count - v23:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + v24:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_monomorphic IncrCounter complex_arg_pass_caller_kw_splat - v26:BasicObject = Send v10, :foo, v20, v23 # SendFallbackReason: Complex argument passing + v27:BasicObject = Send v11, :foo, v21, v24 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts - Return v26 + Return v27 "); } @@ -15110,45 +15033,33 @@ mod hir_opt_tests { Jump bb3(v6, v7) bb3(v9:BasicObject, v10:BasicObject): v16:ArrayExact = ToArray v10 - v27:CInt64 = ArrayLength v16 - v28:CInt64[3] = Const CInt64(3) - v29:CBool = IsBitEqual v27, v28 - CondBranch v29, bb7(), bb5() - bb7(): - v31:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 - v32:CInt64[0] = Const CInt64(0) - v33:CBool = IsBitEqual v31, v32 - CondBranch v33, bb4(), bb5() - bb4(): + v26:CInt64 = ArrayLength v16 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) recompile + v28:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) - v36:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile - v37:CInt64[0] = Const CInt64(0) - v38:BasicObject = ArrayAref v16, v37 - v39:CInt64[1] = Const CInt64(1) - v40:BasicObject = ArrayAref v16, v39 - v41:CInt64[2] = Const CInt64(2) - v42:BasicObject = ArrayAref v16, v41 - v43:ArrayExact = NewArray v38, v40, v42 - PushInlineFrame :foo, v36 (0x1040), num_args=1 - PatchPoint NoSingletonClass(Array@0x1068) - PatchPoint MethodRedefined(Array@0x1068, length@0x1070, cme:0x1078) - v76:CInt64 = ArrayLength v43 - v77:Fixnum = BoxFixnum v76 - v59:CPtr = GetEP 0 - v60:CInt64 = LoadField v59, :VM_ENV_DATA_INDEX_SPECVAL@0x10a0 - v61:CInt64[-4] = Const CInt64(-4) - v62:CInt64 = IntAnd v60, v61 - v63:BasicObject = InvokeBlockIseqDirect (0x10a8), v62, v77 + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v16, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v16, v34 + v36:CInt64[2] = Const CInt64(2) + v37:BasicObject = ArrayAref v16, v36 + v38:ArrayExact = NewArray v33, v35, v37 + PushInlineFrame :foo, v31 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1060) + PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + v68:CInt64 = ArrayLength v38 + v69:Fixnum = BoxFixnum v68 + v51:CPtr = GetEP 0 + v52:CInt64 = LoadField v51, :VM_ENV_DATA_INDEX_SPECVAL@0x1098 + v53:CInt64[-4] = Const CInt64(-4) + v54:CInt64 = IntAnd v52, v53 + v55:BasicObject = InvokeBlockIseqDirect (0x10a0), v54, v69 CheckInterrupts PopInlineFrame - Jump bb6(v63) - bb5(): - v47:BasicObject = Send v9, 0x10a8, :foo, v16 # SendFallbackReason: Complex argument passing - Jump bb6(v47) - bb6(v26:BasicObject): PatchPoint NoEPEscape(test) - CheckInterrupts - Return v26 + Return v55 "); } @@ -15168,22 +15079,23 @@ mod hir_opt_tests { v1:BasicObject = LoadSelf v2:CPtr = LoadSP v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr Jump bb3(v1, v3) bb2(): EntryPoint JIT(0) - v6:BasicObject = LoadArg :self@0 - v7:BasicObject = LoadArg :args@1 + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 IncrCounterPtr - Jump bb3(v6, v7) - bb3(v10:BasicObject, v11:BasicObject): + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): IncrCounter zjit_insn_count IncrCounter zjit_insn_count IncrCounter zjit_insn_count - v20:ArrayExact = ToArray v11 + v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count IncrCounter caller_splat_profile_monomorphic IncrCounter send_direct_fallback_context_send - v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Argument count does not match parameter count + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Argument count does not match parameter count IncrCounter zjit_insn_count CheckInterrupts Return v24 @@ -15224,7 +15136,50 @@ mod hir_opt_tests { IncrCounter zjit_insn_count IncrCounter caller_splat_profile_polymorphic IncrCounter complex_arg_pass_caller_splat - v23:BasicObject = Send v10, :foo, v20 # SendFallbackReason: Complex argument passing + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_caller_splat_on_final_version() { + enable_zjit_stats(); + set_max_versions(2); + eval(" + def foo(*args) = args + def test(args) = foo(*args) + test([1]); test([1]) + "); + + // Trigger the length guard enough times to recompile under the + // no-side-exits policy. + eval("50.times { test([1, 2]) }"); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_polymorphic + IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts Return v24 @@ -15234,7 +15189,7 @@ mod hir_opt_tests { #[test] fn specialize_call_to_iseq_with_caller_splat_result_used_by_hash_aset() { // Hash#[]= returns its value argument from its CFunc inline. Ensure it can - // consume the caller-splat join result before infer_types runs again. + // consume the guarded caller-splat result in the same specialization pass. eval(" def target(value) = value def test(args) @@ -15251,47 +15206,35 @@ mod hir_opt_tests { v1:BasicObject = LoadSelf v2:CPtr = LoadSP v3:BasicObject = LoadField v2, :args@0x1000 - v4:NilClass = Const Value(nil) - Jump bb3(v1, v3, v4) + Jump bb3(v1, v3) bb2(): EntryPoint JIT(0) v7:BasicObject = LoadArg :self@0 v8:BasicObject = LoadArg :args@1 - v9:NilClass = Const Value(nil) - Jump bb3(v7, v8, v9) - bb3(v11:BasicObject, v12:BasicObject, v13:NilClass): + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + v72:NilClass = Const Value(nil) v17:HashExact = NewHash PatchPoint NoEPEscape(test) v23:NilClass = Const Value(nil) v26:StaticSymbol[:value] = Const Value(VALUE(0x1008)) v30:ArrayExact = ToArray v12 - v44:CInt64 = ArrayLength v30 - v45:CInt64[1] = Const CInt64(1) - v46:CBool = IsBitEqual v44, v45 - CondBranch v46, bb7(), bb5() - bb7(): - v48:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 - v49:CInt64[0] = Const CInt64(0) - v50:CBool = IsBitEqual v48, v49 - CondBranch v50, bb4(), bb5() - bb4(): + v43:CInt64 = ArrayLength v30 + v44:CInt64[1] = GuardBitEquals v43, CInt64(1) recompile + v45:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 + v46:CInt64[0] = GuardBitEquals v45, CInt64(0) PatchPoint MethodRedefined(Object@0x1018, target@0x1020, cme:0x1028) - v53:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile - v54:CInt64[0] = Const CInt64(0) - v55:BasicObject = ArrayAref v30, v54 - PushInlineFrame :target, v53 (0x1050), num_args=1 + v48:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile + v49:CInt64[0] = Const CInt64(0) + v50:BasicObject = ArrayAref v30, v49 + PushInlineFrame :target, v48 (0x1050), num_args=1 CheckInterrupts PopInlineFrame - Jump bb6(v55) - bb5(): - v59:BasicObject = Send v11, :target, v30 # SendFallbackReason: Complex argument passing - Jump bb6(v59) - bb6(v43:BasicObject): - PatchPoint NoSingletonClass(Hash@0x1078) - PatchPoint MethodRedefined(Hash@0x1078, []=@0x1080, cme:0x1088) - HashAset v17, v26, v43 + PatchPoint NoSingletonClass(Hash@0x1070) + PatchPoint MethodRedefined(Hash@0x1070, []=@0x1078, cme:0x1080) + HashAset v17, v26, v50 CheckInterrupts - Return v43 + Return v50 "); } diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index 31f15d8e0f8632..7574893b7119cf 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -246,6 +246,8 @@ make_counters! { exit_splatkw_not_nil_or_hash, exit_splatkw_polymorphic, exit_splatkw_not_profiled, + exit_caller_splat_length_mismatch, + exit_caller_splat_ruby2_keywords, exit_directive_induced, exit_send_while_tracing, exit_invokeblock_not_ifunc, @@ -645,6 +647,8 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { SplatKwNotNilOrHash => exit_splatkw_not_nil_or_hash, SplatKwPolymorphic => exit_splatkw_polymorphic, SplatKwNotProfiled => exit_splatkw_not_profiled, + CallerSplatLengthMismatch => exit_caller_splat_length_mismatch, + CallerSplatRuby2Keywords => exit_caller_splat_ruby2_keywords, DirectiveInduced => exit_directive_induced, PatchPoint(Invariant::BOPRedefined { .. }) => exit_patchpoint_bop_redefined, From 1d3b0594d40d4759729949c8cc90076f077ac6ae Mon Sep 17 00:00:00 2001 From: nozomemein Date: Wed, 2 Sep 2026 08:34:47 +0900 Subject: [PATCH 5/8] ZJIT: Test caller-splat recompilation with a second length Document that observing a second splat length makes the profile non-monomorphic, so the next version keeps the dynamic Send instead of emitting the same length guard. Add an opt test that verifies this before the final-version policy applies. --- zjit/src/hir.rs | 5 ++-- zjit/src/hir/opt_tests.rs | 57 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index ea0fe799aa1297..cd9149536fba61 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -3987,8 +3987,9 @@ impl Function { state: InsnId, emit_optimized: impl FnOnce(&mut Function, BlockId) -> InsnId, ) -> InsnId { - // Recompile when the runtime length changes so the call site can collect - // a new length profile instead of repeatedly taking the same side exit. + // Recompile after enough side exits have re-profiled the original Send. Any + // second observed length makes the distribution non-monomorphic, so the next + // version keeps the dynamic Send instead of emitting the same guard again. let length = self.push_insn(block, Insn::ArrayLength { array: caller_splat.array }); self.push_insn(block, Insn::GuardBitEquals { val: length, diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 86984403081e64..d1a46ce1ec88e8 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -15143,6 +15143,63 @@ mod hir_opt_tests { "); } + #[test] + fn dont_repeat_caller_splat_length_guard_for_skewed_polymorphic_profile() { + enable_zjit_stats(); + set_call_threshold(5); + set_max_versions(4); + // Profile length 1 on calls 1-4, then compile its monomorphic guard on call 5. + eval(" + def foo(*args) = args + def capture(*args) = args + ruby2_keywords(:capture) + def test(args) = foo(*args) + 5.times { test([1]) } + "); + + // Record a less frequent second length through the recompiling length guard. + eval("test([1, 2])"); + + // Finish the profile window with the first length. These calls exit through + // the non-recompiling ruby2_keywords guard, so the version remains active. + eval("4.times { test(capture(k: 1)) }"); + + // With the profile window complete, the next length mismatch invalidates + // the monomorphic version for recompilation. + eval("test([1, 2])"); + + // The next version must keep the dynamic Send because the accumulated + // length profile is skewed polymorphic rather than monomorphic. + assert_snapshot!(hir_string("test"), @" + fn test@:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_skewed_polymorphic + IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v24 + "); + } + #[test] fn dont_specialize_call_to_iseq_with_caller_splat_on_final_version() { enable_zjit_stats(); From 11d6a46a42473fa2bc011728ac6648b4fcfd7f64 Mon Sep 17 00:00:00 2001 From: nozomemein Date: Thu, 3 Sep 2026 00:30:16 +0900 Subject: [PATCH 6/8] ZJIT: Select caller-splat lengths while building HIR Select the profiled caller-splat length in add_iseq_to_hir and attach it to each Send before receiver dispatch creates its specialized arms. type_specialize then uses the selected length to build each SendDirect path. This prepares caller-splat dispatch for supporting multiple profiled lengths without making type_specialize responsible for building the outer dispatch CFG. --- zjit/src/hir.rs | 117 +++++++++++++++++--------------------- zjit/src/hir/opt_tests.rs | 77 +++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 64 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index cd9149536fba61..b567c8d2794c84 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -15,7 +15,7 @@ use std::{ use crate::hir_type::{Type, types}; use crate::hir_effect::{Effect, abstract_heaps, effects}; use crate::bitset::BitSet; -use crate::profile::{ProfiledType, SplatLength, SplatLengthDistributionSummary, TypeDistributionSummary}; +use crate::profile::{ProfiledType, SplatLength, TypeDistributionSummary}; use crate::stats::{Counter, incr_counter}; use SendFallbackReason::*; @@ -1171,6 +1171,8 @@ pub enum Insn { cd: *const rb_call_data, block: Option, args: Vec, + /// Caller-splat length selected by `add_iseq_to_hir`. + caller_splat_length: Option, state: InsnId, reason: SendFallbackReason, }, @@ -3889,14 +3891,6 @@ impl Function { } } - /// Return the caller splat length profile at the given Snapshot, if available. - /// These are historical observations, so specialized paths must still check - /// the runtime array length before expanding the splat. - fn profiled_splat_length_summary_at(&self, state: InsnId) -> Option { - let state = self.frame_state(state); - get_or_create_iseq_payload(state.iseq).profile.get_splat_length_summary(state.insn_idx) - } - /// Validate and normalize SendDirect arguments without emitting HIR. fn build_send_direct_args(&self, caller_args: &CallerArguments, caller_splat: Option, iseq: IseqPtr, has_block: bool) -> Result { can_direct_send(iseq, caller_args, has_block, caller_splat)?; @@ -3979,14 +3973,31 @@ impl Function { args } - /// Guard a monomorphic caller splat and generate its direct path in place. - fn dispatch_caller_splat( + /// Select the monomorphic caller-splat length while translating the Send. + /// The selected length is attached to every receiver dispatch arm so later + /// specialization does not need to read the profile again. + fn monomorphic_caller_splat_length(&self, ci: *const rb_callinfo, state: InsnId) -> Option { + if self.policy.no_side_exits { + return None; + } + if unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_SPLAT == 0 { + return None; + } + let frame_state = self.frame_state_ref(state); + let summary = get_or_create_iseq_payload(frame_state.iseq).profile.get_splat_length_summary(frame_state.insn_idx)?; + if !summary.is_monomorphic() { + return None; + } + summary.bucket(0) + } + + /// Guard the caller-splat length selected for this runtime path. + fn emit_caller_splat( &mut self, block: BlockId, caller_splat: CallerSplat, state: InsnId, - emit_optimized: impl FnOnce(&mut Function, BlockId) -> InsnId, - ) -> InsnId { + ) { // Recompile after enough side exits have re-profiled the original Send. Any // second observed length makes the distribution non-monomorphic, so the next // version keeps the dynamic Send instead of emitting the same guard again. @@ -4022,8 +4033,6 @@ impl Function { recompile: None, }); } - - emit_optimized(self, block) } /// Reorder keyword arguments to match the callee's expected order, and synthesize @@ -4621,7 +4630,7 @@ impl Function { self.try_rewrite_freeze(block, insn_id, recv, state), &Insn::Send { recv, block: None, ref args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => self.try_rewrite_uminus(block, insn_id, recv, state), - &Insn::Send { mut recv, cd, state, block: send_block, .. } => { + &Insn::Send { mut recv, cd, state, block: send_block, caller_splat_length, .. } => { let mut has_block = send_block.is_some(); let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), state) { ReceiverTypeResolution::StaticallyKnown { class } => (class, None), @@ -4754,21 +4763,9 @@ impl Function { // Count the profile shape for every caller-splat execution; // complex_arg_pass_caller_splat separately tracks fallbacks. self.count_caller_splat_profile(block, state); - // The final ISEQ version cannot recover from guard exits by - // recompiling, so keep the dynamic Send instead of specializing. - if self.policy.no_side_exits { - self.count(block, Counter::complex_arg_pass_caller_splat); - self.set_dynamic_send_reason(insn_id, ComplexArgPass); - self.push_insn_id(block, insn_id); continue; - } - // Expand a caller splat only when profiling observed one stable - // array length; otherwise keep the original dynamic Send. - // TODO: Support polymorphic caller-splat length profiles. - let profiled_length = match self.profiled_splat_length_summary_at(state) { - Some(summary) if summary.is_monomorphic() => summary.bucket(0), - Some(_) | None => None, - }; - let Some(length) = profiled_length else { + // `add_iseq_to_hir` selects caller-splat lengths before building + // receiver dispatch. A Send without a selected length stays dynamic. + let Some(length) = caller_splat_length else { self.count(block, Counter::complex_arg_pass_caller_splat); self.set_dynamic_send_reason(insn_id, ComplexArgPass); self.push_insn_id(block, insn_id); continue; @@ -4792,35 +4789,25 @@ impl Function { self.push_insn_id(block, insn_id); continue; } - let emit_optimized = move |function: &mut Function, block: BlockId| { - if caller_splat.is_some() { - // Count caller-splat executions that take this optimized path. - // This is a feature-specific counter, not part of optimized_send_count. - function.count(block, Counter::caller_splat_optimized); - } - - // Add PatchPoint for method redefinition - function.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); + if let Some(caller_splat) = caller_splat { + self.emit_caller_splat(block, caller_splat, state); + // Count caller-splat executions that take this optimized path. + // This is a feature-specific counter, not part of optimized_send_count. + self.count(block, Counter::caller_splat_optimized); + } - // Add GuardType for profiled receiver - let recv = if let Some(profiled_type) = profiled_type { - let recv = function.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); - function.insn_types[recv] = function.infer_type(recv); - recv - } else { - recv - }; + // Add PatchPoint for method redefinition + self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); - let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = - function.emit_send_direct_args(block, call, &args, send_frame_state); - function.try_inline_send_direct(block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))) - }; + // Add GuardType for profiled receiver + if let Some(profiled_type) = profiled_type { + recv = self.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state, recompile: Some(Recompile) }); + self.insn_types[recv] = self.infer_type(recv); + } - let replacement = if let Some(caller_splat) = caller_splat { - self.dispatch_caller_splat(block, caller_splat, state, emit_optimized) - } else { - emit_optimized(self, block) - }; + let SendDirectArgs { state: send_state, args: send_args, kw_bits, jit_entry_idx } = + self.emit_send_direct_args(block, call, &args, send_frame_state); + let replacement = self.try_inline_send_direct(block, Insn::SendDirect(Box::new(SendDirectData { recv, cd, cme, iseq, args: send_args, kw_bits, jit_entry_idx, state: send_state, block: send_block }))); self.make_equal_to(insn_id, replacement); } else if !has_block && def_type == VM_METHOD_TYPE_BMETHOD { let procv = unsafe { rb_get_def_bmethod_proc((*cme).def) }; @@ -9973,7 +9960,7 @@ fn add_iseq_to_hir( } let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length: None, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); } YARVINSN_opt_hash_freeze => { @@ -10100,6 +10087,7 @@ fn add_iseq_to_hir( let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; + let caller_splat_length = fun.monomorphic_caller_splat_length(call_info, exit_id); if let Some(summary) = fun.polymorphic_summary(&profiles, recv, exit_id) { let join_block = fun.new_block(insn_idx); @@ -10135,19 +10123,19 @@ fn add_iseq_to_hir( // exact type, and resolve_receiver_type prefers profiles over types. profiles.copy_entries_except(exit_id, snapshot, recv, fun); let refined_recv = fun.push_insn(iftrue_block, Insn::RefineType { val: recv, new_type: expected }); - let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: None, args: args.clone(), state: snapshot, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: None, args: args.clone(), caller_splat_length, state: snapshot, reason: Uncategorized(opcode.into()) }); fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); } // In the fallthrough case, do a generic interpreter send and then join. let reason = SendPolymorphicFallback; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length, state: exit_id, reason }); fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); state.stack_push(join_param); // Continue compilation from the join block at the next instruction. block = join_block; } else { // Maybe monomorphic; handled in type_specialize - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); } } @@ -10177,7 +10165,8 @@ fn add_iseq_to_hir( } else { None }; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let caller_splat_length = fun.monomorphic_caller_splat_length(call_info, exit_id); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); if let Some(BlockHandler::BlockIseq(blockiseq)) = block_handler { @@ -10627,7 +10616,7 @@ fn add_iseq_to_hir( fun.push_insn(block, Insn::GuardType { val: recv, guard_type: types::String, state: exit_id, recompile: None }) } else { let recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state: exit_id, recompile: None }); - fun.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], state: exit_id, reason: ObjToStringNotString }) + fun.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], caller_splat_length: None, state: exit_id, reason: ObjToStringNotString }) } } else { let has_type = fun.push_insn(block, Insn::HasType { val: recv, expected: types::String }); @@ -10644,7 +10633,7 @@ fn add_iseq_to_hir( fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![refined] })); // false block let refined = fun.push_insn(iffalse_block, Insn::RefineType { val: recv, new_type: types::NotString }); - let send = fun.push_insn(iffalse_block, Insn::Send { recv: refined, cd, block: None, args: vec![], state: exit_id, reason: ObjToStringNotString }); + let send = fun.push_insn(iffalse_block, Insn::Send { recv: refined, cd, block: None, args: vec![], caller_splat_length: None, state: exit_id, reason: ObjToStringNotString }); fun.push_insn(iffalse_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); // join block block = join_block; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index d1a46ce1ec88e8..f8a8fd6d0f7a14 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -14789,6 +14789,83 @@ mod hir_opt_tests { "); } + #[test] + fn specialize_polymorphic_receiver_with_monomorphic_caller_splat() { + set_call_threshold(4); + eval(" + class CallerSplatA + def target(*args) = args + end + class CallerSplatB + def target(*args) = args + end + def test(recv, args) = recv.target(*args) + test(CallerSplatA.new, [1]) + test(CallerSplatB.new, [2]) + test(CallerSplatA.new, [3]) + test(CallerSplatB.new, [4]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :recv@0x1000 + v4:BasicObject = LoadField v2, :args@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :recv@1 + v9:BasicObject = LoadArg :args@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = ToArray v13 + v22:CBool = HasType v12, ObjectSubclass[class_exact:CallerSplatA] + CondBranch v22, bb5(), bb6() + bb5(): + v25:ObjectSubclass[class_exact:CallerSplatA] = RefineType v12, ObjectSubclass[class_exact:CallerSplatA] + PatchPoint NoSingletonClass(CallerSplatA@0x1008) + v42:CInt64 = ArrayLength v19 + v43:CInt64[1] = GuardBitEquals v42, CInt64(1) recompile + v44:CInt64 = CCall v19, :rb_jit_ruby2_keywords_splat_p@0x1010 + v45:CInt64[0] = GuardBitEquals v44, CInt64(0) + PatchPoint MethodRedefined(CallerSplatA@0x1008, target@0x1011, cme:0x1018) + v47:CInt64[0] = Const CInt64(0) + v48:BasicObject = ArrayAref v19, v47 + v49:ArrayExact = NewArray v48 + PushInlineFrame :target, v25 (0x1040), num_args=1 + CheckInterrupts + PopInlineFrame + Jump bb4(v49) + bb6(): + v28:CBool = HasType v12, ObjectSubclass[class_exact:CallerSplatB] + CondBranch v28, bb7(), bb8() + bb7(): + v31:ObjectSubclass[class_exact:CallerSplatB] = RefineType v12, ObjectSubclass[class_exact:CallerSplatB] + PatchPoint NoSingletonClass(CallerSplatB@0x1060) + v53:CInt64 = ArrayLength v19 + v54:CInt64[1] = GuardBitEquals v53, CInt64(1) recompile + v55:CInt64 = CCall v19, :rb_jit_ruby2_keywords_splat_p@0x1010 + v56:CInt64[0] = GuardBitEquals v55, CInt64(0) + PatchPoint MethodRedefined(CallerSplatB@0x1060, target@0x1011, cme:0x1068) + v58:CInt64[0] = Const CInt64(0) + v59:BasicObject = ArrayAref v19, v58 + v60:ArrayExact = NewArray v59 + PushInlineFrame :target, v31 (0x1090), num_args=1 + CheckInterrupts + PopInlineFrame + Jump bb4(v60) + bb8(): + v34:BasicObject = Send v12, :target, v19 # SendFallbackReason: Send: polymorphic call site + Jump bb4(v34) + bb4(v21:BasicObject): + CheckInterrupts + Return v21 + "); + } + #[test] fn specialize_call_to_iseq_with_empty_caller_splat() { eval(" From 59595825ce5e64bb1e83f4f1d8744875788f00cb Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 1 Sep 2026 20:21:51 +0900 Subject: [PATCH 7/8] Define IMEMO_MASK from FL_USER bits --- .gdbinit | 6 +++--- ext/objspace/objspace.c | 2 +- internal/imemo.h | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gdbinit b/.gdbinit index f045b12fb0f913..9281744c137b63 100644 --- a/.gdbinit +++ b/.gdbinit @@ -237,7 +237,7 @@ define rp else if ($flags & RUBY_T_MASK) == RUBY_T_IMEMO printf "%sT_IMEMO%s(", $color_type, $color_end - output (enum imemo_type)(($flags>>RUBY_FL_USHIFT)&RUBY_IMEMO_MASK) + output (enum imemo_type)(($flags&RUBY_IMEMO_MASK)>>RUBY_FL_USHIFT) printf "): " rp_imemo $arg0 else @@ -539,7 +539,7 @@ document rp_class end define rp_imemo - set $flags = (enum imemo_type)((((struct RBasic *)($arg0))->flags >> RUBY_FL_USHIFT) & RUBY_IMEMO_MASK) + set $flags = (enum imemo_type)((((struct RBasic *)($arg0))->flags & RUBY_IMEMO_MASK) >> RUBY_FL_USHIFT) if $flags == imemo_cref printf "(rb_cref_t *) %p\n", (void*)$arg0 print *(rb_cref_t *)$arg0 @@ -1100,7 +1100,7 @@ define rb_ps_thread while $cfp < $cfpend if $cfp->_iseq set $iseq = rb_get_cfp_iseq($cfp) - if !((VALUE)$iseq & RUBY_IMMEDIATE_MASK) && (((imemo_ifunc << RUBY_FL_USHIFT) | RUBY_T_IMEMO)==$iseq->flags & ((RUBY_IMEMO_MASK << RUBY_FL_USHIFT) | RUBY_T_MASK)) + if !((VALUE)$iseq & RUBY_IMMEDIATE_MASK) && (((imemo_ifunc << RUBY_FL_USHIFT) | RUBY_T_IMEMO)==$iseq->flags & (RUBY_IMEMO_MASK | RUBY_T_MASK)) printf "%d:ifunc ", $cfpend-$cfp set print symbol-filename on output/a $iseq.body diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c index 753f77a6010904..a05d491e35b57a 100644 --- a/ext/objspace/objspace.c +++ b/ext/objspace/objspace.c @@ -403,7 +403,7 @@ count_tdata_objects(int argc, VALUE *argv, VALUE self) return hash; } -static ID imemo_type_ids[IMEMO_MASK+1]; +static ID imemo_type_ids[(IMEMO_MASK >> FL_USHIFT) + 1]; static void count_imemo_objects_i(VALUE v, void *data) diff --git a/internal/imemo.h b/internal/imemo.h index fee5c07bc33703..68e253e7ff1ae5 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -15,7 +15,7 @@ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for rb_block_call_func_t */ -#define IMEMO_MASK 0x0f +#define IMEMO_MASK (FL_USER0 | FL_USER1 | FL_USER2 | FL_USER3) /* FL_USER0 to FL_USER3 is for type */ #define IMEMO_FL_USHIFT (FL_USHIFT + 4) @@ -171,7 +171,7 @@ RUBY_SYMBOL_EXPORT_END static inline enum imemo_type imemo_type(VALUE imemo) { - return (RBASIC(imemo)->flags >> FL_USHIFT) & IMEMO_MASK; + return (RBASIC(imemo)->flags & IMEMO_MASK) >> FL_USHIFT; } static inline int @@ -179,7 +179,7 @@ imemo_type_p(VALUE imemo, enum imemo_type imemo_type) { if (LIKELY(!RB_SPECIAL_CONST_P(imemo))) { /* fixed at compile time if imemo_type is given. */ - const VALUE mask = (IMEMO_MASK << FL_USHIFT) | RUBY_T_MASK; + const VALUE mask = IMEMO_MASK | RUBY_T_MASK; const VALUE expected_type = (imemo_type << FL_USHIFT) | T_IMEMO; /* fixed at runtime. */ return expected_type == (RBASIC(imemo)->flags & mask); From 1d8d84b3d6c27bab724eb2e27b05c2581e0bc3e4 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 2 Sep 2026 09:45:15 +0900 Subject: [PATCH 8/8] Increase IMEMO_MASK by one bit We are out of imemo types because IMEMO_MASK uses 4 bits and we have 16 imemo types. This makes it not possible to experiment with new imemo types, so we can increase it by one bit to have 16 more imemo types. --- internal/imemo.h | 22 +++++++++++----------- misc/lldb_cruby.py | 2 +- misc/lldb_rb/constants.py | 2 +- tool/timeline/lib/converter_defs.rb | 2 +- yjit/src/cruby.rs | 2 +- zjit/src/cruby.rs | 2 +- 6 files changed, 16 insertions(+), 16 deletions(-) diff --git a/internal/imemo.h b/internal/imemo.h index 68e253e7ff1ae5..02fb0a131e1345 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -15,17 +15,17 @@ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for rb_block_call_func_t */ -#define IMEMO_MASK (FL_USER0 | FL_USER1 | FL_USER2 | FL_USER3) - -/* FL_USER0 to FL_USER3 is for type */ -#define IMEMO_FL_USHIFT (FL_USHIFT + 4) -#define IMEMO_FL_USER0 FL_USER4 -#define IMEMO_FL_USER1 FL_USER5 -#define IMEMO_FL_USER2 FL_USER6 -#define IMEMO_FL_USER3 FL_USER7 -#define IMEMO_FL_USER4 FL_USER8 -#define IMEMO_FL_USER5 FL_USER9 -#define IMEMO_FL_USER6 FL_USER10 +#define IMEMO_MASK (FL_USER0 | FL_USER1 | FL_USER2 | FL_USER3 | FL_USER4) + +/* FL_USER0 to FL_USER4 is for type */ +#define IMEMO_FL_USHIFT (FL_USHIFT + 5) +#define IMEMO_FL_USER0 FL_USER5 +#define IMEMO_FL_USER1 FL_USER6 +#define IMEMO_FL_USER2 FL_USER7 +#define IMEMO_FL_USER3 FL_USER8 +#define IMEMO_FL_USER4 FL_USER9 +#define IMEMO_FL_USER5 FL_USER10 +#define IMEMO_FL_USER6 FL_USER11 enum imemo_type { imemo_env = 0, diff --git a/misc/lldb_cruby.py b/misc/lldb_cruby.py index b3d4fb509add14..2eec8cfce75724 100644 --- a/misc/lldb_cruby.py +++ b/misc/lldb_cruby.py @@ -417,7 +417,7 @@ def lldb_inspect(debugger, target, result, val): append_expression(debugger, "*(struct RMatch *) %0#x" % val.GetValueAsUnsigned(), result) elif flType == RUBY_T_IMEMO: # I'm not sure how to get IMEMO_MASK out of lldb. It's not in globals() - imemo_type = (flags >> RUBY_FL_USHIFT) & 0x0F # IMEMO_MASK + imemo_type = (flags >> RUBY_FL_USHIFT) & 0x1F # IMEMO_MASK print("T_IMEMO: ", file=result) append_expression(debugger, "(enum imemo_type) %d" % imemo_type, result) diff --git a/misc/lldb_rb/constants.py b/misc/lldb_rb/constants.py index 9cd56eccb0ebdc..c3132c366b25f8 100644 --- a/misc/lldb_rb/constants.py +++ b/misc/lldb_rb/constants.py @@ -3,4 +3,4 @@ HEAP_PAGE_ALIGN = (1 << HEAP_PAGE_ALIGN_LOG) HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN -IMEMO_MASK = 0x0F +IMEMO_MASK = 0x1F diff --git a/tool/timeline/lib/converter_defs.rb b/tool/timeline/lib/converter_defs.rb index e4cd2bc079bfc1..46a19fe27457b9 100644 --- a/tool/timeline/lib/converter_defs.rb +++ b/tool/timeline/lib/converter_defs.rb @@ -85,7 +85,7 @@ def self.FL_USER_N(n) }) # Keep in sync with `IMEMO_MASK` in `internal/imemo.h`. - IMEMO_MASK = 0x0f + IMEMO_MASK = 0x1f # Keep in sync with both `internal/string.h` and `include/ruby/internal/core/rstring.h`. StringFlags = FlagsConverter.new({ diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index c97e50ac1cc18d..dc8b3200aa9fd5 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -754,7 +754,7 @@ mod manual_defs { pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; // From iseq.h - via a different constant, which seems to confuse bindgen - pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER8 as usize; // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants. diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index e144f21a7a17d9..2111266500df4a 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1224,7 +1224,7 @@ mod manual_defs { pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; // From iseq.h - via a different constant, which seems to confuse bindgen - pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER8 as usize; // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants.