Ruby: vm_insnhelper.c Source File

Ruby 4.0.6p0 (2026-07-14 revision 03b6d3f8898a28604fe6cb00eae3226b821168f4)

vm_insnhelper.c
1/**********************************************************************
2
3 vm_insnhelper.c - instruction helper functions. Included into vm.c.
4
5 $Author$
6
7 Copyright (C) 2007 Koichi Sasada
8
9**********************************************************************/
10
11#include "ruby/internal/config.h"
12
13#include <math.h>
14
15#ifdef HAVE_STDATOMIC_H
16 #include <stdatomic.h>
17#endif
18
19#include "constant.h"
20#include "debug_counter.h"
21#include "internal.h"
22#include "internal/class.h"
23#include "internal/compar.h"
24#include "internal/hash.h"
25#include "internal/numeric.h"
26#include "internal/proc.h"
27#include "internal/random.h"
28#include "internal/variable.h"
29#include "internal/set_table.h"
30#include "internal/struct.h"
31#include "variable.h"
32
33/* finish iseq array */
34#include "insns.inc"
35#include "insns_info.inc"
36
37extern rb_method_definition_t *rb_method_definition_create(rb_method_type_t type, ID mid);
38extern void rb_method_definition_set(const rb_method_entry_t *me, rb_method_definition_t *def, void *opts);
39extern int rb_method_definition_eq(const rb_method_definition_t *d1, const rb_method_definition_t *d2);
40extern VALUE rb_make_no_method_exception(VALUE exc, VALUE format, VALUE obj,
41 int argc, const VALUE *argv, int priv);
42
43static const struct rb_callcache vm_empty_cc;
44static const struct rb_callcache vm_empty_cc_for_super;
45
46/* control stack frame */
47
48static rb_control_frame_t *vm_get_ruby_level_caller_cfp(const rb_execution_context_t *ec, const rb_control_frame_t *cfp);
49
50 VALUE
51ruby_vm_special_exception_copy(VALUE exc)
52{
53 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
54 rb_obj_copy_ivar(e, exc);
55 return e;
56}
57
58NORETURN(static void ec_stack_overflow(rb_execution_context_t *ec, int));
59static void
60ec_stack_overflow(rb_execution_context_t *ec, int setup)
61{
62 VALUE mesg = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_sysstack];
63 ec->raised_flag = RAISED_STACKOVERFLOW;
64 if (setup) {
65 VALUE at = rb_ec_backtrace_object(ec);
66 mesg = ruby_vm_special_exception_copy(mesg);
67 rb_ivar_set(mesg, idBt, at);
68 rb_ivar_set(mesg, idBt_locations, at);
69 }
70 ec->errinfo = mesg;
71 EC_JUMP_TAG(ec, TAG_RAISE);
72}
73
74NORETURN(static void vm_stackoverflow(void));
75
76static void
77vm_stackoverflow(void)
78{
79 ec_stack_overflow(GET_EC(), TRUE);
80}
81
82void
83rb_ec_stack_overflow(rb_execution_context_t *ec, ruby_stack_overflow_critical_level crit)
84{
85 if (rb_during_gc()) {
86 rb_bug("system stack overflow during GC. Faulty native extension?");
87 }
88 if (crit >= rb_stack_overflow_fatal) {
89 ec->raised_flag = RAISED_STACKOVERFLOW;
90 ec->errinfo = rb_ec_vm_ptr(ec)->special_exceptions[ruby_error_stackfatal];
91 EC_JUMP_TAG(ec, TAG_RAISE);
92 }
93 ec_stack_overflow(ec, crit < rb_stack_overflow_signal);
94}
95
96static inline void stack_check(rb_execution_context_t *ec);
97
98#if VM_CHECK_MODE > 0
99static int
100callable_class_p(VALUE klass)
101{
102#if VM_CHECK_MODE >= 2
103 if (!klass) return FALSE;
104 switch (RB_BUILTIN_TYPE(klass)) {
105 default:
106 break;
107 case T_ICLASS:
108 if (!RB_TYPE_P(RCLASS_SUPER(klass), T_MODULE)) break;
109 case T_MODULE:
110 return TRUE;
111 }
112 while (klass) {
113 if (klass == rb_cBasicObject) {
114 return TRUE;
115 }
116 klass = RCLASS_SUPER(klass);
117 }
118 return FALSE;
119#else
120 return klass != 0;
121#endif
122}
123
124static int
125callable_method_entry_p(const rb_callable_method_entry_t *cme)
126{
127 if (cme == NULL) {
128 return TRUE;
129 }
130 else {
131 VM_ASSERT(IMEMO_TYPE_P((VALUE)cme, imemo_ment), "imemo_type:%s", rb_imemo_name(imemo_type((VALUE)cme)));
132
133 if (callable_class_p(cme->defined_class)) {
134 return TRUE;
135 }
136 else {
137 return FALSE;
138 }
139 }
140}
141
142static void
143vm_check_frame_detail(VALUE type, int req_block, int req_me, int req_cref, VALUE specval, VALUE cref_or_me, int is_cframe, const rb_iseq_t *iseq)
144{
145 unsigned int magic = (unsigned int)(type & VM_FRAME_MAGIC_MASK);
146 enum imemo_type cref_or_me_type = imemo_env; /* impossible value */
147
148 if (RB_TYPE_P(cref_or_me, T_IMEMO)) {
149 cref_or_me_type = imemo_type(cref_or_me);
150 }
151 if (type & VM_FRAME_FLAG_BMETHOD) {
152 req_me = TRUE;
153 }
154
155 if (req_block && (type & VM_ENV_FLAG_LOCAL) == 0) {
156 rb_bug("vm_push_frame: specval (%p) should be a block_ptr on %x frame", (void *)specval, magic);
157 }
158 if (!req_block && (type & VM_ENV_FLAG_LOCAL) != 0) {
159 rb_bug("vm_push_frame: specval (%p) should not be a block_ptr on %x frame", (void *)specval, magic);
160 }
161
162 if (req_me) {
163 if (cref_or_me_type != imemo_ment) {
164 rb_bug("vm_push_frame: (%s) should be method entry on %x frame", rb_obj_info(cref_or_me), magic);
165 }
166 }
167 else {
168 if (req_cref && cref_or_me_type != imemo_cref) {
169 rb_bug("vm_push_frame: (%s) should be CREF on %x frame", rb_obj_info(cref_or_me), magic);
170 }
171 else { /* cref or Qfalse */
172 if (cref_or_me != Qfalse && cref_or_me_type != imemo_cref) {
173 if (((type & VM_FRAME_FLAG_LAMBDA) || magic == VM_FRAME_MAGIC_IFUNC || magic == VM_FRAME_MAGIC_DUMMY) && (cref_or_me_type == imemo_ment)) {
174 /* ignore */
175 }
176 else {
177 rb_bug("vm_push_frame: (%s) should be false or cref on %x frame", rb_obj_info(cref_or_me), magic);
178 }
179 }
180 }
181 }
182
183 if (cref_or_me_type == imemo_ment) {
184 const rb_callable_method_entry_t *me = (const rb_callable_method_entry_t *)cref_or_me;
185
186 if (!callable_method_entry_p(me)) {
187 rb_bug("vm_push_frame: ment (%s) should be callable on %x frame.", rb_obj_info(cref_or_me), magic);
188 }
189 }
190
191 if ((type & VM_FRAME_MAGIC_MASK) == VM_FRAME_MAGIC_DUMMY) {
192 VM_ASSERT(iseq == NULL ||
193 RBASIC_CLASS((VALUE)iseq) == 0 || // dummy frame for loading
194 RUBY_VM_NORMAL_ISEQ_P(iseq) //argument error
195 );
196 }
197 else {
198 VM_ASSERT(is_cframe == !RUBY_VM_NORMAL_ISEQ_P(iseq));
199 }
200}
201
202static void
203vm_check_frame(VALUE type,
204 VALUE specval,
205 VALUE cref_or_me,
206 const rb_iseq_t *iseq)
207{
208 VALUE given_magic = type & VM_FRAME_MAGIC_MASK;
209 VM_ASSERT(FIXNUM_P(type));
210
211#define CHECK(magic, req_block, req_me, req_cref, is_cframe) \
212 case magic: \
213 vm_check_frame_detail(type, req_block, req_me, req_cref, \
214 specval, cref_or_me, is_cframe, iseq); \
215 break
216 switch (given_magic) {
217 /* BLK ME CREF CFRAME */
218 CHECK(VM_FRAME_MAGIC_METHOD, TRUE, TRUE, FALSE, FALSE);
219 CHECK(VM_FRAME_MAGIC_CLASS, TRUE, FALSE, TRUE, FALSE);
220 CHECK(VM_FRAME_MAGIC_TOP, TRUE, FALSE, TRUE, FALSE);
221 CHECK(VM_FRAME_MAGIC_CFUNC, TRUE, TRUE, FALSE, TRUE);
222 CHECK(VM_FRAME_MAGIC_BLOCK, FALSE, FALSE, FALSE, FALSE);
223 CHECK(VM_FRAME_MAGIC_IFUNC, FALSE, FALSE, FALSE, TRUE);
224 CHECK(VM_FRAME_MAGIC_EVAL, FALSE, FALSE, FALSE, FALSE);
225 CHECK(VM_FRAME_MAGIC_RESCUE, FALSE, FALSE, FALSE, FALSE);
226 CHECK(VM_FRAME_MAGIC_DUMMY, TRUE, FALSE, FALSE, FALSE);
227 default:
228 rb_bug("vm_push_frame: unknown type (%x)", (unsigned int)given_magic);
229 }
230#undef CHECK
231}
232
233static VALUE vm_stack_canary; /* Initialized later */
234static bool vm_stack_canary_was_born = false;
235
236// Return the index of the instruction right before the given PC.
237// This is needed because insn_entry advances PC before the insn body.
238static unsigned int
239previous_insn_index(const rb_iseq_t *iseq, const VALUE *pc)
240{
241 unsigned int pos = 0;
242 while (pos < ISEQ_BODY(iseq)->iseq_size) {
243 int opcode = rb_vm_insn_addr2opcode((void *)ISEQ_BODY(iseq)->iseq_encoded[pos]);
244 unsigned int next_pos = pos + insn_len(opcode);
245 if (ISEQ_BODY(iseq)->iseq_encoded + next_pos == pc) {
246 return pos;
247 }
248 pos = next_pos;
249 }
250 rb_bug("failed to find the previous insn");
251}
252
253void
254rb_vm_check_canary(const rb_execution_context_t *ec, VALUE *sp)
255{
256 const struct rb_control_frame_struct *reg_cfp = ec->cfp;
257 const struct rb_iseq_struct *iseq;
258
259 if (! LIKELY(vm_stack_canary_was_born)) {
260 return; /* :FIXME: isn't it rather fatal to enter this branch? */
261 }
262 else if ((VALUE *)reg_cfp == ec->vm_stack + ec->vm_stack_size) {
263 /* This is at the very beginning of a thread. cfp does not exist. */
264 return;
265 }
266 else if (! (iseq = GET_ISEQ())) {
267 return;
268 }
269 else if (LIKELY(sp[0] != vm_stack_canary)) {
270 return;
271 }
272 else {
273 /* we are going to call methods below; squash the canary to
274 * prevent infinite loop. */
275 sp[0] = Qundef;
276 }
277
278 const VALUE *orig = rb_iseq_original_iseq(iseq);
279 const VALUE iseqw = rb_iseqw_new(iseq);
280 const VALUE inspection = rb_inspect(iseqw);
281 const char *stri = rb_str_to_cstr(inspection);
282 const VALUE disasm = rb_iseq_disasm(iseq);
283 const char *strd = rb_str_to_cstr(disasm);
284 const ptrdiff_t pos = previous_insn_index(iseq, GET_PC());
285 const enum ruby_vminsn_type insn = (enum ruby_vminsn_type)orig[pos];
286 const char *name = insn_name(insn);
287
288 /* rb_bug() is not capable of outputting this large contents. It
289 is designed to run form a SIGSEGV handler, which tends to be
290 very restricted. */
291 ruby_debug_printf(
292 "We are killing the stack canary set by %s, "
293 "at %s@pc=%"PRIdPTR"\n"
294 "watch out the C stack trace.\n"
295 "%s",
296 name, stri, pos, strd);
297 rb_bug("see above.");
298}
299#define vm_check_canary(ec, sp) rb_vm_check_canary(ec, sp)
300
301#else
302#define vm_check_canary(ec, sp)
303#define vm_check_frame(a, b, c, d)
304#endif /* VM_CHECK_MODE > 0 */
305
306#if USE_DEBUG_COUNTER
307static void
308vm_push_frame_debug_counter_inc(
309 const struct rb_execution_context_struct *ec,
310 const struct rb_control_frame_struct *reg_cfp,
311 VALUE type)
312{
313 const struct rb_control_frame_struct *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(reg_cfp);
314
315 RB_DEBUG_COUNTER_INC(frame_push);
316
317 if (RUBY_VM_END_CONTROL_FRAME(ec) != prev_cfp) {
318 const bool curr = VM_FRAME_RUBYFRAME_P(reg_cfp);
319 const bool prev = VM_FRAME_RUBYFRAME_P(prev_cfp);
320 if (prev) {
321 if (curr) {
322 RB_DEBUG_COUNTER_INC(frame_R2R);
323 }
324 else {
325 RB_DEBUG_COUNTER_INC(frame_R2C);
326 }
327 }
328 else {
329 if (curr) {
330 RB_DEBUG_COUNTER_INC(frame_C2R);
331 }
332 else {
333 RB_DEBUG_COUNTER_INC(frame_C2C);
334 }
335 }
336 }
337
338 switch (type & VM_FRAME_MAGIC_MASK) {
339 case VM_FRAME_MAGIC_METHOD: RB_DEBUG_COUNTER_INC(frame_push_method); return;
340 case VM_FRAME_MAGIC_BLOCK: RB_DEBUG_COUNTER_INC(frame_push_block); return;
341 case VM_FRAME_MAGIC_CLASS: RB_DEBUG_COUNTER_INC(frame_push_class); return;
342 case VM_FRAME_MAGIC_TOP: RB_DEBUG_COUNTER_INC(frame_push_top); return;
343 case VM_FRAME_MAGIC_CFUNC: RB_DEBUG_COUNTER_INC(frame_push_cfunc); return;
344 case VM_FRAME_MAGIC_IFUNC: RB_DEBUG_COUNTER_INC(frame_push_ifunc); return;
345 case VM_FRAME_MAGIC_EVAL: RB_DEBUG_COUNTER_INC(frame_push_eval); return;
346 case VM_FRAME_MAGIC_RESCUE: RB_DEBUG_COUNTER_INC(frame_push_rescue); return;
347 case VM_FRAME_MAGIC_DUMMY: RB_DEBUG_COUNTER_INC(frame_push_dummy); return;
348 }
349
350 rb_bug("unreachable");
351}
352#else
353#define vm_push_frame_debug_counter_inc(ec, cfp, t) /* void */
354#endif
355
356// Return a poison value to be set above the stack top to verify leafness.
357 VALUE
358rb_vm_stack_canary(void)
359{
360#if VM_CHECK_MODE > 0
361 return vm_stack_canary;
362#else
363 return 0;
364#endif
365}
366
367STATIC_ASSERT(VM_ENV_DATA_INDEX_ME_CREF, VM_ENV_DATA_INDEX_ME_CREF == -2);
368STATIC_ASSERT(VM_ENV_DATA_INDEX_SPECVAL, VM_ENV_DATA_INDEX_SPECVAL == -1);
369STATIC_ASSERT(VM_ENV_DATA_INDEX_FLAGS, VM_ENV_DATA_INDEX_FLAGS == -0);
370
371static void
372vm_push_frame(rb_execution_context_t *ec,
373 const rb_iseq_t *iseq,
374 VALUE type,
375 VALUE self,
376 VALUE specval,
377 VALUE cref_or_me,
378 const VALUE *pc,
379 VALUE *sp,
380 int local_size,
381 int stack_max)
382{
383 rb_control_frame_t *const cfp = RUBY_VM_NEXT_CONTROL_FRAME(ec->cfp);
384
385 vm_check_frame(type, specval, cref_or_me, iseq);
386 VM_ASSERT(local_size >= 0);
387
388 /* check stack overflow */
389 CHECK_VM_STACK_OVERFLOW0(cfp, sp, local_size + stack_max);
390 vm_check_canary(ec, sp);
391
392 /* setup vm value stack */
393
394 /* initialize local variables */
395 for (int i=0; i < local_size; i++) {
396 *sp++ = Qnil;
397 }
398
399 /* setup ep with managing data */
400 *sp++ = cref_or_me; /* ep[-2] / Qnil or T_IMEMO(cref) or T_IMEMO(ment) */
401 *sp++ = specval /* ep[-1] / block handler or prev env ptr */;
402 *sp++ = type; /* ep[-0] / ENV_FLAGS */
403
404 /* setup new frame */
405 *cfp = (const struct rb_control_frame_struct) {
406 .pc = pc,
407 .sp = sp,
408 .iseq = iseq,
409 .self = self,
410 .ep = sp - 1,
411 .block_code = NULL,
412#if VM_DEBUG_BP_CHECK
413 .bp_check = sp,
414#endif
415 .jit_return = NULL,
416 };
417
418 /* Ensure the initialization of `*cfp` above never gets reordered with the update of `ec->cfp` below.
419 This is a no-op in all cases we've looked at (https://godbolt.org/z/3oxd1446K), but should guarantee it for all
420 future/untested compilers/platforms. */
421
422 #if defined HAVE_DECL_ATOMIC_SIGNAL_FENCE && HAVE_DECL_ATOMIC_SIGNAL_FENCE
423 atomic_signal_fence(memory_order_seq_cst);
424 #endif
425
426 ec->cfp = cfp;
427
428 if (VMDEBUG == 2) {
429 SDR();
430 }
431 vm_push_frame_debug_counter_inc(ec, cfp, type);
432}
433
434void
435rb_vm_pop_frame_no_int(rb_execution_context_t *ec)
436{
437 rb_control_frame_t *cfp = ec->cfp;
438
439 if (VMDEBUG == 2) SDR();
440
441 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
442}
443
444/* return TRUE if the frame is finished */
445static inline int
446vm_pop_frame(rb_execution_context_t *ec, rb_control_frame_t *cfp, const VALUE *ep)
447{
448 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
449
450 if (VMDEBUG == 2) SDR();
451
452 RUBY_VM_CHECK_INTS(ec);
453 ec->cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
454
455 return flags & VM_FRAME_FLAG_FINISH;
456}
457
458void
459rb_vm_pop_frame(rb_execution_context_t *ec)
460{
461 vm_pop_frame(ec, ec->cfp, ec->cfp->ep);
462}
463
464// it pushes pseudo-frame with fname filename.
465 VALUE
466rb_vm_push_frame_fname(rb_execution_context_t *ec, VALUE fname)
467{
468 rb_iseq_t *rb_iseq_alloc_with_dummy_path(VALUE fname);
469 rb_iseq_t *dmy_iseq = rb_iseq_alloc_with_dummy_path(fname);
470
471 vm_push_frame(ec,
472 dmy_iseq, //const rb_iseq_t *iseq,
473 VM_FRAME_MAGIC_DUMMY | VM_ENV_FLAG_LOCAL | VM_FRAME_FLAG_FINISH, // VALUE type,
474 ec->cfp->self, // VALUE self,
475 VM_BLOCK_HANDLER_NONE, // VALUE specval,
476 Qfalse, // VALUE cref_or_me,
477 NULL, // const VALUE *pc,
478 ec->cfp->sp, // VALUE *sp,
479 0, // int local_size,
480 0); // int stack_max
481
482 return (VALUE)dmy_iseq;
483}
484
485/* method dispatch */
486static inline VALUE
487rb_arity_error_new(int argc, int min, int max)
488{
489 VALUE err_mess = rb_sprintf("wrong number of arguments (given %d, expected %d", argc, min);
490 if (min == max) {
491 /* max is not needed */
492 }
493 else if (max == UNLIMITED_ARGUMENTS) {
494 rb_str_cat_cstr(err_mess, "+");
495 }
496 else {
497 rb_str_catf(err_mess, "..%d", max);
498 }
499 rb_str_cat_cstr(err_mess, ")");
500 return rb_exc_new3(rb_eArgError, err_mess);
501}
502
503void
504rb_error_arity(int argc, int min, int max)
505{
506 rb_exc_raise(rb_arity_error_new(argc, min, max));
507}
508
509/* lvar */
510
511NOINLINE(static void vm_env_write_slowpath(const VALUE *ep, int index, VALUE v));
512
513static void
514vm_env_write_slowpath(const VALUE *ep, int index, VALUE v)
515{
516 /* remember env value forcely */
517 rb_gc_writebarrier_remember(VM_ENV_ENVVAL(ep));
518 VM_FORCE_WRITE(&ep[index], v);
519 VM_ENV_FLAGS_UNSET(ep, VM_ENV_FLAG_WB_REQUIRED);
520 RB_DEBUG_COUNTER_INC(lvar_set_slowpath);
521}
522
523// YJIT assumes this function never runs GC
524static inline void
525vm_env_write(const VALUE *ep, int index, VALUE v)
526{
527 VALUE flags = ep[VM_ENV_DATA_INDEX_FLAGS];
528 if (LIKELY((flags & VM_ENV_FLAG_WB_REQUIRED) == 0)) {
529 VM_STACK_ENV_WRITE(ep, index, v);
530 }
531 else {
532 vm_env_write_slowpath(ep, index, v);
533 }
534}
535
536void
537rb_vm_env_write(const VALUE *ep, int index, VALUE v)
538{
539 vm_env_write(ep, index, v);
540}
541
542 VALUE
543rb_vm_bh_to_procval(const rb_execution_context_t *ec, VALUE block_handler)
544{
545 if (block_handler == VM_BLOCK_HANDLER_NONE) {
546 return Qnil;
547 }
548 else {
549 switch (vm_block_handler_type(block_handler)) {
550 case block_handler_type_iseq:
551 case block_handler_type_ifunc:
552 return rb_vm_make_proc(ec, VM_BH_TO_CAPT_BLOCK(block_handler), rb_cProc);
553 case block_handler_type_symbol:
554 return rb_sym_to_proc(VM_BH_TO_SYMBOL(block_handler));
555 case block_handler_type_proc:
556 return VM_BH_TO_PROC(block_handler);
557 default:
558 VM_UNREACHABLE(rb_vm_bh_to_procval);
559 }
560 }
561}
562
563/* svar */
564
565#if VM_CHECK_MODE > 0
566static int
567vm_svar_valid_p(VALUE svar)
568{
569 if (RB_TYPE_P((VALUE)svar, T_IMEMO)) {
570 switch (imemo_type(svar)) {
571 case imemo_svar:
572 case imemo_cref:
573 case imemo_ment:
574 return TRUE;
575 default:
576 break;
577 }
578 }
579 rb_bug("vm_svar_valid_p: unknown type: %s", rb_obj_info(svar));
580 return FALSE;
581}
582#endif
583
584static inline struct vm_svar *
585lep_svar(const rb_execution_context_t *ec, const VALUE *lep)
586{
587 VALUE svar;
588
589 if (lep && (ec == NULL || ec->root_lep != lep)) {
590 svar = lep[VM_ENV_DATA_INDEX_ME_CREF];
591 }
592 else {
593 svar = ec->root_svar;
594 }
595
596 VM_ASSERT(svar == Qfalse || vm_svar_valid_p(svar));
597
598 return (struct vm_svar *)svar;
599}
600
601static inline void
602lep_svar_write(const rb_execution_context_t *ec, const VALUE *lep, const struct vm_svar *svar)
603{
604 VM_ASSERT(vm_svar_valid_p((VALUE)svar));
605
606 if (lep && (ec == NULL || ec->root_lep != lep)) {
607 vm_env_write(lep, VM_ENV_DATA_INDEX_ME_CREF, (VALUE)svar);
608 }
609 else {
610 RB_OBJ_WRITE(rb_ec_thread_ptr(ec)->self, &ec->root_svar, svar);
611 }
612}
613
614static VALUE
615lep_svar_get(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key)
616{
617 const struct vm_svar *svar = lep_svar(ec, lep);
618
619 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) return Qnil;
620
621 switch (key) {
622 case VM_SVAR_LASTLINE:
623 return svar->lastline;
624 case VM_SVAR_BACKREF:
625 return svar->backref;
626 default: {
627 const VALUE ary = svar->others;
628
629 if (NIL_P(ary)) {
630 return Qnil;
631 }
632 else {
633 return rb_ary_entry(ary, key - VM_SVAR_EXTRA_START);
634 }
635 }
636 }
637}
638
639static struct vm_svar *
640svar_new(VALUE obj)
641{
642 struct vm_svar *svar = IMEMO_NEW(struct vm_svar, imemo_svar, obj);
643 *((VALUE *)&svar->lastline) = Qnil;
644 *((VALUE *)&svar->backref) = Qnil;
645 *((VALUE *)&svar->others) = Qnil;
646
647 return svar;
648}
649
650static void
651lep_svar_set(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, VALUE val)
652{
653 struct vm_svar *svar = lep_svar(ec, lep);
654
655 if ((VALUE)svar == Qfalse || imemo_type((VALUE)svar) != imemo_svar) {
656 lep_svar_write(ec, lep, svar = svar_new((VALUE)svar));
657 }
658
659 switch (key) {
660 case VM_SVAR_LASTLINE:
661 RB_OBJ_WRITE(svar, &svar->lastline, val);
662 return;
663 case VM_SVAR_BACKREF:
664 RB_OBJ_WRITE(svar, &svar->backref, val);
665 return;
666 default: {
667 VALUE ary = svar->others;
668
669 if (NIL_P(ary)) {
670 RB_OBJ_WRITE(svar, &svar->others, ary = rb_ary_new());
671 }
672 rb_ary_store(ary, key - VM_SVAR_EXTRA_START, val);
673 }
674 }
675}
676
677static inline VALUE
678vm_getspecial(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t key, rb_num_t type)
679{
680 VALUE val;
681
682 if (type == 0) {
683 val = lep_svar_get(ec, lep, key);
684 }
685 else {
686 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
687
688 if (type & 0x01) {
689 switch (type >> 1) {
690 case '&':
691 val = rb_reg_last_match(backref);
692 break;
693 case '`':
694 val = rb_reg_match_pre(backref);
695 break;
696 case '\'':
697 val = rb_reg_match_post(backref);
698 break;
699 case '+':
700 val = rb_reg_match_last(backref);
701 break;
702 default:
703 rb_bug("unexpected back-ref");
704 }
705 }
706 else {
707 val = rb_reg_nth_match((int)(type >> 1), backref);
708 }
709 }
710 return val;
711}
712
713static inline VALUE
714vm_backref_defined(const rb_execution_context_t *ec, const VALUE *lep, rb_num_t type)
715{
716 VALUE backref = lep_svar_get(ec, lep, VM_SVAR_BACKREF);
717 int nth = 0;
718
719 if (type & 0x01) {
720 switch (type >> 1) {
721 case '&':
722 case '`':
723 case '\'':
724 break;
725 case '+':
726 return rb_reg_last_defined(backref);
727 default:
728 rb_bug("unexpected back-ref");
729 }
730 }
731 else {
732 nth = (int)(type >> 1);
733 }
734 return rb_reg_nth_defined(nth, backref);
735}
736
737PUREFUNC(static rb_callable_method_entry_t *check_method_entry(VALUE obj, int can_be_svar));
738static rb_callable_method_entry_t *
739check_method_entry(VALUE obj, int can_be_svar)
740{
741 if (obj == Qfalse) return NULL;
742
743#if VM_CHECK_MODE > 0
744 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_method_entry: unknown type: %s", rb_obj_info(obj));
745#endif
746
747 switch (imemo_type(obj)) {
748 case imemo_ment:
749 return (rb_callable_method_entry_t *)obj;
750 case imemo_cref:
751 return NULL;
752 case imemo_svar:
753 if (can_be_svar) {
754 return check_method_entry(((struct vm_svar *)obj)->cref_or_me, FALSE);
755 }
756 default:
757#if VM_CHECK_MODE > 0
758 rb_bug("check_method_entry: svar should not be there:");
759#endif
760 return NULL;
761 }
762}
763
764static rb_callable_method_entry_t *
765env_method_entry_unchecked(VALUE obj, int can_be_svar)
766{
767 if (obj == Qfalse) return NULL;
768
769 switch (imemo_type(obj)) {
770 case imemo_ment:
771 return (rb_callable_method_entry_t *)obj;
772 case imemo_cref:
773 return NULL;
774 case imemo_svar:
775 if (can_be_svar) {
776 return env_method_entry_unchecked(((struct vm_svar *)obj)->cref_or_me, FALSE);
777 }
778 default:
779 return NULL;
780 }
781}
782
783const rb_callable_method_entry_t *
784rb_vm_frame_method_entry(const rb_control_frame_t *cfp)
785{
786 const VALUE *ep = cfp->ep;
787 rb_callable_method_entry_t *me;
788
789 while (!VM_ENV_LOCAL_P(ep)) {
790 if ((me = check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
791 ep = VM_ENV_PREV_EP(ep);
792 }
793
794 return check_method_entry(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
795}
796
797const rb_callable_method_entry_t *
798rb_vm_frame_method_entry_unchecked(const rb_control_frame_t *cfp)
799{
800 const VALUE *ep = cfp->ep;
801 rb_callable_method_entry_t *me;
802
803 while (!VM_ENV_LOCAL_P_UNCHECKED(ep)) {
804 if ((me = env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return me;
805 ep = VM_ENV_PREV_EP_UNCHECKED(ep);
806 }
807
808 return env_method_entry_unchecked(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
809}
810
811static const rb_iseq_t *
812method_entry_iseqptr(const rb_callable_method_entry_t *me)
813{
814 switch (me->def->type) {
815 case VM_METHOD_TYPE_ISEQ:
816 return me->def->body.iseq.iseqptr;
817 default:
818 return NULL;
819 }
820}
821
822static rb_cref_t *
823method_entry_cref(const rb_callable_method_entry_t *me)
824{
825 switch (me->def->type) {
826 case VM_METHOD_TYPE_ISEQ:
827 return me->def->body.iseq.cref;
828 default:
829 return NULL;
830 }
831}
832
833#if VM_CHECK_MODE == 0
834PUREFUNC(static rb_cref_t *check_cref(VALUE, int));
835#endif
836static rb_cref_t *
837check_cref(VALUE obj, int can_be_svar)
838{
839 if (obj == Qfalse) return NULL;
840
841#if VM_CHECK_MODE > 0
842 if (!RB_TYPE_P(obj, T_IMEMO)) rb_bug("check_cref: unknown type: %s", rb_obj_info(obj));
843#endif
844
845 switch (imemo_type(obj)) {
846 case imemo_ment:
847 return method_entry_cref((rb_callable_method_entry_t *)obj);
848 case imemo_cref:
849 return (rb_cref_t *)obj;
850 case imemo_svar:
851 if (can_be_svar) {
852 return check_cref(((struct vm_svar *)obj)->cref_or_me, FALSE);
853 }
854 default:
855#if VM_CHECK_MODE > 0
856 rb_bug("check_method_entry: svar should not be there:");
857#endif
858 return NULL;
859 }
860}
861
862static inline rb_cref_t *
863vm_env_cref(const VALUE *ep)
864{
865 rb_cref_t *cref;
866
867 while (!VM_ENV_LOCAL_P(ep)) {
868 if ((cref = check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) != NULL) return cref;
869 ep = VM_ENV_PREV_EP(ep);
870 }
871
872 return check_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
873}
874
875static int
876is_cref(const VALUE v, int can_be_svar)
877{
878 if (RB_TYPE_P(v, T_IMEMO)) {
879 switch (imemo_type(v)) {
880 case imemo_cref:
881 return TRUE;
882 case imemo_svar:
883 if (can_be_svar) return is_cref(((struct vm_svar *)v)->cref_or_me, FALSE);
884 default:
885 break;
886 }
887 }
888 return FALSE;
889}
890
891static int
892vm_env_cref_by_cref(const VALUE *ep)
893{
894 while (!VM_ENV_LOCAL_P(ep)) {
895 if (is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE)) return TRUE;
896 ep = VM_ENV_PREV_EP(ep);
897 }
898 return is_cref(ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE);
899}
900
901static rb_cref_t *
902cref_replace_with_duplicated_cref_each_frame(const VALUE *vptr, int can_be_svar, VALUE parent)
903{
904 const VALUE v = *vptr;
905 rb_cref_t *cref, *new_cref;
906
907 if (RB_TYPE_P(v, T_IMEMO)) {
908 switch (imemo_type(v)) {
909 case imemo_cref:
910 cref = (rb_cref_t *)v;
911 new_cref = vm_cref_dup(cref);
912 if (parent) {
913 RB_OBJ_WRITE(parent, vptr, new_cref);
914 }
915 else {
916 VM_FORCE_WRITE(vptr, (VALUE)new_cref);
917 }
918 return (rb_cref_t *)new_cref;
919 case imemo_svar:
920 if (can_be_svar) {
921 return cref_replace_with_duplicated_cref_each_frame(&((struct vm_svar *)v)->cref_or_me, FALSE, v);
922 }
923 /* fall through */
924 case imemo_ment:
925 rb_bug("cref_replace_with_duplicated_cref_each_frame: unreachable");
926 default:
927 break;
928 }
929 }
930 return NULL;
931}
932
933static rb_cref_t *
934vm_cref_replace_with_duplicated_cref(const VALUE *ep)
935{
936 if (vm_env_cref_by_cref(ep)) {
937 rb_cref_t *cref;
938 VALUE envval;
939
940 while (!VM_ENV_LOCAL_P(ep)) {
941 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
942 if ((cref = cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], FALSE, envval)) != NULL) {
943 return cref;
944 }
945 ep = VM_ENV_PREV_EP(ep);
946 }
947 envval = VM_ENV_ESCAPED_P(ep) ? VM_ENV_ENVVAL(ep) : Qfalse;
948 return cref_replace_with_duplicated_cref_each_frame(&ep[VM_ENV_DATA_INDEX_ME_CREF], TRUE, envval);
949 }
950 else {
951 rb_bug("vm_cref_dup: unreachable");
952 }
953}
954
955static rb_cref_t *
956vm_get_cref(const VALUE *ep)
957{
958 rb_cref_t *cref = vm_env_cref(ep);
959
960 if (cref != NULL) {
961 return cref;
962 }
963 else {
964 rb_bug("vm_get_cref: unreachable");
965 }
966}
967
968rb_cref_t *
969rb_vm_get_cref(const VALUE *ep)
970{
971 return vm_get_cref(ep);
972}
973
974static rb_cref_t *
975vm_ec_cref(const rb_execution_context_t *ec)
976{
977 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
978
979 if (cfp == NULL) {
980 return NULL;
981 }
982 return vm_get_cref(cfp->ep);
983}
984
985static const rb_cref_t *
986vm_get_const_key_cref(const VALUE *ep)
987{
988 const rb_cref_t *cref = vm_get_cref(ep);
989 const rb_cref_t *key_cref = cref;
990
991 while (cref) {
992 if (RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
993 RCLASS_CLONED_P(CREF_CLASS(cref)) ) {
994 return key_cref;
995 }
996 cref = CREF_NEXT(cref);
997 }
998
999 /* does not include singleton class */
1000 return NULL;
1001}
1002
1003rb_cref_t *
1004rb_vm_rewrite_cref(rb_cref_t *cref, VALUE old_klass, VALUE new_klass)
1005{
1006 rb_cref_t *new_cref_head = NULL;
1007 rb_cref_t *new_cref_tail = NULL;
1008
1009 #define ADD_NEW_CREF(new_cref) \
1010 if (new_cref_tail) { \
1011 RB_OBJ_WRITE(new_cref_tail, &new_cref_tail->next, new_cref); \
1012 } \
1013 else { \
1014 new_cref_head = new_cref; \
1015 } \
1016 new_cref_tail = new_cref;
1017
1018 while (cref) {
1019 rb_cref_t *new_cref;
1020 if (CREF_CLASS(cref) == old_klass) {
1021 new_cref = vm_cref_new_use_prev(new_klass, METHOD_VISI_UNDEF, FALSE, cref, FALSE);
1022 ADD_NEW_CREF(new_cref);
1023 return new_cref_head;
1024 }
1025 new_cref = vm_cref_new_use_prev(CREF_CLASS(cref), METHOD_VISI_UNDEF, FALSE, cref, FALSE);
1026 cref = CREF_NEXT(cref);
1027 ADD_NEW_CREF(new_cref);
1028 }
1029
1030 #undef ADD_NEW_CREF
1031
1032 // Could we just reuse the original cref?
1033 return new_cref_head;
1034}
1035
1036static rb_cref_t *
1037vm_cref_push(const rb_execution_context_t *ec, VALUE klass, const VALUE *ep, int pushed_by_eval, int singleton)
1038{
1039 rb_cref_t *prev_cref = NULL;
1040
1041 if (ep) {
1042 prev_cref = vm_env_cref(ep);
1043 }
1044 else {
1045 rb_control_frame_t *cfp = vm_get_ruby_level_caller_cfp(ec, ec->cfp);
1046
1047 if (cfp) {
1048 prev_cref = vm_env_cref(cfp->ep);
1049 }
1050 }
1051
1052 return vm_cref_new(klass, METHOD_VISI_PUBLIC, FALSE, prev_cref, pushed_by_eval, singleton);
1053}
1054
1055static inline VALUE
1056vm_get_cbase(const VALUE *ep)
1057{
1058 const rb_cref_t *cref = vm_get_cref(ep);
1059
1060 return CREF_CLASS_FOR_DEFINITION(cref);
1061}
1062
1063static inline VALUE
1064vm_get_const_base(const VALUE *ep)
1065{
1066 const rb_cref_t *cref = vm_get_cref(ep);
1067
1068 while (cref) {
1069 if (!CREF_PUSHED_BY_EVAL(cref)) {
1070 return CREF_CLASS_FOR_DEFINITION(cref);
1071 }
1072 cref = CREF_NEXT(cref);
1073 }
1074
1075 return Qundef;
1076}
1077
1078static inline void
1079vm_check_if_namespace(VALUE klass)
1080{
1081 if (!RB_TYPE_P(klass, T_CLASS) && !RB_TYPE_P(klass, T_MODULE)) {
1082 rb_raise(rb_eTypeError, "%+"PRIsVALUE" is not a class/module", klass);
1083 }
1084}
1085
1086static inline void
1087vm_ensure_not_refinement_module(VALUE self)
1088{
1089 if (RB_TYPE_P(self, T_MODULE) && FL_TEST(self, RMODULE_IS_REFINEMENT)) {
1090 rb_warn("not defined at the refinement, but at the outer class/module");
1091 }
1092}
1093
1094static inline VALUE
1095vm_get_iclass(const rb_control_frame_t *cfp, VALUE klass)
1096{
1097 return klass;
1098}
1099
1100static inline VALUE
1101vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, bool allow_nil, int is_defined)
1102{
1103 void rb_const_warn_if_deprecated(const rb_const_entry_t *ce, VALUE klass, ID id);
1104 VALUE val;
1105
1106 if (NIL_P(orig_klass) && allow_nil) {
1107 /* in current lexical scope */
1108 const rb_cref_t *root_cref = vm_get_cref(ec->cfp->ep);
1109 const rb_cref_t *cref;
1110 VALUE klass = Qnil;
1111
1112 while (root_cref && CREF_PUSHED_BY_EVAL(root_cref)) {
1113 root_cref = CREF_NEXT(root_cref);
1114 }
1115 cref = root_cref;
1116 while (cref && CREF_NEXT(cref)) {
1117 if (CREF_PUSHED_BY_EVAL(cref)) {
1118 klass = Qnil;
1119 }
1120 else {
1121 klass = CREF_CLASS(cref);
1122 }
1123 cref = CREF_NEXT(cref);
1124
1125 if (!NIL_P(klass)) {
1126 VALUE av, am = 0;
1127 rb_const_entry_t *ce;
1128 search_continue:
1129 if ((ce = rb_const_lookup(klass, id))) {
1130 rb_const_warn_if_deprecated(ce, klass, id);
1131 val = ce->value;
1132 if (UNDEF_P(val)) {
1133 if (am == klass) break;
1134 am = klass;
1135 if (is_defined) return 1;
1136 if (rb_autoloading_value(klass, id, &av, NULL)) return av;
1137 rb_autoload_load(klass, id);
1138 goto search_continue;
1139 }
1140 else {
1141 if (is_defined) {
1142 return 1;
1143 }
1144 else {
1145 if (UNLIKELY(!rb_ractor_main_p())) {
1146 if (!rb_ractor_shareable_p(val)) {
1147 rb_raise(rb_eRactorIsolationError,
1148 "can not access non-shareable objects in constant %"PRIsVALUE"::%"PRIsVALUE" by non-main ractor.", rb_class_path(klass), rb_id2str(id));
1149 }
1150 }
1151 return val;
1152 }
1153 }
1154 }
1155 }
1156 }
1157
1158 /* search self */
1159 if (root_cref && !NIL_P(CREF_CLASS(root_cref))) {
1160 klass = vm_get_iclass(ec->cfp, CREF_CLASS(root_cref));
1161 }
1162 else {
1163 klass = CLASS_OF(ec->cfp->self);
1164 }
1165
1166 if (is_defined) {
1167 return rb_const_defined(klass, id);
1168 }
1169 else {
1170 return rb_const_get(klass, id);
1171 }
1172 }
1173 else {
1174 vm_check_if_namespace(orig_klass);
1175 if (is_defined) {
1176 return rb_public_const_defined_from(orig_klass, id);
1177 }
1178 else {
1179 return rb_public_const_get_from(orig_klass, id);
1180 }
1181 }
1182}
1183
1184 VALUE
1185rb_vm_get_ev_const(rb_execution_context_t *ec, VALUE orig_klass, ID id, VALUE allow_nil)
1186{
1187 return vm_get_ev_const(ec, orig_klass, id, allow_nil == Qtrue, 0);
1188}
1189
1190static inline VALUE
1191vm_get_ev_const_chain(rb_execution_context_t *ec, const ID *segments)
1192{
1193 VALUE val = Qnil;
1194 int idx = 0;
1195 int allow_nil = TRUE;
1196 if (segments[0] == idNULL) {
1197 val = rb_cObject;
1198 idx++;
1199 allow_nil = FALSE;
1200 }
1201 while (segments[idx]) {
1202 ID id = segments[idx++];
1203 val = vm_get_ev_const(ec, val, id, allow_nil, 0);
1204 allow_nil = FALSE;
1205 }
1206 return val;
1207}
1208
1209
1210static inline VALUE
1211vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_level_raise)
1212{
1213 VALUE klass;
1214
1215 if (!cref) {
1216 rb_bug("vm_get_cvar_base: no cref");
1217 }
1218
1219 while (CREF_NEXT(cref) &&
1220 (NIL_P(CREF_CLASS(cref)) || RCLASS_SINGLETON_P(CREF_CLASS(cref)) ||
1221 CREF_PUSHED_BY_EVAL(cref) || CREF_SINGLETON(cref))) {
1222 cref = CREF_NEXT(cref);
1223 }
1224 if (top_level_raise && !CREF_NEXT(cref)) {
1225 rb_raise(rb_eRuntimeError, "class variable access from toplevel");
1226 }
1227
1228 klass = vm_get_iclass(cfp, CREF_CLASS(cref));
1229
1230 if (NIL_P(klass)) {
1231 rb_raise(rb_eTypeError, "no class variables available");
1232 }
1233 return klass;
1234}
1235
1236ALWAYS_INLINE(static void fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, attr_index_t index, shape_id_t shape_id));
1237static inline void
1238fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, attr_index_t index, shape_id_t shape_id)
1239{
1240 if (is_attr) {
1241 vm_cc_attr_index_set(cc, index, shape_id);
1242 }
1243 else {
1244 vm_ic_attr_index_set(iseq, ic, index, shape_id);
1245 }
1246}
1247
1248#define ractor_incidental_shareable_p(cond, val) \
1249 (!(cond) || rb_ractor_shareable_p(val))
1250#define ractor_object_incidental_shareable_p(obj, val) \
1251 ractor_incidental_shareable_p(rb_ractor_shareable_p(obj), val)
1252
1253ALWAYS_INLINE(static VALUE vm_getivar(VALUE, ID, const rb_iseq_t *, IVC, const struct rb_callcache *, int, VALUE));
1254static inline VALUE
1255vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, VALUE default_value)
1256{
1257 VALUE fields_obj;
1258#if OPT_IC_FOR_IVAR
1259 if (SPECIAL_CONST_P(obj)) {
1260 return default_value;
1261 }
1262
1263 switch (BUILTIN_TYPE(obj)) {
1264 case T_OBJECT:
1265 fields_obj = obj;
1266 break;
1267 case T_CLASS:
1268 case T_MODULE:
1269 {
1270 if (UNLIKELY(!rb_ractor_main_p())) {
1271 // For two reasons we can only use the fast path on the main
1272 // ractor.
1273 // First, only the main ractor is allowed to set ivars on classes
1274 // and modules. So we can skip locking.
1275 // Second, other ractors need to check the shareability of the
1276 // values returned from the class ivars.
1277
1278 if (default_value == Qundef) { // defined?
1279 return rb_ivar_defined(obj, id) ? Qtrue : Qundef;
1280 }
1281 else {
1282 goto general_path;
1283 }
1284 }
1285
1286 fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1287 break;
1288 }
1289 default:
1290 fields_obj = rb_obj_fields(obj, id);
1291 }
1292
1293 if (!fields_obj) {
1294 return default_value;
1295 }
1296
1297 VALUE val = Qundef;
1298
1299 shape_id_t shape_id = RBASIC_SHAPE_ID_FOR_READ(fields_obj);
1300 VALUE *ivar_list = rb_imemo_fields_ptr(fields_obj);
1301
1302 shape_id_t cached_id;
1303 attr_index_t index;
1304
1305 if (is_attr) {
1306 vm_cc_atomic_shape_and_index(cc, &cached_id, &index);
1307 }
1308 else {
1309 vm_ic_atomic_shape_and_index(ic, &cached_id, &index);
1310 }
1311
1312 if (LIKELY(cached_id == shape_id)) {
1313 RUBY_ASSERT(!rb_shape_too_complex_p(cached_id));
1314
1315 if (index == ATTR_INDEX_NOT_SET) {
1316 return default_value;
1317 }
1318
1319 val = ivar_list[index];
1320#if USE_DEBUG_COUNTER
1321 RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
1322
1323 if (RB_TYPE_P(obj, T_OBJECT)) {
1324 RB_DEBUG_COUNTER_INC(ivar_get_obj_hit);
1325 }
1326#endif
1327 RUBY_ASSERT(!UNDEF_P(val));
1328 }
1329 else { // cache miss case
1330#if USE_DEBUG_COUNTER
1331 if (is_attr) {
1332 if (cached_id != INVALID_SHAPE_ID) {
1333 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_set);
1334 }
1335 else {
1336 RB_DEBUG_COUNTER_INC(ivar_get_cc_miss_unset);
1337 }
1338 }
1339 else {
1340 if (cached_id != INVALID_SHAPE_ID) {
1341 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_set);
1342 }
1343 else {
1344 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss_unset);
1345 }
1346 }
1347 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1348
1349 if (RB_TYPE_P(obj, T_OBJECT)) {
1350 RB_DEBUG_COUNTER_INC(ivar_get_obj_miss);
1351 }
1352#endif
1353
1354 if (UNLIKELY(rb_shape_too_complex_p(shape_id))) {
1355 st_table *table = (st_table *)ivar_list;
1356
1357 RUBY_ASSERT(table);
1358 RUBY_ASSERT(table == rb_imemo_fields_complex_tbl(fields_obj));
1359
1360 if (!st_lookup(table, id, &val)) {
1361 val = default_value;
1362 }
1363 }
1364 else {
1365 shape_id_t previous_cached_id = cached_id;
1366 if (rb_shape_get_iv_index_with_hint(shape_id, id, &index, &cached_id)) {
1367 // This fills in the cache with the shared cache object.
1368 // "ent" is the shared cache object
1369 if (cached_id != previous_cached_id) {
1370 fill_ivar_cache(iseq, ic, cc, is_attr, index, cached_id);
1371 }
1372
1373 if (index == ATTR_INDEX_NOT_SET) {
1374 val = default_value;
1375 }
1376 else {
1377 // We fetched the ivar list above
1378 val = ivar_list[index];
1379 RUBY_ASSERT(!UNDEF_P(val));
1380 }
1381 }
1382 else {
1383 if (is_attr) {
1384 vm_cc_attr_index_initialize(cc, shape_id);
1385 }
1386 else {
1387 vm_ic_attr_index_initialize(ic, shape_id);
1388 }
1389
1390 val = default_value;
1391 }
1392 }
1393 }
1394
1395 if (!UNDEF_P(default_value)) {
1396 RUBY_ASSERT(!UNDEF_P(val));
1397 }
1398
1399 return val;
1400
1401general_path:
1402#endif /* OPT_IC_FOR_IVAR */
1403 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
1404
1405 if (is_attr) {
1406 return rb_attr_get(obj, id);
1407 }
1408 else {
1409 return rb_ivar_get(obj, id);
1410 }
1411}
1412
1413static void
1414populate_cache(attr_index_t index, shape_id_t next_shape_id, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, bool is_attr)
1415{
1416 RUBY_ASSERT(!rb_shape_too_complex_p(next_shape_id));
1417
1418 // Cache population code
1419 if (is_attr) {
1420 vm_cc_attr_index_set(cc, index, next_shape_id);
1421 }
1422 else {
1423 vm_ic_attr_index_set(iseq, ic, index, next_shape_id);
1424 }
1425}
1426
1427ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
1428NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
1429NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
1430
1431static VALUE
1432vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
1433{
1434#if OPT_IC_FOR_IVAR
1435 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
1436
1437 rb_check_frozen(obj);
1438
1439 attr_index_t index = rb_ivar_set_index(obj, id, val);
1440 shape_id_t next_shape_id = RBASIC_SHAPE_ID(obj);
1441
1442 if (!rb_shape_too_complex_p(next_shape_id)) {
1443 populate_cache(index, next_shape_id, id, iseq, ic, cc, is_attr);
1444 }
1445
1446 RB_DEBUG_COUNTER_INC(ivar_set_obj_miss);
1447 return val;
1448#else
1449 return rb_ivar_set(obj, id, val);
1450#endif
1451}
1452
1453static VALUE
1454vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic)
1455{
1456 return vm_setivar_slowpath(obj, id, val, iseq, ic, NULL, false);
1457}
1458
1459static VALUE
1460vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc)
1461{
1462 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
1463}
1464
1465NOINLINE(static VALUE vm_setivar_class(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index));
1466static VALUE
1467vm_setivar_class(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1468{
1469 if (UNLIKELY(!rb_ractor_main_p())) {
1470 return Qundef;
1471 }
1472
1473 VALUE fields_obj = RCLASS_WRITABLE_FIELDS_OBJ(obj);
1474 if (UNLIKELY(!fields_obj)) {
1475 return Qundef;
1476 }
1477
1478 shape_id_t shape_id = RBASIC_SHAPE_ID(fields_obj);
1479
1480 // Cache hit case
1481 if (shape_id == dest_shape_id) {
1482 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1483 }
1484 else if (dest_shape_id != INVALID_SHAPE_ID) {
1485 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1486 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1487 }
1488 else {
1489 return Qundef;
1490 }
1491 }
1492 else {
1493 return Qundef;
1494 }
1495
1496 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[index], val);
1497
1498 if (shape_id != dest_shape_id) {
1499 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1500 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1501 }
1502
1503 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1504
1505 return val;
1506}
1507
1508NOINLINE(static VALUE vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index));
1509static VALUE
1510vm_setivar_default(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1511{
1512 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1513
1514 // Cache hit case
1515 if (shape_id == dest_shape_id) {
1516 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1517 }
1518 else if (dest_shape_id != INVALID_SHAPE_ID) {
1519 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1520 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1521 }
1522 else {
1523 return Qundef;
1524 }
1525 }
1526 else {
1527 return Qundef;
1528 }
1529
1530 VALUE fields_obj = rb_obj_fields(obj, id);
1531 RUBY_ASSERT(fields_obj);
1532 RB_OBJ_WRITE(fields_obj, &rb_imemo_fields_ptr(fields_obj)[index], val);
1533
1534 if (shape_id != dest_shape_id) {
1535 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1536 RBASIC_SET_SHAPE_ID(fields_obj, dest_shape_id);
1537 }
1538
1539 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1540
1541 return val;
1542}
1543
1544static inline VALUE
1545vm_setivar(VALUE obj, ID id, VALUE val, shape_id_t dest_shape_id, attr_index_t index)
1546{
1547#if OPT_IC_FOR_IVAR
1548 switch (BUILTIN_TYPE(obj)) {
1549 case T_OBJECT:
1550 {
1551 VM_ASSERT(!rb_ractor_shareable_p(obj) || rb_obj_frozen_p(obj));
1552
1553 shape_id_t shape_id = RBASIC_SHAPE_ID(obj);
1554 RUBY_ASSERT(dest_shape_id == INVALID_SHAPE_ID || !rb_shape_too_complex_p(dest_shape_id));
1555
1556 if (LIKELY(shape_id == dest_shape_id)) {
1557 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1558 VM_ASSERT(!rb_ractor_shareable_p(obj));
1559 }
1560 else if (dest_shape_id != INVALID_SHAPE_ID) {
1561 if (RSHAPE_DIRECT_CHILD_P(shape_id, dest_shape_id) && RSHAPE_EDGE_NAME(dest_shape_id) == id && RSHAPE_CAPACITY(shape_id) == RSHAPE_CAPACITY(dest_shape_id)) {
1562 RUBY_ASSERT(dest_shape_id != INVALID_SHAPE_ID && shape_id != INVALID_SHAPE_ID);
1563
1564 RBASIC_SET_SHAPE_ID(obj, dest_shape_id);
1565
1566 RUBY_ASSERT(index < RSHAPE_CAPACITY(dest_shape_id));
1567 }
1568 else {
1569 break;
1570 }
1571 }
1572 else {
1573 break;
1574 }
1575
1576 VALUE *ptr = ROBJECT_FIELDS(obj);
1577
1578 RUBY_ASSERT(!rb_shape_obj_too_complex_p(obj));
1579 RB_OBJ_WRITE(obj, &ptr[index], val);
1580
1581 RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
1582 RB_DEBUG_COUNTER_INC(ivar_set_obj_hit);
1583 return val;
1584 }
1585 break;
1586 case T_CLASS:
1587 case T_MODULE:
1588 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
1589 default:
1590 break;
1591 }
1592
1593 return Qundef;
1594#endif /* OPT_IC_FOR_IVAR */
1595}
1596
1597static VALUE
1598update_classvariable_cache(const rb_iseq_t *iseq, VALUE klass, ID id, const rb_cref_t * cref, ICVARC ic)
1599{
1600 VALUE defined_class = 0;
1601 VALUE cvar_value = rb_cvar_find(klass, id, &defined_class);
1602
1603 if (RB_TYPE_P(defined_class, T_ICLASS)) {
1604 defined_class = RBASIC(defined_class)->klass;
1605 }
1606
1607 VALUE rb_cvc_tbl = RCLASS_CVC_TBL(defined_class);
1608 if (!rb_cvc_tbl) {
1609 rb_bug("the cvc table should be set");
1610 }
1611
1612 VALUE ent_data;
1613 if (!rb_marked_id_table_lookup(rb_cvc_tbl, id, &ent_data)) {
1614 rb_bug("should have cvar cache entry");
1615 }
1616
1617 struct rb_cvar_class_tbl_entry *ent = (void *)ent_data;
1618
1619 ent->global_cvar_state = GET_GLOBAL_CVAR_STATE();
1620 RB_OBJ_WRITE((VALUE)ent, &ent->cref, cref);
1621 ic->entry = ent;
1622
1623 RUBY_ASSERT(BUILTIN_TYPE((VALUE)cref) == T_IMEMO && IMEMO_TYPE_P(cref, imemo_cref));
1624 RB_OBJ_WRITTEN(iseq, Qundef, ent->class_value);
1625 RB_OBJ_WRITTEN(ent->class_value, Qundef, ent->cref);
1626
1627 return cvar_value;
1628}
1629
1630static inline VALUE
1631vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, ICVARC ic)
1632{
1633 const rb_cref_t *cref;
1634 cref = vm_get_cref(GET_EP());
1635
1636 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1637 RB_DEBUG_COUNTER_INC(cvar_read_inline_hit);
1638
1639 VALUE v = rb_ivar_lookup(ic->entry->class_value, id, Qundef);
1640 RUBY_ASSERT(!UNDEF_P(v));
1641
1642 return v;
1643 }
1644
1645 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1646
1647 return update_classvariable_cache(iseq, klass, id, cref, ic);
1648}
1649
1650 VALUE
1651rb_vm_getclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, ICVARC ic)
1652{
1653 return vm_getclassvariable(iseq, cfp, id, ic);
1654}
1655
1656static inline void
1657vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *reg_cfp, ID id, VALUE val, ICVARC ic)
1658{
1659 const rb_cref_t *cref;
1660 cref = vm_get_cref(GET_EP());
1661
1662 if (ic->entry && ic->entry->global_cvar_state == GET_GLOBAL_CVAR_STATE() && ic->entry->cref == cref && LIKELY(rb_ractor_main_p())) {
1663 RB_DEBUG_COUNTER_INC(cvar_write_inline_hit);
1664
1665 rb_class_ivar_set(ic->entry->class_value, id, val);
1666 return;
1667 }
1668
1669 VALUE klass = vm_get_cvar_base(cref, reg_cfp, 1);
1670
1671 rb_cvar_set(klass, id, val);
1672
1673 update_classvariable_cache(iseq, klass, id, cref, ic);
1674}
1675
1676void
1677rb_vm_setclassvariable(const rb_iseq_t *iseq, const rb_control_frame_t *cfp, ID id, VALUE val, ICVARC ic)
1678{
1679 vm_setclassvariable(iseq, cfp, id, val, ic);
1680}
1681
1682ALWAYS_INLINE(static VALUE vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic));
1683static inline VALUE
1684vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1685{
1686 return vm_getivar(obj, id, iseq, ic, NULL, FALSE, Qnil);
1687}
1688
1689static inline void
1690vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1691{
1692 if (RB_SPECIAL_CONST_P(obj)) {
1693 rb_error_frozen_object(obj);
1694 return;
1695 }
1696
1697 shape_id_t dest_shape_id;
1698 attr_index_t index;
1699 vm_ic_atomic_shape_and_index(ic, &dest_shape_id, &index);
1700
1701 if (UNLIKELY(UNDEF_P(vm_setivar(obj, id, val, dest_shape_id, index)))) {
1702 switch (BUILTIN_TYPE(obj)) {
1703 case T_OBJECT:
1704 break;
1705 case T_CLASS:
1706 case T_MODULE:
1707 if (!UNDEF_P(vm_setivar_class(obj, id, val, dest_shape_id, index))) {
1708 return;
1709 }
1710 break;
1711 default:
1712 if (!UNDEF_P(vm_setivar_default(obj, id, val, dest_shape_id, index))) {
1713 return;
1714 }
1715 }
1716 vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
1717 }
1718}
1719
1720void
1721rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
1722{
1723 vm_setinstancevariable(iseq, obj, id, val, ic);
1724}
1725
1726 VALUE
1727rb_vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
1728{
1729 return vm_getinstancevariable(iseq, obj, id, ic);
1730}
1731
1732static VALUE
1733vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
1734{
1735 /* continue throw */
1736
1737 if (FIXNUM_P(err)) {
1738 ec->tag->state = RUBY_TAG_FATAL;
1739 }
1740 else if (SYMBOL_P(err)) {
1741 ec->tag->state = TAG_THROW;
1742 }
1743 else if (THROW_DATA_P(err)) {
1744 ec->tag->state = THROW_DATA_STATE((struct vm_throw_data *)err);
1745 }
1746 else {
1747 ec->tag->state = TAG_RAISE;
1748 }
1749 return err;
1750}
1751
1752static VALUE
1753vm_throw_start(const rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, enum ruby_tag_type state,
1754 const int flag, const VALUE throwobj)
1755{
1756 const rb_control_frame_t *escape_cfp = NULL;
1757 const rb_control_frame_t * const eocfp = RUBY_VM_END_CONTROL_FRAME(ec); /* end of control frame pointer */
1758
1759 if (flag != 0) {
1760 /* do nothing */
1761 }
1762 else if (state == TAG_BREAK) {
1763 int is_orphan = 1;
1764 const VALUE *ep = GET_EP();
1765 const rb_iseq_t *base_iseq = GET_ISEQ();
1766 escape_cfp = reg_cfp;
1767
1768 while (ISEQ_BODY(base_iseq)->type != ISEQ_TYPE_BLOCK) {
1769 if (ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) {
1770 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1771 ep = escape_cfp->ep;
1772 base_iseq = escape_cfp->iseq;
1773 }
1774 else {
1775 ep = VM_ENV_PREV_EP(ep);
1776 base_iseq = ISEQ_BODY(base_iseq)->parent_iseq;
1777 escape_cfp = rb_vm_search_cf_from_ep(ec, escape_cfp, ep);
1778 VM_ASSERT(escape_cfp->iseq == base_iseq);
1779 }
1780 }
1781
1782 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1783 /* lambda{... break ...} */
1784 is_orphan = 0;
1785 state = TAG_RETURN;
1786 }
1787 else {
1788 ep = VM_ENV_PREV_EP(ep);
1789
1790 while (escape_cfp < eocfp) {
1791 if (escape_cfp->ep == ep) {
1792 const rb_iseq_t *const iseq = escape_cfp->iseq;
1793 const VALUE epc = escape_cfp->pc - ISEQ_BODY(iseq)->iseq_encoded;
1794 const struct iseq_catch_table *const ct = ISEQ_BODY(iseq)->catch_table;
1795 unsigned int i;
1796
1797 if (!ct) break;
1798 for (i=0; i < ct->size; i++) {
1799 const struct iseq_catch_table_entry *const entry =
1800 UNALIGNED_MEMBER_PTR(ct, entries[i]);
1801
1802 if (entry->type == CATCH_TYPE_BREAK &&
1803 entry->iseq == base_iseq &&
1804 entry->start < epc && entry->end >= epc) {
1805 if (entry->cont == epc) { /* found! */
1806 is_orphan = 0;
1807 }
1808 break;
1809 }
1810 }
1811 break;
1812 }
1813
1814 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1815 }
1816 }
1817
1818 if (is_orphan) {
1819 rb_vm_localjump_error("break from proc-closure", throwobj, TAG_BREAK);
1820 }
1821 }
1822 else if (state == TAG_RETRY) {
1823 const VALUE *ep = VM_ENV_PREV_EP(GET_EP());
1824
1825 escape_cfp = rb_vm_search_cf_from_ep(ec, reg_cfp, ep);
1826 }
1827 else if (state == TAG_RETURN) {
1828 const VALUE *current_ep = GET_EP();
1829 const VALUE *target_ep = NULL, *target_lep, *ep = current_ep;
1830 int in_class_frame = 0;
1831 int toplevel = 1;
1832 escape_cfp = reg_cfp;
1833
1834 // find target_lep, target_ep
1835 while (!VM_ENV_LOCAL_P(ep)) {
1836 if (VM_ENV_FLAGS(ep, VM_FRAME_FLAG_LAMBDA) && target_ep == NULL) {
1837 target_ep = ep;
1838 }
1839 ep = VM_ENV_PREV_EP(ep);
1840 }
1841 target_lep = ep;
1842
1843 while (escape_cfp < eocfp) {
1844 const VALUE *lep = VM_CF_LEP(escape_cfp);
1845
1846 if (!target_lep) {
1847 target_lep = lep;
1848 }
1849
1850 if (lep == target_lep &&
1851 VM_FRAME_RUBYFRAME_P(escape_cfp) &&
1852 ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_CLASS) {
1853 in_class_frame = 1;
1854 target_lep = 0;
1855 }
1856
1857 if (lep == target_lep) {
1858 if (VM_FRAME_LAMBDA_P(escape_cfp)) {
1859 toplevel = 0;
1860 if (in_class_frame) {
1861 /* lambda {class A; ... return ...; end} */
1862 goto valid_return;
1863 }
1864 else {
1865 const VALUE *tep = current_ep;
1866
1867 while (target_lep != tep) {
1868 if (escape_cfp->ep == tep) {
1869 /* in lambda */
1870 if (tep == target_ep) {
1871 goto valid_return;
1872 }
1873 else {
1874 goto unexpected_return;
1875 }
1876 }
1877 tep = VM_ENV_PREV_EP(tep);
1878 }
1879 }
1880 }
1881 else if (VM_FRAME_RUBYFRAME_P(escape_cfp)) {
1882 switch (ISEQ_BODY(escape_cfp->iseq)->type) {
1883 case ISEQ_TYPE_TOP:
1884 case ISEQ_TYPE_MAIN:
1885 if (toplevel) {
1886 if (in_class_frame) goto unexpected_return;
1887 if (target_ep == NULL) {
1888 goto valid_return;
1889 }
1890 else {
1891 goto unexpected_return;
1892 }
1893 }
1894 break;
1895 case ISEQ_TYPE_EVAL: {
1896 const rb_iseq_t *is = escape_cfp->iseq;
1897 enum rb_iseq_type t = ISEQ_BODY(is)->type;
1898 while (t == ISEQ_TYPE_RESCUE || t == ISEQ_TYPE_ENSURE || t == ISEQ_TYPE_EVAL) {
1899 if (!(is = ISEQ_BODY(is)->parent_iseq)) break;
1900 t = ISEQ_BODY(is)->type;
1901 }
1902 toplevel = t == ISEQ_TYPE_TOP || t == ISEQ_TYPE_MAIN;
1903 break;
1904 }
1905 case ISEQ_TYPE_CLASS:
1906 toplevel = 0;
1907 break;
1908 default:
1909 break;
1910 }
1911 }
1912 }
1913
1914 if (escape_cfp->ep == target_lep && ISEQ_BODY(escape_cfp->iseq)->type == ISEQ_TYPE_METHOD) {
1915 if (target_ep == NULL) {
1916 goto valid_return;
1917 }
1918 else {
1919 goto unexpected_return;
1920 }
1921 }
1922
1923 escape_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(escape_cfp);
1924 }
1925 unexpected_return:;
1926 rb_vm_localjump_error("unexpected return", throwobj, TAG_RETURN);
1927
1928 valid_return:;
1929 /* do nothing */
1930 }
1931 else {
1932 rb_bug("isns(throw): unsupported throw type");
1933 }
1934
1935 ec->tag->state = state;
1936 return (VALUE)THROW_DATA_NEW(throwobj, escape_cfp, state);
1937}
1938
1939static VALUE
1940vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
1941 rb_num_t throw_state, VALUE throwobj)
1942{
1943 const int state = (int)(throw_state & VM_THROW_STATE_MASK);
1944 const int flag = (int)(throw_state & VM_THROW_NO_ESCAPE_FLAG);
1945
1946 if (state != 0) {
1947 return vm_throw_start(ec, reg_cfp, state, flag, throwobj);
1948 }
1949 else {
1950 return vm_throw_continue(ec, throwobj);
1951 }
1952}
1953
1954 VALUE
1955rb_vm_throw(const rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t throw_state, VALUE throwobj)
1956{
1957 return vm_throw(ec, reg_cfp, throw_state, throwobj);
1958}
1959
1960static inline void
1961vm_expandarray(struct rb_control_frame_struct *cfp, VALUE ary, rb_num_t num, int flag)
1962{
1963 int is_splat = flag & 0x01;
1964 const VALUE *ptr;
1965 rb_num_t len;
1966 const VALUE obj = ary;
1967
1968 if (!RB_TYPE_P(ary, T_ARRAY) && NIL_P(ary = rb_check_array_type(ary))) {
1969 ary = obj;
1970 ptr = &ary;
1971 len = 1;
1972 }
1973 else {
1974 ptr = RARRAY_CONST_PTR(ary);
1975 len = (rb_num_t)RARRAY_LEN(ary);
1976 }
1977
1978 if (num + is_splat == 0) {
1979 /* no space left on stack */
1980 }
1981 else if (flag & 0x02) {
1982 /* post: ..., nil ,ary[-1], ..., ary[0..-num] # top */
1983 rb_num_t i = 0, j;
1984
1985 if (len < num) {
1986 for (i = 0; i < num - len; i++) {
1987 *cfp->sp++ = Qnil;
1988 }
1989 }
1990
1991 for (j = 0; i < num; i++, j++) {
1992 VALUE v = ptr[len - j - 1];
1993 *cfp->sp++ = v;
1994 }
1995
1996 if (is_splat) {
1997 *cfp->sp++ = rb_ary_new4(len - j, ptr);
1998 }
1999 }
2000 else {
2001 /* normal: ary[num..-1], ary[num-2], ary[num-3], ..., ary[0] # top */
2002 if (is_splat) {
2003 if (num > len) {
2004 *cfp->sp++ = rb_ary_new();
2005 }
2006 else {
2007 *cfp->sp++ = rb_ary_new4(len - num, ptr + num);
2008 }
2009 }
2010
2011 if (num > len) {
2012 rb_num_t i = 0;
2013 for (; i < num - len; i++) {
2014 *cfp->sp++ = Qnil;
2015 }
2016
2017 for (rb_num_t j = 0; i < num; i++, j++) {
2018 *cfp->sp++ = ptr[len - j - 1];
2019 }
2020 }
2021 else {
2022 for (rb_num_t j = 0; j < num; j++) {
2023 *cfp->sp++ = ptr[num - j - 1];
2024 }
2025 }
2026 }
2027
2028 RB_GC_GUARD(ary);
2029}
2030
2031static VALUE vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2032
2033static VALUE vm_mtbl_dump(VALUE klass, ID target_mid);
2034
2035static struct rb_class_cc_entries *
2036vm_ccs_create(VALUE klass, VALUE cc_tbl, ID mid, const rb_callable_method_entry_t *cme)
2037{
2038 int initial_capa = 2;
2039 struct rb_class_cc_entries *ccs = ruby_xmalloc(vm_ccs_alloc_size(initial_capa));
2040#if VM_CHECK_MODE > 0
2041 ccs->debug_sig = ~(VALUE)ccs;
2042#endif
2043 ccs->capa = initial_capa;
2044 ccs->len = 0;
2045 ccs->cme = cme;
2046 METHOD_ENTRY_CACHED_SET((rb_callable_method_entry_t *)cme);
2047
2048 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2049 RB_OBJ_WRITTEN(cc_tbl, Qundef, cme);
2050 return ccs;
2051}
2052
2053static void
2054vm_ccs_push(VALUE cc_tbl, ID mid, struct rb_class_cc_entries *ccs, const struct rb_callinfo *ci, const struct rb_callcache *cc)
2055{
2056 if (! vm_cc_markable(cc)) {
2057 return;
2058 }
2059
2060 if (UNLIKELY(ccs->len == ccs->capa)) {
2061 RUBY_ASSERT(ccs->capa > 0);
2062 ccs->capa *= 2;
2063 ccs = ruby_xrealloc(ccs, vm_ccs_alloc_size(ccs->capa));
2064#if VM_CHECK_MODE > 0
2065 ccs->debug_sig = ~(VALUE)ccs;
2066#endif
2067 // GC?
2068 rb_managed_id_table_insert(cc_tbl, mid, (VALUE)ccs);
2069 }
2070 VM_ASSERT(ccs->len < ccs->capa);
2071
2072 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2073 const int pos = ccs->len++;
2074 ccs->entries[pos].argc = vm_ci_argc(ci);
2075 ccs->entries[pos].flag = (unsigned short)vm_ci_flag(ci);
2076 ccs->entries[pos].kw_len = (unsigned short)(kwarg ? kwarg->keyword_len : 0);
2077 RB_OBJ_WRITE(cc_tbl, &ccs->entries[pos].cc, cc);
2078
2079 if (RB_DEBUG_COUNTER_SETMAX(ccs_maxlen, ccs->len)) {
2080 // for tuning
2081 // vm_mtbl_dump(klass, 0);
2082 }
2083}
2084
2085#if VM_CHECK_MODE > 0
2086void
2087rb_vm_ccs_dump(struct rb_class_cc_entries *ccs)
2088{
2089 ruby_debug_printf("ccs:%p (%d,%d)\n", (void *)ccs, ccs->len, ccs->capa);
2090 for (int i=0; i<ccs->len; i++) {
2091 ruby_debug_printf("CCS CI ID:flag:%x argc:%u kw_len:%u\n",
2092 ccs->entries[i].flag,
2093 ccs->entries[i].argc,
2094 ccs->entries[i].kw_len);
2095 rp(ccs->entries[i].cc);
2096 }
2097}
2098
2099static int
2100vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass)
2101{
2102 VM_ASSERT(vm_ccs_p(ccs));
2103 VM_ASSERT(ccs->len <= ccs->capa);
2104
2105 for (int i=0; i<ccs->len; i++) {
2106 const struct rb_callcache *cc = ccs->entries[i].cc;
2107
2108 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2109 VM_ASSERT(vm_cc_class_check(cc, klass));
2110 VM_ASSERT(vm_cc_check_cme(cc, ccs->cme));
2111 VM_ASSERT(!vm_cc_super_p(cc));
2112 VM_ASSERT(!vm_cc_refinement_p(cc));
2113 }
2114 return TRUE;
2115}
2116#endif
2117
2118const rb_callable_method_entry_t *rb_check_overloaded_cme(const rb_callable_method_entry_t *cme, const struct rb_callinfo * const ci);
2119
2120static void
2121vm_evict_cc(VALUE klass, VALUE cc_tbl, ID mid)
2122{
2123 ASSERT_vm_locking();
2124
2125 if (rb_multi_ractor_p()) {
2126 if (RCLASS_WRITABLE_CC_TBL(klass) != cc_tbl) {
2127 // Another ractor updated the CC table while we were waiting on the VM lock.
2128 // We have to retry.
2129 return;
2130 }
2131
2132 VALUE ccs_obj = 0;
2133 rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj);
2134 struct rb_class_cc_entries *ccs = (struct rb_class_cc_entries *)ccs_obj;
2135
2136 if (!ccs || !METHOD_ENTRY_INVALIDATED(ccs->cme)) {
2137 // Another ractor replaced that entry while we were waiting on the VM lock.
2138 return;
2139 }
2140
2141 VALUE new_table = rb_vm_cc_table_dup(cc_tbl);
2142 rb_vm_cc_table_delete(new_table, mid);
2143 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), new_table);
2144 }
2145 else {
2146 rb_vm_cc_table_delete(cc_tbl, mid);
2147 }
2148}
2149
2150static const struct rb_callcache *
2151vm_populate_cc(VALUE klass, const struct rb_callinfo * const ci, ID mid)
2152{
2153 ASSERT_vm_locking();
2154
2155 RB_DEBUG_COUNTER_INC(cc_not_found_in_ccs);
2156
2157 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
2158
2159 VM_ASSERT(cme == NULL || IMEMO_TYPE_P(cme, imemo_ment));
2160
2161 if (cme == NULL) {
2162 // undef or not found: can't cache the information
2163 VM_ASSERT(vm_cc_cme(&vm_empty_cc) == NULL);
2164 return &vm_empty_cc;
2165 }
2166
2167 VALUE cc_tbl = RCLASS_WRITABLE_CC_TBL(klass);
2168 const VALUE original_cc_table = cc_tbl;
2169 if (!cc_tbl) {
2170 // Is this possible after rb_callable_method_entry ?
2171 cc_tbl = rb_vm_cc_table_create(1);
2172 }
2173 else if (rb_multi_ractor_p()) {
2174 cc_tbl = rb_vm_cc_table_dup(cc_tbl);
2175 }
2176
2177 VM_ASSERT(cme == rb_callable_method_entry(klass, mid));
2178
2179 METHOD_ENTRY_CACHED_SET((struct rb_callable_method_entry_struct *)cme);
2180
2181 VM_ASSERT(cc_tbl);
2182
2183 struct rb_class_cc_entries *ccs = NULL;
2184 {
2185 VALUE ccs_obj;
2186 if (UNLIKELY(rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj))) {
2187 ccs = (struct rb_class_cc_entries *)ccs_obj;
2188 }
2189 else {
2190 // TODO: required?
2191 ccs = vm_ccs_create(klass, cc_tbl, mid, cme);
2192 }
2193 }
2194
2195 cme = rb_check_overloaded_cme(cme, ci);
2196
2197 const struct rb_callcache *cc = vm_cc_new(klass, cme, vm_call_general, cc_type_normal);
2198 vm_ccs_push(cc_tbl, mid, ccs, ci, cc);
2199
2200 VM_ASSERT(vm_cc_cme(cc) != NULL);
2201 VM_ASSERT(cme->called_id == mid);
2202 VM_ASSERT(vm_cc_cme(cc)->called_id == mid);
2203
2204 if (original_cc_table != cc_tbl) {
2205 RB_OBJ_ATOMIC_WRITE(klass, &RCLASS_WRITABLE_CC_TBL(klass), cc_tbl);
2206 }
2207
2208 return cc;
2209}
2210
2211static const struct rb_callcache *
2212vm_lookup_cc(const VALUE klass, const struct rb_callinfo * const ci, ID mid)
2213{
2214 VALUE cc_tbl;
2215 struct rb_class_cc_entries *ccs;
2216retry:
2217 cc_tbl = RUBY_ATOMIC_VALUE_LOAD(RCLASS_WRITABLE_CC_TBL(klass));
2218 ccs = NULL;
2219
2220 if (cc_tbl) {
2221 // CCS data is keyed on method id, so we don't need the method id
2222 // for doing comparisons in the `for` loop below.
2223
2224 VALUE ccs_obj;
2225 if (rb_managed_id_table_lookup(cc_tbl, mid, &ccs_obj)) {
2226 ccs = (struct rb_class_cc_entries *)ccs_obj;
2227 const int ccs_len = ccs->len;
2228
2229 if (UNLIKELY(METHOD_ENTRY_INVALIDATED(ccs->cme))) {
2230 RB_VM_LOCKING() {
2231 vm_evict_cc(klass, cc_tbl, mid);
2232 }
2233 goto retry;
2234 }
2235 else {
2236 VM_ASSERT(vm_ccs_verify(ccs, mid, klass));
2237
2238 // We already know the method id is correct because we had
2239 // to look up the ccs_data by method id. We compare argc and
2240 // flag, plus the keyword argument count: two call sites with
2241 // the same argc and flag can still differ in how many of the
2242 // arguments are keywords (e.g. `m(a: 1, b: 2)` vs `m(1, b: 2)`),
2243 // which selects a different argument setup fastpath.
2244 const struct rb_callinfo_kwarg *kwarg = vm_ci_kwarg(ci);
2245 unsigned int argc = vm_ci_argc(ci);
2246 unsigned int flag = vm_ci_flag(ci);
2247 unsigned int kw_len = kwarg ? kwarg->keyword_len : 0;
2248
2249 for (int i=0; i<ccs_len; i++) {
2250 unsigned int ccs_ci_argc = ccs->entries[i].argc;
2251 unsigned int ccs_ci_flag = ccs->entries[i].flag;
2252 unsigned int ccs_ci_kw_len = ccs->entries[i].kw_len;
2253 const struct rb_callcache *ccs_cc = ccs->entries[i].cc;
2254
2255 VM_ASSERT(IMEMO_TYPE_P(ccs_cc, imemo_callcache));
2256
2257 if (ccs_ci_argc == argc && ccs_ci_flag == flag && ccs_ci_kw_len == kw_len) {
2258 RB_DEBUG_COUNTER_INC(cc_found_in_ccs);
2259
2260 VM_ASSERT(vm_cc_cme(ccs_cc)->called_id == mid);
2261 VM_ASSERT(ccs_cc->klass == klass);
2262 VM_ASSERT(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(ccs_cc)));
2263
2264 return ccs_cc;
2265 }
2266 }
2267 }
2268 }
2269 }
2270
2271 RB_GC_GUARD(cc_tbl);
2272 return NULL;
2273}
2274
2275static const struct rb_callcache *
2276vm_search_cc(const VALUE klass, const struct rb_callinfo * const ci)
2277{
2278 const ID mid = vm_ci_mid(ci);
2279
2280 const struct rb_callcache *cc = vm_lookup_cc(klass, ci, mid);
2281 if (cc) {
2282 return cc;
2283 }
2284
2285 RB_VM_LOCKING() {
2286 if (rb_multi_ractor_p()) {
2287 // The CC may have been populated by another ractor while we were waiting on the lock,
2288 // so we must lookup a second time.
2289 cc = vm_lookup_cc(klass, ci, mid);
2290 }
2291
2292 if (!cc) {
2293 cc = vm_populate_cc(klass, ci, mid);
2294 }
2295 }
2296
2297 return cc;
2298}
2299
2300const struct rb_callcache *
2301rb_vm_search_method_slowpath(const struct rb_callinfo *ci, VALUE klass)
2302{
2303 const struct rb_callcache *cc;
2304
2305 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2306
2307 cc = vm_search_cc(klass, ci);
2308
2309 VM_ASSERT(cc);
2310 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
2311 VM_ASSERT(cc == vm_cc_empty() || cc->klass == klass);
2312 VM_ASSERT(cc == vm_cc_empty() || callable_method_entry_p(vm_cc_cme(cc)));
2313 VM_ASSERT(cc == vm_cc_empty() || !METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)));
2314 VM_ASSERT(cc == vm_cc_empty() || vm_cc_cme(cc)->called_id == vm_ci_mid(ci));
2315
2316 return cc;
2317}
2318
2319static const struct rb_callcache *
2320vm_search_method_slowpath0(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2321{
2322#if USE_DEBUG_COUNTER
2323 const struct rb_callcache *old_cc = cd->cc;
2324#endif
2325
2326 const struct rb_callcache *cc = rb_vm_search_method_slowpath(cd->ci, klass);
2327
2328#if OPT_INLINE_METHOD_CACHE
2329 cd->cc = cc;
2330
2331 const struct rb_callcache *empty_cc = &vm_empty_cc;
2332 if (cd_owner && cc != empty_cc) {
2333 RB_OBJ_WRITTEN(cd_owner, Qundef, cc);
2334 }
2335
2336#if USE_DEBUG_COUNTER
2337 if (!old_cc || old_cc == empty_cc) {
2338 // empty
2339 RB_DEBUG_COUNTER_INC(mc_inline_miss_empty);
2340 }
2341 else if (old_cc == cc) {
2342 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cc);
2343 }
2344 else if (vm_cc_cme(old_cc) == vm_cc_cme(cc)) {
2345 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_cme);
2346 }
2347 else if (vm_cc_cme(old_cc) && vm_cc_cme(cc) &&
2348 vm_cc_cme(old_cc)->def == vm_cc_cme(cc)->def) {
2349 RB_DEBUG_COUNTER_INC(mc_inline_miss_same_def);
2350 }
2351 else {
2352 RB_DEBUG_COUNTER_INC(mc_inline_miss_diff);
2353 }
2354#endif
2355#endif // OPT_INLINE_METHOD_CACHE
2356
2357 VM_ASSERT(vm_cc_cme(cc) == NULL ||
2358 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci));
2359
2360 return cc;
2361}
2362
2363ALWAYS_INLINE(static const struct rb_callcache *vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass));
2364static const struct rb_callcache *
2365vm_search_method_fastpath(VALUE cd_owner, struct rb_call_data *cd, VALUE klass)
2366{
2367 const struct rb_callcache *cc = cd->cc;
2368
2369 VM_ASSERT_TYPE2(klass, T_CLASS, T_ICLASS);
2370
2371#if OPT_INLINE_METHOD_CACHE
2372 if (LIKELY(vm_cc_class_check(cc, klass))) {
2373 if (LIKELY(!METHOD_ENTRY_INVALIDATED(vm_cc_cme(cc)))) {
2374 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
2375 RB_DEBUG_COUNTER_INC(mc_inline_hit);
2376 VM_ASSERT(vm_cc_cme(cc) == NULL || // not found
2377 (vm_ci_flag(cd->ci) & VM_CALL_SUPER) || // search_super w/ define_method
2378 vm_cc_cme(cc)->called_id == vm_ci_mid(cd->ci)); // cme->called_id == ci->mid
2379
2380 return cc;
2381 }
2382 RB_DEBUG_COUNTER_INC(mc_inline_miss_invalidated);
2383 }
2384 else {
2385 RB_DEBUG_COUNTER_INC(mc_inline_miss_klass);
2386 }
2387#endif
2388
2389 return vm_search_method_slowpath0(cd_owner, cd, klass);
2390}
2391
2392static const struct rb_callable_method_entry_struct *
2393vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2394{
2395 VALUE klass = CLASS_OF(recv);
2396 VM_ASSERT(klass != Qfalse);
2397 VM_ASSERT(RBASIC_CLASS(klass) == 0 || rb_obj_is_kind_of(klass, rb_cClass));
2398
2399 const struct rb_callcache *cc = vm_search_method_fastpath(cd_owner, cd, klass);
2400 return vm_cc_cme(cc);
2401}
2402
2403const struct rb_callable_method_entry_struct *
2404rb_zjit_vm_search_method(VALUE cd_owner, struct rb_call_data *cd, VALUE recv)
2405{
2406 return vm_search_method(cd_owner, cd, recv);
2407}
2408
2409#if __has_attribute(transparent_union)
2410 typedef union {
2411 VALUE (*anyargs)(ANYARGS);
2412 VALUE (*f00)(VALUE);
2413 VALUE (*f01)(VALUE, VALUE);
2414 VALUE (*f02)(VALUE, VALUE, VALUE);
2415 VALUE (*f03)(VALUE, VALUE, VALUE, VALUE);
2416 VALUE (*f04)(VALUE, VALUE, VALUE, VALUE, VALUE);
2417 VALUE (*f05)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2418 VALUE (*f06)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2419 VALUE (*f07)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2420 VALUE (*f08)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2421 VALUE (*f09)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2422 VALUE (*f10)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2423 VALUE (*f11)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2424 VALUE (*f12)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2425 VALUE (*f13)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2426 VALUE (*f14)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2427 VALUE (*f15)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE);
2428 VALUE (*fm1)(int, union { VALUE *x; const VALUE *y; } __attribute__((__transparent_union__)), VALUE);
2429} __attribute__((__transparent_union__)) cfunc_type;
2430# define make_cfunc_type(f) (cfunc_type){.anyargs = (VALUE (*)(ANYARGS))(f)}
2431#else
2432typedef VALUE (*cfunc_type)(ANYARGS);
2433# define make_cfunc_type(f) (cfunc_type)(f)
2434#endif
2435
2436static inline int
2437check_cfunc(const rb_callable_method_entry_t *me, cfunc_type func)
2438{
2439 if (! me) {
2440 return false;
2441 }
2442 else {
2443 VM_ASSERT(IMEMO_TYPE_P(me, imemo_ment));
2444 VM_ASSERT(callable_method_entry_p(me));
2445 VM_ASSERT(me->def);
2446 if (me->def->type != VM_METHOD_TYPE_CFUNC) {
2447 return false;
2448 }
2449 else {
2450#if __has_attribute(transparent_union)
2451 return me->def->body.cfunc.func == func.anyargs;
2452#else
2453 return me->def->body.cfunc.func == func;
2454#endif
2455 }
2456 }
2457}
2458
2459static inline int
2460check_method_basic_definition(const rb_callable_method_entry_t *me)
2461{
2462 return me && METHOD_ENTRY_BASIC(me);
2463}
2464
2465static inline int
2466vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2467{
2468 VM_ASSERT(iseq != NULL);
2469 const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv);
2470 return check_cfunc(cme, func);
2471}
2472
2473bool
2474rb_zjit_cme_is_cfunc(const rb_callable_method_entry_t *me, const cfunc_type func)
2475{
2476 return check_cfunc(me, func);
2477}
2478
2479int
2480rb_vm_method_cfunc_is(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv, cfunc_type func)
2481{
2482 return vm_method_cfunc_is(iseq, cd, recv, func);
2483}
2484
2485#define check_cfunc(me, func) check_cfunc(me, make_cfunc_type(func))
2486#define vm_method_cfunc_is(iseq, cd, recv, func) vm_method_cfunc_is(iseq, cd, recv, make_cfunc_type(func))
2487
2488#define EQ_UNREDEFINED_P(t) BASIC_OP_UNREDEFINED_P(BOP_EQ, t##_REDEFINED_OP_FLAG)
2489
2490static inline bool
2491FIXNUM_2_P(VALUE a, VALUE b)
2492{
2493 /* FIXNUM_P(a) && FIXNUM_P(b)
2494 * == ((a & 1) && (b & 1))
2495 * == a & b & 1 */
2496 SIGNED_VALUE x = a;
2497 SIGNED_VALUE y = b;
2498 SIGNED_VALUE z = x & y & 1;
2499 return z == 1;
2500}
2501
2502static inline bool
2503FLONUM_2_P(VALUE a, VALUE b)
2504{
2505#if USE_FLONUM
2506 /* FLONUM_P(a) && FLONUM_P(b)
2507 * == ((a & 3) == 2) && ((b & 3) == 2)
2508 * == ! ((a ^ 2) | (b ^ 2) & 3)
2509 */
2510 SIGNED_VALUE x = a;
2511 SIGNED_VALUE y = b;
2512 SIGNED_VALUE z = ((x ^ 2) | (y ^ 2)) & 3;
2513 return !z;
2514#else
2515 return false;
2516#endif
2517}
2518
2519static VALUE
2520opt_equality_specialized(VALUE recv, VALUE obj)
2521{
2522 if (FIXNUM_2_P(recv, obj) && EQ_UNREDEFINED_P(INTEGER)) {
2523 goto compare_by_identity;
2524 }
2525 else if (FLONUM_2_P(recv, obj) && EQ_UNREDEFINED_P(FLOAT)) {
2526 goto compare_by_identity;
2527 }
2528 else if (STATIC_SYM_P(recv) && STATIC_SYM_P(obj) && EQ_UNREDEFINED_P(SYMBOL)) {
2529 goto compare_by_identity;
2530 }
2531 else if (SPECIAL_CONST_P(recv)) {
2532 //
2533 }
2534 else if (RBASIC_CLASS(recv) == rb_cFloat && RB_FLOAT_TYPE_P(obj) && EQ_UNREDEFINED_P(FLOAT)) {
2535 double a = RFLOAT_VALUE(recv);
2536 double b = RFLOAT_VALUE(obj);
2537
2538 return RBOOL(a == b);
2539 }
2540 else if (RBASIC_CLASS(recv) == rb_cString && EQ_UNREDEFINED_P(STRING)) {
2541 if (recv == obj) {
2542 return Qtrue;
2543 }
2544 else if (RB_TYPE_P(obj, T_STRING)) {
2545 return rb_str_eql_internal(obj, recv);
2546 }
2547 }
2548 return Qundef;
2549
2550 compare_by_identity:
2551 return RBOOL(recv == obj);
2552}
2553
2554static VALUE
2555opt_equality(const rb_iseq_t *cd_owner, VALUE recv, VALUE obj, CALL_DATA cd)
2556{
2557 VM_ASSERT(cd_owner != NULL);
2558
2559 VALUE val = opt_equality_specialized(recv, obj);
2560 if (!UNDEF_P(val)) return val;
2561
2562 if (!vm_method_cfunc_is(cd_owner, cd, recv, rb_obj_equal)) {
2563 return Qundef;
2564 }
2565 else {
2566 return RBOOL(recv == obj);
2567 }
2568}
2569
2570#undef EQ_UNREDEFINED_P
2571
2572static inline const struct rb_callcache *gccct_method_search(rb_execution_context_t *ec, VALUE recv, ID mid, const struct rb_callinfo *ci); // vm_eval.c
2573NOINLINE(static VALUE opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid));
2574
2575static VALUE
2576opt_equality_by_mid_slowpath(VALUE recv, VALUE obj, ID mid)
2577{
2578 const struct rb_callcache *cc = gccct_method_search(GET_EC(), recv, mid, &VM_CI_ON_STACK(mid, 0, 1, NULL));
2579
2580 if (cc && check_cfunc(vm_cc_cme(cc), rb_obj_equal)) {
2581 return RBOOL(recv == obj);
2582 }
2583 else {
2584 return Qundef;
2585 }
2586}
2587
2588static VALUE
2589opt_equality_by_mid(VALUE recv, VALUE obj, ID mid)
2590{
2591 VALUE val = opt_equality_specialized(recv, obj);
2592 if (!UNDEF_P(val)) {
2593 return val;
2594 }
2595 else {
2596 return opt_equality_by_mid_slowpath(recv, obj, mid);
2597 }
2598}
2599
2600 VALUE
2601rb_equal_opt(VALUE obj1, VALUE obj2)
2602{
2603 return opt_equality_by_mid(obj1, obj2, idEq);
2604}
2605
2606 VALUE
2607rb_eql_opt(VALUE obj1, VALUE obj2)
2608{
2609 return opt_equality_by_mid(obj1, obj2, idEqlP);
2610}
2611
2612extern VALUE rb_vm_call0(rb_execution_context_t *ec, VALUE, ID, int, const VALUE*, const rb_callable_method_entry_t *, int kw_splat);
2613extern VALUE rb_vm_call_with_refinements(rb_execution_context_t *, VALUE, ID, int, const VALUE *, int);
2614
2615static VALUE
2616check_match(rb_execution_context_t *ec, VALUE pattern, VALUE target, enum vm_check_match_type type)
2617{
2618 switch (type) {
2619 case VM_CHECKMATCH_TYPE_WHEN:
2620 return pattern;
2621 case VM_CHECKMATCH_TYPE_RESCUE:
2622 if (!rb_obj_is_kind_of(pattern, rb_cModule)) {
2623 rb_raise(rb_eTypeError, "class or module required for rescue clause");
2624 }
2625 /* fall through */
2626 case VM_CHECKMATCH_TYPE_CASE: {
2627 return rb_vm_call_with_refinements(ec, pattern, idEqq, 1, &target, RB_NO_KEYWORDS);
2628 }
2629 default:
2630 rb_bug("check_match: unreachable");
2631 }
2632}
2633
2634
2635static inline VALUE
2636double_cmp_lt(double a, double b)
2637{
2638 return RBOOL(a < b);
2639}
2640
2641static inline VALUE
2642double_cmp_le(double a, double b)
2643{
2644 return RBOOL(a <= b);
2645}
2646
2647static inline VALUE
2648double_cmp_gt(double a, double b)
2649{
2650 return RBOOL(a > b);
2651}
2652
2653static inline VALUE
2654double_cmp_ge(double a, double b)
2655{
2656 return RBOOL(a >= b);
2657}
2658
2659// Copied by vm_dump.c
2660static inline VALUE *
2661vm_base_ptr(const rb_control_frame_t *cfp)
2662{
2663 const rb_control_frame_t *prev_cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
2664
2665 if (cfp->iseq && VM_FRAME_RUBYFRAME_P(cfp)) {
2666 VALUE *bp = prev_cfp->sp + ISEQ_BODY(cfp->iseq)->local_table_size + VM_ENV_DATA_SIZE;
2667
2668 if (ISEQ_BODY(cfp->iseq)->param.flags.forwardable && VM_ENV_LOCAL_P(cfp->ep)) {
2669 int lts = ISEQ_BODY(cfp->iseq)->local_table_size;
2670 int params = ISEQ_BODY(cfp->iseq)->param.size;
2671
2672 CALL_INFO ci = (CALL_INFO)cfp->ep[-(VM_ENV_DATA_SIZE + (lts - params))]; // skip EP stuff, CI should be last local
2673 bp += vm_ci_argc(ci);
2674 }
2675
2676 if (ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_METHOD || VM_FRAME_BMETHOD_P(cfp)) {
2677 /* adjust `self' */
2678 bp += 1;
2679 }
2680#if VM_DEBUG_BP_CHECK
2681 if (bp != cfp->bp_check) {
2682 ruby_debug_printf("bp_check: %ld, bp: %ld\n",
2683 (long)(cfp->bp_check - GET_EC()->vm_stack),
2684 (long)(bp - GET_EC()->vm_stack));
2685 rb_bug("vm_base_ptr: unreachable");
2686 }
2687#endif
2688 return bp;
2689 }
2690 else {
2691 return NULL;
2692 }
2693}
2694
2695 VALUE *
2696rb_vm_base_ptr(const rb_control_frame_t *cfp)
2697{
2698 return vm_base_ptr(cfp);
2699}
2700
2701/* method call processes with call_info */
2702
2703#include "vm_args.c"
2704
2705static inline VALUE vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc, int param_size, int local_size);
2706ALWAYS_INLINE(static VALUE vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me, int opt_pc, int param_size, int local_size));
2707static inline VALUE vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc);
2708static VALUE vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
2709static VALUE vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2710static VALUE vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2711static inline VALUE vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling);
2712
2713static vm_call_handler vm_call_iseq_setup_func(const struct rb_callinfo *ci, const int param_size, const int local_size);
2714
2715static VALUE
2716vm_call_iseq_setup_tailcall_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2717{
2718 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_tailcall_0start);
2719
2720 return vm_call_iseq_setup_tailcall(ec, cfp, calling, 0);
2721}
2722
2723static VALUE
2724vm_call_iseq_setup_normal_0start(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
2725{
2726 RB_DEBUG_COUNTER_INC(ccf_iseq_setup_0start);
2727
2728 const struct rb_callcache *cc = calling->cc;
2729 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
2730 int param = ISEQ_BODY(iseq)->param.size;
2731 int local = ISEQ_BODY(iseq)->local_table_size;
2732 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
2733}
2734
2735bool
2736rb_simple_iseq_p(const rb_iseq_t *iseq)
2737{
2738 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2739 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2740 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2741 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2742 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2743 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2744 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2745 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2746}
2747
2748bool
2749rb_iseq_only_optparam_p(const rb_iseq_t *iseq)
2750{
2751 return ISEQ_BODY(iseq)->param.flags.has_opt == TRUE &&
2752 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2753 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2754 ISEQ_BODY(iseq)->param.flags.has_kw == FALSE &&
2755 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2756 ISEQ_BODY(iseq)->param.flags.accepts_no_kwarg == FALSE &&
2757 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2758 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2759}
2760
2761bool
2762rb_iseq_only_kwparam_p(const rb_iseq_t *iseq)
2763{
2764 return ISEQ_BODY(iseq)->param.flags.has_opt == FALSE &&
2765 ISEQ_BODY(iseq)->param.flags.has_rest == FALSE &&
2766 ISEQ_BODY(iseq)->param.flags.has_post == FALSE &&
2767 ISEQ_BODY(iseq)->param.flags.has_kw == TRUE &&
2768 ISEQ_BODY(iseq)->param.flags.has_kwrest == FALSE &&
2769 ISEQ_BODY(iseq)->param.flags.forwardable == FALSE &&
2770 ISEQ_BODY(iseq)->param.flags.has_block == FALSE;
2771}
2772
2773#define ALLOW_HEAP_ARGV (-2)
2774#define ALLOW_HEAP_ARGV_KEEP_KWSPLAT (-3)
2775
2776static inline bool
2777vm_caller_setup_arg_splat(rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE ary, int max_args)
2778{
2779 vm_check_canary(GET_EC(), cfp->sp);
2780 bool ret = false;
2781
2782 if (!NIL_P(ary)) {
2783 const VALUE *ptr = RARRAY_CONST_PTR(ary);
2784 long len = RARRAY_LEN(ary);
2785 int argc = calling->argc;
2786
2787 if (UNLIKELY(max_args <= ALLOW_HEAP_ARGV && len + argc > VM_ARGC_STACK_MAX)) {
2788 /* Avoid SystemStackError when splatting large arrays by storing arguments in
2789 * a temporary array, instead of trying to keeping arguments on the VM stack.
2790 */
2791 VALUE *argv = cfp->sp - argc;
2792 VALUE argv_ary = rb_ary_hidden_new(len + argc + 1);
2793 rb_ary_cat(argv_ary, argv, argc);
2794 rb_ary_cat(argv_ary, ptr, len);
2795 cfp->sp -= argc - 1;
2796 cfp->sp[-1] = argv_ary;
2797 calling->argc = 1;
2798 calling->heap_argv = argv_ary;
2799 RB_GC_GUARD(ary);
2800 }
2801 else {
2802 long i;
2803
2804 if (max_args >= 0 && len + argc > max_args) {
2805 /* If only a given max_args is allowed, copy up to max args.
2806 * Used by vm_callee_setup_block_arg for non-lambda blocks,
2807 * where additional arguments are ignored.
2808 *
2809 * Also, copy up to one more argument than the maximum,
2810 * in case it is an empty keyword hash that will be removed.
2811 */
2812 calling->argc += len - (max_args - argc + 1);
2813 len = max_args - argc + 1;
2814 ret = true;
2815 }
2816 else {
2817 /* Unset heap_argv if set originally. Can happen when
2818 * forwarding modified arguments, where heap_argv was used
2819 * originally, but heap_argv not supported by the forwarded
2820 * method in all cases.
2821 */
2822 calling->heap_argv = 0;
2823 }
2824 CHECK_VM_STACK_OVERFLOW(cfp, len);
2825
2826 for (i = 0; i < len; i++) {
2827 *cfp->sp++ = ptr[i];
2828 }
2829 calling->argc += i;
2830 }
2831 }
2832
2833 return ret;
2834}
2835
2836static inline void
2837vm_caller_setup_arg_kw(rb_control_frame_t *cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci)
2838{
2839 const VALUE *const passed_keywords = vm_ci_kwarg(ci)->keywords;
2840 const int kw_len = vm_ci_kwarg(ci)->keyword_len;
2841 const VALUE h = rb_hash_new_with_size(kw_len);
2842 VALUE *sp = cfp->sp;
2843 int i;
2844
2845 for (i=0; i<kw_len; i++) {
2846 rb_hash_aset(h, passed_keywords[i], (sp - kw_len)[i]);
2847 }
2848 (sp-kw_len)[0] = h;
2849
2850 cfp->sp -= kw_len - 1;
2851 calling->argc -= kw_len - 1;
2852 calling->kw_splat = 1;
2853}
2854
2855static inline VALUE
2856vm_caller_setup_keyword_hash(const struct rb_callinfo *ci, VALUE keyword_hash)
2857{
2858 if (UNLIKELY(!RB_TYPE_P(keyword_hash, T_HASH))) {
2859 if (keyword_hash != Qnil) {
2860 /* Convert a non-hash keyword splat to a new hash */
2861 keyword_hash = rb_hash_dup(rb_to_hash_type(keyword_hash));
2862 }
2863 }
2864 else if (!IS_ARGS_KW_SPLAT_MUT(ci) && !RHASH_EMPTY_P(keyword_hash)) {
2865 /* Convert a hash keyword splat to a new hash unless
2866 * a mutable keyword splat was passed.
2867 * Skip allocating new hash for empty keyword splat, as empty
2868 * keyword splat will be ignored by both callers.
2869 */
2870 keyword_hash = rb_hash_dup(keyword_hash);
2871 }
2872 return keyword_hash;
2873}
2874
2875static inline void
2876CALLER_SETUP_ARG(struct rb_control_frame_struct *restrict cfp,
2877 struct rb_calling_info *restrict calling,
2878 const struct rb_callinfo *restrict ci, int max_args)
2879{
2880 if (UNLIKELY(IS_ARGS_SPLAT(ci))) {
2881 if (IS_ARGS_KW_SPLAT(ci)) {
2882 // f(*a, **kw)
2883 VM_ASSERT(calling->kw_splat == 1);
2884
2885 cfp->sp -= 2;
2886 calling->argc -= 2;
2887 VALUE ary = cfp->sp[0];
2888 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[1]);
2889
2890 // splat a
2891 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) return;
2892
2893 // put kw
2894 if (kwh != Qnil && !RHASH_EMPTY_P(kwh)) {
2895 if (UNLIKELY(calling->heap_argv)) {
2896 rb_ary_push(calling->heap_argv, kwh);
2897 ((struct RHash *)kwh)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
2898 if (max_args != ALLOW_HEAP_ARGV_KEEP_KWSPLAT) {
2899 calling->kw_splat = 0;
2900 }
2901 }
2902 else {
2903 cfp->sp[0] = kwh;
2904 cfp->sp++;
2905 calling->argc++;
2906
2907 VM_ASSERT(calling->kw_splat == 1);
2908 }
2909 }
2910 else {
2911 calling->kw_splat = 0;
2912 }
2913 }
2914 else {
2915 // f(*a)
2916 VM_ASSERT(calling->kw_splat == 0);
2917
2918 cfp->sp -= 1;
2919 calling->argc -= 1;
2920 VALUE ary = cfp->sp[0];
2921
2922 if (vm_caller_setup_arg_splat(cfp, calling, ary, max_args)) {
2923 goto check_keyword;
2924 }
2925
2926 // check the last argument
2927 VALUE last_hash, argv_ary;
2928 if (UNLIKELY(argv_ary = calling->heap_argv)) {
2929 if (!IS_ARGS_KEYWORD(ci) &&
2930 RARRAY_LEN(argv_ary) > 0 &&
2931 RB_TYPE_P((last_hash = rb_ary_last(0, NULL, argv_ary)), T_HASH) &&
2932 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2933
2934 rb_ary_pop(argv_ary);
2935 if (!RHASH_EMPTY_P(last_hash)) {
2936 rb_ary_push(argv_ary, rb_hash_dup(last_hash));
2937 calling->kw_splat = 1;
2938 }
2939 }
2940 }
2941 else {
2942check_keyword:
2943 if (!IS_ARGS_KEYWORD(ci) &&
2944 calling->argc > 0 &&
2945 RB_TYPE_P((last_hash = cfp->sp[-1]), T_HASH) &&
2946 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS)) {
2947
2948 if (RHASH_EMPTY_P(last_hash)) {
2949 calling->argc--;
2950 cfp->sp -= 1;
2951 }
2952 else {
2953 cfp->sp[-1] = rb_hash_dup(last_hash);
2954 calling->kw_splat = 1;
2955 }
2956 }
2957 }
2958 }
2959 }
2960 else if (UNLIKELY(IS_ARGS_KW_SPLAT(ci))) {
2961 // f(**kw)
2962 VM_ASSERT(calling->kw_splat == 1);
2963 VALUE kwh = vm_caller_setup_keyword_hash(ci, cfp->sp[-1]);
2964
2965 if (kwh == Qnil || RHASH_EMPTY_P(kwh)) {
2966 cfp->sp--;
2967 calling->argc--;
2968 calling->kw_splat = 0;
2969 }
2970 else {
2971 cfp->sp[-1] = kwh;
2972 }
2973 }
2974 else if (UNLIKELY(IS_ARGS_KEYWORD(ci))) {
2975 // f(k1:1, k2:2)
2976 VM_ASSERT(calling->kw_splat == 0);
2977
2978 /* This converts VM_CALL_KWARG style to VM_CALL_KW_SPLAT style
2979 * by creating a keyword hash.
2980 * So, vm_ci_flag(ci) & VM_CALL_KWARG is now inconsistent.
2981 */
2982 vm_caller_setup_arg_kw(cfp, calling, ci);
2983 }
2984}
2985
2986#define USE_OPT_HIST 0
2987
2988#if USE_OPT_HIST
2989#define OPT_HIST_MAX 64
2990static int opt_hist[OPT_HIST_MAX+1];
2991
2992 __attribute__((destructor))
2993static void
2994opt_hist_show_results_at_exit(void)
2995{
2996 for (int i=0; i<OPT_HIST_MAX; i++) {
2997 ruby_debug_printf("opt_hist\t%d\t%d\n", i, opt_hist[i]);
2998 }
2999}
3000#endif
3001
3002static VALUE
3003vm_call_iseq_setup_normal_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3004 struct rb_calling_info *calling)
3005{
3006 const struct rb_callcache *cc = calling->cc;
3007 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3008 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3009 const int opt = calling->argc - lead_num;
3010 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3011 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3012 const int param = ISEQ_BODY(iseq)->param.size;
3013 const int local = ISEQ_BODY(iseq)->local_table_size;
3014 const int delta = opt_num - opt;
3015
3016 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3017
3018#if USE_OPT_HIST
3019 if (opt_pc < OPT_HIST_MAX) {
3020 opt_hist[opt]++;
3021 }
3022 else {
3023 opt_hist[OPT_HIST_MAX]++;
3024 }
3025#endif
3026
3027 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param - delta, local);
3028}
3029
3030static VALUE
3031vm_call_iseq_setup_tailcall_opt_start(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3032 struct rb_calling_info *calling)
3033{
3034 const struct rb_callcache *cc = calling->cc;
3035 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3036 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3037 const int opt = calling->argc - lead_num;
3038 const int opt_pc = (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3039
3040 RB_DEBUG_COUNTER_INC(ccf_iseq_opt);
3041
3042#if USE_OPT_HIST
3043 if (opt_pc < OPT_HIST_MAX) {
3044 opt_hist[opt]++;
3045 }
3046 else {
3047 opt_hist[OPT_HIST_MAX]++;
3048 }
3049#endif
3050
3051 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3052}
3053
3054static void
3055args_setup_kw_parameters(rb_execution_context_t *const ec, const rb_iseq_t *const iseq, const rb_callable_method_entry_t *cme,
3056 VALUE *const passed_values, const int passed_keyword_len, const VALUE *const passed_keywords,
3057 VALUE *const locals);
3058
3059static VALUE
3060vm_call_iseq_forwardable(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3061 struct rb_calling_info *calling)
3062{
3063 const struct rb_callcache *cc = calling->cc;
3064 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3065 int param_size = ISEQ_BODY(iseq)->param.size;
3066 int local_size = ISEQ_BODY(iseq)->local_table_size;
3067
3068 // Setting up local size and param size
3069 VM_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3070
3071 local_size = local_size + vm_ci_argc(calling->cd->ci);
3072 param_size = param_size + vm_ci_argc(calling->cd->ci);
3073
3074 cfp->sp[0] = (VALUE)calling->cd->ci;
3075
3076 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param_size, local_size);
3077}
3078
3079static VALUE
3080vm_call_iseq_setup_kwparm_kwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3081 struct rb_calling_info *calling)
3082{
3083 const struct rb_callinfo *ci = calling->cd->ci;
3084 const struct rb_callcache *cc = calling->cc;
3085
3086 VM_ASSERT(vm_ci_flag(ci) & VM_CALL_KWARG);
3087 RB_DEBUG_COUNTER_INC(ccf_iseq_kw1);
3088
3089 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3090 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3091 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3092 const int ci_kw_len = kw_arg->keyword_len;
3093 const VALUE * const ci_keywords = kw_arg->keywords;
3094 VALUE *argv = cfp->sp - calling->argc;
3095 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3096 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3097 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3098 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3099 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3100
3101 int param = ISEQ_BODY(iseq)->param.size;
3102 int local = ISEQ_BODY(iseq)->local_table_size;
3103 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3104}
3105
3106static VALUE
3107vm_call_iseq_setup_kwparm_nokwarg(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3108 struct rb_calling_info *calling)
3109{
3110 const struct rb_callinfo *MAYBE_UNUSED(ci) = calling->cd->ci;
3111 const struct rb_callcache *cc = calling->cc;
3112
3113 VM_ASSERT((vm_ci_flag(ci) & VM_CALL_KWARG) == 0);
3114 RB_DEBUG_COUNTER_INC(ccf_iseq_kw2);
3115
3116 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3117 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3118 VALUE * const argv = cfp->sp - calling->argc;
3119 VALUE * const klocals = argv + kw_param->bits_start - kw_param->num;
3120
3121 int i;
3122 for (i=0; i<kw_param->num; i++) {
3123 klocals[i] = kw_param->default_values[i];
3124 }
3125 klocals[i] = INT2FIX(0); // kw specify flag
3126 // NOTE:
3127 // nobody check this value, but it should be cleared because it can
3128 // points invalid VALUE (T_NONE objects, raw pointer and so on).
3129
3130 int param = ISEQ_BODY(iseq)->param.size;
3131 int local = ISEQ_BODY(iseq)->local_table_size;
3132 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), 0, param, local);
3133}
3134
3135static VALUE builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
3136
3137static VALUE
3138vm_call_single_noarg_leaf_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp,
3139 struct rb_calling_info *calling)
3140{
3141 const struct rb_builtin_function *bf = calling->cc->aux_.bf;
3142 cfp->sp -= (calling->argc + 1);
3143 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
3144 return builtin_invoker0(ec, calling->recv, NULL, func_ptr);
3145}
3146
3147 VALUE rb_gen_method_name(VALUE owner, VALUE name); // in vm_backtrace.c
3148
3149static void
3150warn_unused_block(const rb_callable_method_entry_t *cme, const rb_iseq_t *iseq, void *pc)
3151{
3152 rb_vm_t *vm = GET_VM();
3153 set_table *dup_check_table = vm->unused_block_warning_table;
3154 st_data_t key;
3155 bool strict_unused_block = rb_warning_category_enabled_p(RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK);
3156
3157 union {
3158 VALUE v;
3159 unsigned char b[SIZEOF_VALUE];
3160 } k1 = {
3161 .v = (VALUE)pc,
3162 }, k2 = {
3163 .v = (VALUE)cme->def,
3164 };
3165
3166 // relax check
3167 if (!strict_unused_block) {
3168 key = (st_data_t)cme->def->original_id;
3169
3170 if (set_table_lookup(dup_check_table, key)) {
3171 return;
3172 }
3173 }
3174
3175 // strict check
3176 // make unique key from pc and me->def pointer
3177 key = 0;
3178 for (int i=0; i<SIZEOF_VALUE; i++) {
3179 // fprintf(stderr, "k1:%3d k2:%3d\n", k1.b[i], k2.b[SIZEOF_VALUE-1-i]);
3180 key |= (st_data_t)(k1.b[i] ^ k2.b[SIZEOF_VALUE-1-i]) << (8 * i);
3181 }
3182
3183 if (0) {
3184 fprintf(stderr, "SIZEOF_VALUE:%d\n", SIZEOF_VALUE);
3185 fprintf(stderr, "pc:%p def:%p\n", pc, (void *)cme->def);
3186 fprintf(stderr, "key:%p\n", (void *)key);
3187 }
3188
3189 // duplication check
3190 if (set_insert(dup_check_table, key)) {
3191 // already shown
3192 }
3193 else if (RTEST(ruby_verbose) || strict_unused_block) {
3194 VALUE m_loc = rb_method_entry_location((const rb_method_entry_t *)cme);
3195 VALUE name = rb_gen_method_name(cme->defined_class, ISEQ_BODY(iseq)->location.base_label);
3196
3197 if (!NIL_P(m_loc)) {
3198 rb_warn("the block passed to '%"PRIsVALUE"' defined at %"PRIsVALUE":%"PRIsVALUE" may be ignored",
3199 name, RARRAY_AREF(m_loc, 0), RARRAY_AREF(m_loc, 1));
3200 }
3201 else {
3202 rb_warn("the block may be ignored because '%"PRIsVALUE"' does not use a block", name);
3203 }
3204 }
3205}
3206
3207static inline int
3208vm_callee_setup_arg(rb_execution_context_t *ec, struct rb_calling_info *calling,
3209 const rb_iseq_t *iseq, VALUE *argv, int param_size, int local_size)
3210{
3211 const struct rb_callinfo *ci = calling->cd->ci;
3212 const struct rb_callcache *cc = calling->cc;
3213
3214 VM_ASSERT((vm_ci_argc(ci), 1));
3215 VM_ASSERT(vm_cc_cme(cc) != NULL);
3216
3217 if (UNLIKELY(!ISEQ_BODY(iseq)->param.flags.use_block &&
3218 calling->block_handler != VM_BLOCK_HANDLER_NONE &&
3219 !(vm_ci_flag(calling->cd->ci) & (VM_CALL_OPT_SEND | VM_CALL_SUPER)))) {
3220 warn_unused_block(vm_cc_cme(cc), iseq, (void *)ec->cfp->pc);
3221 }
3222
3223 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_KW_SPLAT))) {
3224 if (LIKELY(rb_simple_iseq_p(iseq))) {
3225 rb_control_frame_t *cfp = ec->cfp;
3226 int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3227 CALLER_SETUP_ARG(cfp, calling, ci, lead_num);
3228
3229 if (calling->argc != lead_num) {
3230 argument_arity_error(ec, iseq, vm_cc_cme(cc), calling->argc, lead_num, lead_num);
3231 }
3232
3233 //VM_ASSERT(ci == calling->cd->ci);
3234 VM_ASSERT(cc == calling->cc);
3235
3236 if (vm_call_iseq_optimizable_p(ci, cc)) {
3237 if ((iseq->body->builtin_attrs & BUILTIN_ATTR_SINGLE_NOARG_LEAF) && ruby_vm_c_events_enabled == 0) {
3238 VM_ASSERT(iseq->body->builtin_attrs & BUILTIN_ATTR_LEAF);
3239 vm_cc_bf_set(cc, (void *)iseq->body->iseq_encoded[1]);
3240 CC_SET_FASTPATH(cc, vm_call_single_noarg_leaf_builtin, true);
3241 }
3242 else {
3243 CC_SET_FASTPATH(cc, vm_call_iseq_setup_func(ci, param_size, local_size), true);
3244 }
3245 }
3246 return 0;
3247 }
3248 else if (rb_iseq_only_optparam_p(iseq)) {
3249 rb_control_frame_t *cfp = ec->cfp;
3250
3251 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3252 const int opt_num = ISEQ_BODY(iseq)->param.opt_num;
3253
3254 CALLER_SETUP_ARG(cfp, calling, ci, lead_num + opt_num);
3255 const int argc = calling->argc;
3256 const int opt = argc - lead_num;
3257
3258 if (opt < 0 || opt > opt_num) {
3259 argument_arity_error(ec, iseq, vm_cc_cme(cc), argc, lead_num, lead_num + opt_num);
3260 }
3261
3262 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3263 CC_SET_FASTPATH(cc, vm_call_iseq_setup_normal_opt_start,
3264 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3265 vm_call_cacheable(ci, cc));
3266 }
3267 else {
3268 CC_SET_FASTPATH(cc, vm_call_iseq_setup_tailcall_opt_start,
3269 !IS_ARGS_SPLAT(ci) && !IS_ARGS_KEYWORD(ci) &&
3270 vm_call_cacheable(ci, cc));
3271 }
3272
3273 /* initialize opt vars for self-references */
3274 VM_ASSERT((int)ISEQ_BODY(iseq)->param.size == lead_num + opt_num);
3275 for (int i=argc; i<lead_num + opt_num; i++) {
3276 argv[i] = Qnil;
3277 }
3278 return (int)ISEQ_BODY(iseq)->param.opt_table[opt];
3279 }
3280 else if (rb_iseq_only_kwparam_p(iseq) && !IS_ARGS_SPLAT(ci)) {
3281 const int lead_num = ISEQ_BODY(iseq)->param.lead_num;
3282 const int argc = calling->argc;
3283 const struct rb_iseq_param_keyword *kw_param = ISEQ_BODY(iseq)->param.keyword;
3284
3285 if (vm_ci_flag(ci) & VM_CALL_KWARG) {
3286 const struct rb_callinfo_kwarg *kw_arg = vm_ci_kwarg(ci);
3287
3288 if (argc - kw_arg->keyword_len == lead_num) {
3289 const int ci_kw_len = kw_arg->keyword_len;
3290 const VALUE * const ci_keywords = kw_arg->keywords;
3291 VALUE * const ci_kws = ALLOCA_N(VALUE, ci_kw_len);
3292 MEMCPY(ci_kws, argv + lead_num, VALUE, ci_kw_len);
3293
3294 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3295 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), ci_kws, ci_kw_len, ci_keywords, klocals);
3296
3297 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_kwarg,
3298 vm_call_cacheable(ci, cc));
3299
3300 return 0;
3301 }
3302 }
3303 else if (argc == lead_num) {
3304 /* no kwarg */
3305 VALUE *const klocals = argv + kw_param->bits_start - kw_param->num;
3306 args_setup_kw_parameters(ec, iseq, vm_cc_cme(cc), NULL, 0, NULL, klocals);
3307
3308 if (klocals[kw_param->num] == INT2FIX(0)) {
3309 /* copy from default_values */
3310 CC_SET_FASTPATH(cc, vm_call_iseq_setup_kwparm_nokwarg,
3311 vm_call_cacheable(ci, cc));
3312 }
3313
3314 return 0;
3315 }
3316 }
3317 }
3318
3319 // Called iseq is using ... param
3320 // def foo(...) # <- iseq for foo will have "forwardable"
3321 //
3322 // We want to set the `...` local to the caller's CI
3323 // foo(1, 2) # <- the ci for this should end up as `...`
3324 //
3325 // So hopefully the stack looks like:
3326 //
3327 // => 1
3328 // => 2
3329 // => *
3330 // => **
3331 // => &
3332 // => ... # <- points at `foo`s CI
3333 // => cref_or_me
3334 // => specval
3335 // => type
3336 //
3337 if (ISEQ_BODY(iseq)->param.flags.forwardable) {
3338 bool can_fastpath = true;
3339
3340 if ((vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
3341 struct rb_forwarding_call_data * forward_cd = (struct rb_forwarding_call_data *)calling->cd;
3342 if (vm_ci_argc(ci) != vm_ci_argc(forward_cd->caller_ci)) {
3343 ci = vm_ci_new_runtime(
3344 vm_ci_mid(ci),
3345 vm_ci_flag(ci),
3346 vm_ci_argc(ci),
3347 vm_ci_kwarg(ci));
3348 }
3349 else {
3350 ci = forward_cd->caller_ci;
3351 }
3352 can_fastpath = false;
3353 }
3354 // C functions calling iseqs will stack allocate a CI,
3355 // so we need to convert it to heap allocated
3356 if (!vm_ci_markable(ci)) {
3357 ci = vm_ci_new_runtime(
3358 vm_ci_mid(ci),
3359 vm_ci_flag(ci),
3360 vm_ci_argc(ci),
3361 vm_ci_kwarg(ci));
3362 can_fastpath = false;
3363 }
3364 argv[param_size - 1] = (VALUE)ci;
3365 CC_SET_FASTPATH(cc, vm_call_iseq_forwardable, can_fastpath);
3366 return 0;
3367 }
3368
3369 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_method);
3370}
3371
3372static void
3373vm_adjust_stack_forwarding(const struct rb_execution_context_struct *ec, struct rb_control_frame_struct *cfp, int argc, VALUE splat)
3374{
3375 // This case is when the caller is using a ... parameter.
3376 // For example `bar(...)`. The call info will have VM_CALL_FORWARDING
3377 // In this case the caller's caller's CI will be on the stack.
3378 //
3379 // For example:
3380 //
3381 // def bar(a, b); a + b; end
3382 // def foo(...); bar(...); end
3383 // foo(1, 2) # <- this CI will be on the stack when we call `bar(...)`
3384 //
3385 // Stack layout will be:
3386 //
3387 // > 1
3388 // > 2
3389 // > CI for foo(1, 2)
3390 // > cref_or_me
3391 // > specval
3392 // > type
3393 // > receiver
3394 // > CI for foo(1, 2), via `getlocal ...`
3395 // > ( SP points here )
3396 const VALUE * lep = VM_CF_LEP(cfp);
3397
3398 const rb_iseq_t *iseq;
3399
3400 // If we're in an escaped environment (lambda for example), get the iseq
3401 // from the captured env.
3402 if (VM_ENV_FLAGS(lep, VM_ENV_FLAG_ESCAPED)) {
3403 rb_env_t * env = (rb_env_t *)lep[VM_ENV_DATA_INDEX_ENV];
3404 iseq = env->iseq;
3405 }
3406 else { // Otherwise use the lep to find the caller
3407 iseq = rb_vm_search_cf_from_ep(ec, cfp, lep)->iseq;
3408 }
3409
3410 // Our local storage is below the args we need to copy
3411 int local_size = ISEQ_BODY(iseq)->local_table_size + argc;
3412
3413 const VALUE * from = lep - (local_size + VM_ENV_DATA_SIZE - 1); // 2 for EP values
3414 VALUE * to = cfp->sp - 1; // clobber the CI
3415
3416 if (RTEST(splat)) {
3417 to -= 1; // clobber the splat array
3418 CHECK_VM_STACK_OVERFLOW0(cfp, to, RARRAY_LEN(splat));
3419 MEMCPY(to, RARRAY_CONST_PTR(splat), VALUE, RARRAY_LEN(splat));
3420 to += RARRAY_LEN(splat);
3421 }
3422
3423 CHECK_VM_STACK_OVERFLOW0(cfp, to, argc);
3424 MEMCPY(to, from, VALUE, argc);
3425 cfp->sp = to + argc;
3426
3427 // Stack layout should now be:
3428 //
3429 // > 1
3430 // > 2
3431 // > CI for foo(1, 2)
3432 // > cref_or_me
3433 // > specval
3434 // > type
3435 // > receiver
3436 // > 1
3437 // > 2
3438 // > ( SP points here )
3439}
3440
3441static VALUE
3442vm_call_iseq_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3443{
3444 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3445
3446 const struct rb_callcache *cc = calling->cc;
3447 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3448 int param_size = ISEQ_BODY(iseq)->param.size;
3449 int local_size = ISEQ_BODY(iseq)->local_table_size;
3450
3451 RUBY_ASSERT(!ISEQ_BODY(iseq)->param.flags.forwardable);
3452
3453 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3454 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3455}
3456
3457static VALUE
3458vm_call_iseq_fwd_setup(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
3459{
3460 RB_DEBUG_COUNTER_INC(ccf_iseq_setup);
3461
3462 const struct rb_callcache *cc = calling->cc;
3463 const rb_iseq_t *iseq = def_iseq_ptr(vm_cc_cme(cc)->def);
3464 int param_size = ISEQ_BODY(iseq)->param.size;
3465 int local_size = ISEQ_BODY(iseq)->local_table_size;
3466
3467 RUBY_ASSERT(ISEQ_BODY(iseq)->param.flags.forwardable);
3468
3469 // Setting up local size and param size
3470 local_size = local_size + vm_ci_argc(calling->cd->ci);
3471 param_size = param_size + vm_ci_argc(calling->cd->ci);
3472
3473 const int opt_pc = vm_callee_setup_arg(ec, calling, iseq, cfp->sp - calling->argc, param_size, local_size);
3474 return vm_call_iseq_setup_2(ec, cfp, calling, opt_pc, param_size, local_size);
3475}
3476
3477static inline VALUE
3478vm_call_iseq_setup_2(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
3479 int opt_pc, int param_size, int local_size)
3480{
3481 const struct rb_callinfo *ci = calling->cd->ci;
3482 const struct rb_callcache *cc = calling->cc;
3483
3484 if (LIKELY(!(vm_ci_flag(ci) & VM_CALL_TAILCALL))) {
3485 return vm_call_iseq_setup_normal(ec, cfp, calling, vm_cc_cme(cc), opt_pc, param_size, local_size);
3486 }
3487 else {
3488 return vm_call_iseq_setup_tailcall(ec, cfp, calling, opt_pc);
3489 }
3490}
3491
3492static inline VALUE
3493vm_call_iseq_setup_normal(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, const rb_callable_method_entry_t *me,
3494 int opt_pc, int param_size, int local_size)
3495{
3496 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3497 VALUE *argv = cfp->sp - calling->argc;
3498 VALUE *sp = argv + param_size;
3499 cfp->sp = argv - 1 /* recv */;
3500
3501 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL, calling->recv,
3502 calling->block_handler, (VALUE)me,
3503 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3504 local_size - param_size,
3505 ISEQ_BODY(iseq)->stack_max);
3506 return Qundef;
3507}
3508
3509static inline VALUE
3510vm_call_iseq_setup_tailcall(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, int opt_pc)
3511{
3512 const struct rb_callcache *cc = calling->cc;
3513 unsigned int i;
3514 VALUE *argv = cfp->sp - calling->argc;
3515 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3516 const rb_iseq_t *iseq = def_iseq_ptr(me->def);
3517 VALUE *src_argv = argv;
3518 VALUE *sp_orig, *sp;
3519 VALUE finish_flag = VM_FRAME_FINISHED_P(cfp) ? VM_FRAME_FLAG_FINISH : 0;
3520
3521 if (VM_BH_FROM_CFP_P(calling->block_handler, cfp)) {
3522 struct rb_captured_block *dst_captured = VM_CFP_TO_CAPTURED_BLOCK(RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp));
3523 const struct rb_captured_block *src_captured = VM_BH_TO_CAPT_BLOCK(calling->block_handler);
3524 dst_captured->code.val = src_captured->code.val;
3525 if (VM_BH_ISEQ_BLOCK_P(calling->block_handler)) {
3526 calling->block_handler = VM_BH_FROM_ISEQ_BLOCK(dst_captured);
3527 }
3528 else {
3529 calling->block_handler = VM_BH_FROM_IFUNC_BLOCK(dst_captured);
3530 }
3531 }
3532
3533 vm_pop_frame(ec, cfp, cfp->ep);
3534 cfp = ec->cfp;
3535
3536 sp_orig = sp = cfp->sp;
3537
3538 /* push self */
3539 sp[0] = calling->recv;
3540 sp++;
3541
3542 /* copy arguments */
3543 for (i=0; i < ISEQ_BODY(iseq)->param.size; i++) {
3544 *sp++ = src_argv[i];
3545 }
3546
3547 vm_push_frame(ec, iseq, VM_FRAME_MAGIC_METHOD | VM_ENV_FLAG_LOCAL | finish_flag,
3548 calling->recv, calling->block_handler, (VALUE)me,
3549 ISEQ_BODY(iseq)->iseq_encoded + opt_pc, sp,
3550 ISEQ_BODY(iseq)->local_table_size - ISEQ_BODY(iseq)->param.size,
3551 ISEQ_BODY(iseq)->stack_max);
3552
3553 cfp->sp = sp_orig;
3554
3555 return Qundef;
3556}
3557
3558static void
3559ractor_unsafe_check(void)
3560{
3561 if (!rb_ractor_main_p()) {
3562 rb_raise(rb_eRactorUnsafeError, "ractor unsafe method called from not main ractor");
3563 }
3564}
3565
3566static VALUE
3567call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3568{
3569 ractor_unsafe_check();
3570 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3571 return (*f)(recv, rb_ary_new4(argc, argv));
3572}
3573
3574static VALUE
3575call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3576{
3577 ractor_unsafe_check();
3578 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3579 return (*f)(argc, argv, recv);
3580}
3581
3582static VALUE
3583call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3584{
3585 ractor_unsafe_check();
3586 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3587 return (*f)(recv);
3588}
3589
3590static VALUE
3591call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3592{
3593 ractor_unsafe_check();
3594 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3595 return (*f)(recv, argv[0]);
3596}
3597
3598static VALUE
3599call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3600{
3601 ractor_unsafe_check();
3602 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3603 return (*f)(recv, argv[0], argv[1]);
3604}
3605
3606static VALUE
3607call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3608{
3609 ractor_unsafe_check();
3610 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3611 return (*f)(recv, argv[0], argv[1], argv[2]);
3612}
3613
3614static VALUE
3615call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3616{
3617 ractor_unsafe_check();
3618 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3619 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3620}
3621
3622static VALUE
3623call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3624{
3625 ractor_unsafe_check();
3626 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3627 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3628}
3629
3630static VALUE
3631call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3632{
3633 ractor_unsafe_check();
3634 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3635 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3636}
3637
3638static VALUE
3639call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3640{
3641 ractor_unsafe_check();
3642 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3643 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3644}
3645
3646static VALUE
3647call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3648{
3649 ractor_unsafe_check();
3650 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3651 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3652}
3653
3654static VALUE
3655call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3656{
3657 ractor_unsafe_check();
3658 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3659 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3660}
3661
3662static VALUE
3663call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3664{
3665 ractor_unsafe_check();
3666 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3667 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3668}
3669
3670static VALUE
3671call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3672{
3673 ractor_unsafe_check();
3674 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3675 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3676}
3677
3678static VALUE
3679call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3680{
3681 ractor_unsafe_check();
3682 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3683 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3684}
3685
3686static VALUE
3687call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3688{
3689 ractor_unsafe_check();
3690 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3691 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3692}
3693
3694static VALUE
3695call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3696{
3697 ractor_unsafe_check();
3698 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3699 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3700}
3701
3702static VALUE
3703call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3704{
3705 ractor_unsafe_check();
3706 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3707 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3708}
3709
3710static VALUE
3711ractor_safe_call_cfunc_m2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3712{
3713 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3714 return (*f)(recv, rb_ary_new4(argc, argv));
3715}
3716
3717static VALUE
3718ractor_safe_call_cfunc_m1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3719{
3720 VALUE(*f)(int, const VALUE *, VALUE) = (VALUE(*)(int, const VALUE *, VALUE))func;
3721 return (*f)(argc, argv, recv);
3722}
3723
3724static VALUE
3725ractor_safe_call_cfunc_0(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3726{
3727 VALUE(*f)(VALUE) = (VALUE(*)(VALUE))func;
3728 return (*f)(recv);
3729}
3730
3731static VALUE
3732ractor_safe_call_cfunc_1(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3733{
3734 VALUE(*f)(VALUE, VALUE) = (VALUE(*)(VALUE, VALUE))func;
3735 return (*f)(recv, argv[0]);
3736}
3737
3738static VALUE
3739ractor_safe_call_cfunc_2(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3740{
3741 VALUE(*f)(VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE))func;
3742 return (*f)(recv, argv[0], argv[1]);
3743}
3744
3745static VALUE
3746ractor_safe_call_cfunc_3(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3747{
3748 VALUE(*f)(VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE))func;
3749 return (*f)(recv, argv[0], argv[1], argv[2]);
3750}
3751
3752static VALUE
3753ractor_safe_call_cfunc_4(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3754{
3755 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE))func;
3756 return (*f)(recv, argv[0], argv[1], argv[2], argv[3]);
3757}
3758
3759static VALUE
3760ractor_safe_call_cfunc_5(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3761{
3762 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3763 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4]);
3764}
3765
3766static VALUE
3767ractor_safe_call_cfunc_6(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3768{
3769 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3770 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
3771}
3772
3773static VALUE
3774ractor_safe_call_cfunc_7(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3775{
3776 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3777 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
3778}
3779
3780static VALUE
3781ractor_safe_call_cfunc_8(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3782{
3783 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3784 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
3785}
3786
3787static VALUE
3788ractor_safe_call_cfunc_9(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3789{
3790 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3791 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
3792}
3793
3794static VALUE
3795ractor_safe_call_cfunc_10(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3796{
3797 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3798 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
3799}
3800
3801static VALUE
3802ractor_safe_call_cfunc_11(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3803{
3804 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3805 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
3806}
3807
3808static VALUE
3809ractor_safe_call_cfunc_12(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3810{
3811 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3812 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
3813}
3814
3815static VALUE
3816ractor_safe_call_cfunc_13(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3817{
3818 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3819 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
3820}
3821
3822static VALUE
3823ractor_safe_call_cfunc_14(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3824{
3825 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3826 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
3827}
3828
3829static VALUE
3830ractor_safe_call_cfunc_15(VALUE recv, int argc, const VALUE *argv, VALUE (*func)(ANYARGS))
3831{
3832 VALUE(*f)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE) = (VALUE(*)(VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE, VALUE))func;
3833 return (*f)(recv, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
3834}
3835
3836static inline int
3837vm_cfp_consistent_p(rb_execution_context_t *ec, const rb_control_frame_t *reg_cfp)
3838{
3839 const int ov_flags = RAISED_STACKOVERFLOW;
3840 if (LIKELY(reg_cfp == ec->cfp + 1)) return TRUE;
3841 if (rb_ec_raised_p(ec, ov_flags)) {
3842 rb_ec_raised_reset(ec, ov_flags);
3843 return TRUE;
3844 }
3845 return FALSE;
3846}
3847
3848#define CHECK_CFP_CONSISTENCY(func) \
3849 (LIKELY(vm_cfp_consistent_p(ec, reg_cfp)) ? (void)0 : \
3850 rb_bug(func ": cfp consistency error (%p, %p)", (void *)reg_cfp, (void *)(ec->cfp+1)))
3851
3852static inline
3853const rb_method_cfunc_t *
3854vm_method_cfunc_entry(const rb_callable_method_entry_t *me)
3855{
3856#if VM_DEBUG_VERIFY_METHOD_CACHE
3857 switch (me->def->type) {
3858 case VM_METHOD_TYPE_CFUNC:
3859 case VM_METHOD_TYPE_NOTIMPLEMENTED:
3860 break;
3861# define METHOD_BUG(t) case VM_METHOD_TYPE_##t: rb_bug("wrong method type: " #t)
3862 METHOD_BUG(ISEQ);
3863 METHOD_BUG(ATTRSET);
3864 METHOD_BUG(IVAR);
3865 METHOD_BUG(BMETHOD);
3866 METHOD_BUG(ZSUPER);
3867 METHOD_BUG(UNDEF);
3868 METHOD_BUG(OPTIMIZED);
3869 METHOD_BUG(MISSING);
3870 METHOD_BUG(REFINED);
3871 METHOD_BUG(ALIAS);
3872# undef METHOD_BUG
3873 default:
3874 rb_bug("wrong method type: %d", me->def->type);
3875 }
3876#endif
3877 return UNALIGNED_MEMBER_PTR(me->def, body.cfunc);
3878}
3879
3880static VALUE
3881vm_call_cfunc_with_frame_(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
3882 int argc, VALUE *argv, VALUE *stack_bottom)
3883{
3884 RB_DEBUG_COUNTER_INC(ccf_cfunc_with_frame);
3885 const struct rb_callinfo *ci = calling->cd->ci;
3886 const struct rb_callcache *cc = calling->cc;
3887 VALUE val;
3888 const rb_callable_method_entry_t *me = vm_cc_cme(cc);
3889 const rb_method_cfunc_t *cfunc = vm_method_cfunc_entry(me);
3890
3891 VALUE recv = calling->recv;
3892 VALUE block_handler = calling->block_handler;
3893 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3894
3895 if (UNLIKELY(calling->kw_splat)) {
3896 frame_type |= VM_FRAME_FLAG_CFRAME_KW;
3897 }
3898
3899 VM_ASSERT(reg_cfp == ec->cfp);
3900
3901 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, me->owner, me->def->original_id);
3902 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, recv, me->def->original_id, vm_ci_mid(ci), me->owner, Qundef);
3903
3904 vm_push_frame(ec, NULL, frame_type, recv,
3905 block_handler, (VALUE)me,
3906 0, ec->cfp->sp, 0, 0);
3907
3908 int len = cfunc->argc;
3909 if (len >= 0) rb_check_arity(argc, len, len);
3910
3911 reg_cfp->sp = stack_bottom;
3912 val = (*cfunc->invoker)(recv, argc, argv, cfunc->func);
3913
3914 CHECK_CFP_CONSISTENCY("vm_call_cfunc");
3915
3916 rb_vm_pop_frame(ec);
3917
3918 VM_ASSERT(ec->cfp->sp == stack_bottom);
3919
3920 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, recv, me->def->original_id, vm_ci_mid(ci), me->owner, val);
3921 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
3922
3923 return val;
3924}
3925
3926// Push a C method frame for a given cme. This is called when JIT code skipped
3927// pushing a frame but the C method reached a point where a frame is needed.
3928void
3929rb_vm_push_cfunc_frame(const rb_callable_method_entry_t *cme, int recv_idx)
3930{
3931 VM_ASSERT(cme->def->type == VM_METHOD_TYPE_CFUNC);
3932 rb_execution_context_t *ec = GET_EC();
3933 VALUE *sp = ec->cfp->sp;
3934 VALUE recv = *(sp - recv_idx - 1);
3935 VALUE frame_type = VM_FRAME_MAGIC_CFUNC | VM_FRAME_FLAG_CFRAME | VM_ENV_FLAG_LOCAL;
3936 VALUE block_handler = VM_BLOCK_HANDLER_NONE;
3937#if VM_CHECK_MODE > 0
3938 // Clean up the stack canary since we're about to satisfy the "leaf or lazy push" assumption
3939 *(GET_EC()->cfp->sp) = Qfalse;
3940#endif
3941 vm_push_frame(ec, NULL, frame_type, recv, block_handler, (VALUE)cme, 0, ec->cfp->sp, 0, 0);
3942}
3943
3944// If true, cc->call needs to include `CALLER_SETUP_ARG` (i.e. can't be skipped in fastpath)
3945bool
3946rb_splat_or_kwargs_p(const struct rb_callinfo *restrict ci)
3947{
3948 return IS_ARGS_SPLAT(ci) || IS_ARGS_KW_OR_KW_SPLAT(ci);
3949}
3950
3951static VALUE
3952vm_call_cfunc_with_frame(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3953{
3954 int argc = calling->argc;
3955 VALUE *stack_bottom = reg_cfp->sp - argc - 1;
3956 VALUE *argv = &stack_bottom[1];
3957
3958 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3959}
3960
3961static VALUE
3962vm_call_cfunc_other(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
3963{
3964 const struct rb_callinfo *ci = calling->cd->ci;
3965 RB_DEBUG_COUNTER_INC(ccf_cfunc_other);
3966
3967 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
3968 VALUE argv_ary;
3969 if (UNLIKELY(argv_ary = calling->heap_argv)) {
3970 VM_ASSERT(!IS_ARGS_KEYWORD(ci));
3971 int argc = RARRAY_LENINT(argv_ary);
3972 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3973 VALUE *stack_bottom = reg_cfp->sp - 2;
3974
3975 VM_ASSERT(calling->argc == 1);
3976 VM_ASSERT(RB_TYPE_P(argv_ary, T_ARRAY));
3977 VM_ASSERT(RBASIC_CLASS(argv_ary) == 0); // hidden ary
3978
3979 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, argv, stack_bottom);
3980 }
3981 else {
3982 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_with_frame, !rb_splat_or_kwargs_p(ci) && !calling->kw_splat && !(vm_ci_flag(ci) & VM_CALL_FORWARDING));
3983
3984 return vm_call_cfunc_with_frame(ec, reg_cfp, calling);
3985 }
3986}
3987
3988static inline VALUE
3989vm_call_cfunc_array_argv(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int stack_offset, int argc_offset)
3990{
3991 VALUE argv_ary = reg_cfp->sp[-1 - stack_offset];
3992 int argc = RARRAY_LENINT(argv_ary) - argc_offset;
3993
3994 if (UNLIKELY(argc > VM_ARGC_STACK_MAX)) {
3995 return vm_call_cfunc_other(ec, reg_cfp, calling);
3996 }
3997
3998 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
3999 calling->kw_splat = 0;
4000 int i;
4001 VALUE *stack_bottom = reg_cfp->sp - 2 - stack_offset;
4002 VALUE *sp = stack_bottom;
4003 CHECK_VM_STACK_OVERFLOW(reg_cfp, argc);
4004 for(i = 0; i < argc; i++) {
4005 *++sp = argv[i];
4006 }
4007 reg_cfp->sp = sp+1;
4008
4009 return vm_call_cfunc_with_frame_(ec, reg_cfp, calling, argc, stack_bottom+1, stack_bottom);
4010}
4011
4012static inline VALUE
4013vm_call_cfunc_only_splat(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4014{
4015 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat);
4016 VALUE argv_ary = reg_cfp->sp[-1];
4017 int argc = RARRAY_LENINT(argv_ary);
4018 VALUE *argv = (VALUE *)RARRAY_CONST_PTR(argv_ary);
4019 VALUE last_hash;
4020 int argc_offset = 0;
4021
4022 if (UNLIKELY(argc > 0 &&
4023 RB_TYPE_P((last_hash = argv[argc-1]), T_HASH) &&
4024 (((struct RHash *)last_hash)->basic.flags & RHASH_PASS_AS_KEYWORDS))) {
4025 if (!RHASH_EMPTY_P(last_hash)) {
4026 return vm_call_cfunc_other(ec, reg_cfp, calling);
4027 }
4028 argc_offset++;
4029 }
4030 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 0, argc_offset);
4031}
4032
4033static inline VALUE
4034vm_call_cfunc_only_splat_kw(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4035{
4036 RB_DEBUG_COUNTER_INC(ccf_cfunc_only_splat_kw);
4037 VALUE keyword_hash = reg_cfp->sp[-1];
4038
4039 if (keyword_hash == Qnil || (RB_TYPE_P(keyword_hash, T_HASH) && RHASH_EMPTY_P(keyword_hash))) {
4040 return vm_call_cfunc_array_argv(ec, reg_cfp, calling, 1, 0);
4041 }
4042
4043 return vm_call_cfunc_other(ec, reg_cfp, calling);
4044}
4045
4046static VALUE
4047vm_call_cfunc(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4048{
4049 const struct rb_callinfo *ci = calling->cd->ci;
4050 RB_DEBUG_COUNTER_INC(ccf_cfunc);
4051
4052 if (IS_ARGS_SPLAT(ci) && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4053 if (!IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 1) {
4054 // f(*a)
4055 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat, TRUE);
4056 return vm_call_cfunc_only_splat(ec, reg_cfp, calling);
4057 }
4058 if (IS_ARGS_KW_SPLAT(ci) && vm_ci_argc(ci) == 2) {
4059 // f(*a, **kw)
4060 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_only_splat_kw, TRUE);
4061 return vm_call_cfunc_only_splat_kw(ec, reg_cfp, calling);
4062 }
4063 }
4064
4065 CC_SET_FASTPATH(calling->cc, vm_call_cfunc_other, TRUE);
4066 return vm_call_cfunc_other(ec, reg_cfp, calling);
4067}
4068
4069static VALUE
4070vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4071{
4072 const struct rb_callcache *cc = calling->cc;
4073 RB_DEBUG_COUNTER_INC(ccf_ivar);
4074 cfp->sp -= 1;
4075 VALUE ivar = vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE, Qnil);
4076 return ivar;
4077}
4078
4079static VALUE
4080vm_call_attrset_direct(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_callcache *cc, VALUE obj)
4081{
4082 RB_DEBUG_COUNTER_INC(ccf_attrset);
4083 VALUE val = *(cfp->sp - 1);
4084 cfp->sp -= 2;
4085 attr_index_t index;
4086 shape_id_t dest_shape_id;
4087 vm_cc_atomic_shape_and_index(cc, &dest_shape_id, &index);
4088 ID id = vm_cc_cme(cc)->def->body.attr.id;
4089 rb_check_frozen(obj);
4090 VALUE res = vm_setivar(obj, id, val, dest_shape_id, index);
4091 if (UNDEF_P(res)) {
4092 switch (BUILTIN_TYPE(obj)) {
4093 case T_OBJECT:
4094 break;
4095 case T_CLASS:
4096 case T_MODULE:
4097 {
4098 res = vm_setivar_class(obj, id, val, dest_shape_id, index);
4099 if (!UNDEF_P(res)) {
4100 return res;
4101 }
4102 }
4103 break;
4104 default:
4105 {
4106 res = vm_setivar_default(obj, id, val, dest_shape_id, index);
4107 if (!UNDEF_P(res)) {
4108 return res;
4109 }
4110 }
4111 }
4112 res = vm_setivar_slowpath_attr(obj, id, val, cc);
4113 }
4114 return res;
4115}
4116
4117static VALUE
4118vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4119{
4120 return vm_call_attrset_direct(ec, cfp, calling->cc, calling->recv);
4121}
4122
4123static inline VALUE
4124vm_call_bmethod_body(rb_execution_context_t *ec, struct rb_calling_info *calling, const VALUE *argv)
4125{
4126 rb_proc_t *proc;
4127 VALUE val;
4128 const struct rb_callcache *cc = calling->cc;
4129 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4130 VALUE procv = cme->def->body.bmethod.proc;
4131
4132 if (!RB_OBJ_SHAREABLE_P(procv) &&
4133 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4134 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4135 }
4136
4137 /* control block frame */
4138 GetProcPtr(procv, proc);
4139 val = vm_invoke_bmethod(ec, proc, calling->recv, CALLING_ARGC(calling), argv, calling->kw_splat, calling->block_handler, vm_cc_cme(cc));
4140
4141 return val;
4142}
4143
4144static int vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type);
4145
4146static VALUE
4147vm_call_iseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4148{
4149 RB_DEBUG_COUNTER_INC(ccf_iseq_bmethod);
4150
4151 const struct rb_callcache *cc = calling->cc;
4152 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4153 VALUE procv = cme->def->body.bmethod.proc;
4154
4155 if (!RB_OBJ_SHAREABLE_P(procv) &&
4156 cme->def->body.bmethod.defined_ractor_id != rb_ec_ractor_id(ec)) {
4157 rb_raise(rb_eRuntimeError, "defined with an un-shareable Proc in a different Ractor");
4158 }
4159
4160 rb_proc_t *proc;
4161 GetProcPtr(procv, proc);
4162 const struct rb_block *block = &proc->block;
4163
4164 while (vm_block_type(block) == block_type_proc) {
4165 block = vm_proc_block(block->as.proc);
4166 }
4167 VM_ASSERT(vm_block_type(block) == block_type_iseq);
4168
4169 const struct rb_captured_block *captured = &block->as.captured;
4170 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
4171 VALUE * const argv = cfp->sp - calling->argc;
4172 const int arg_size = ISEQ_BODY(iseq)->param.size;
4173
4174 int opt_pc;
4175 if (vm_ci_flag(calling->cd->ci) & VM_CALL_ARGS_SIMPLE) {
4176 opt_pc = vm_callee_setup_block_arg(ec, calling, calling->cd->ci, iseq, argv, arg_setup_method);
4177 }
4178 else {
4179 opt_pc = setup_parameters_complex(ec, iseq, calling, calling->cd->ci, argv, arg_setup_method);
4180 }
4181
4182 cfp->sp = argv - 1; // -1 for the receiver
4183
4184 vm_push_frame(ec, iseq,
4185 VM_FRAME_MAGIC_BLOCK | VM_FRAME_FLAG_BMETHOD | VM_FRAME_FLAG_LAMBDA,
4186 calling->recv,
4187 VM_GUARDED_PREV_EP(captured->ep),
4188 (VALUE)cme,
4189 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
4190 argv + arg_size,
4191 ISEQ_BODY(iseq)->local_table_size - arg_size,
4192 ISEQ_BODY(iseq)->stack_max);
4193
4194 return Qundef;
4195}
4196
4197static VALUE
4198vm_call_noniseq_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4199{
4200 RB_DEBUG_COUNTER_INC(ccf_noniseq_bmethod);
4201
4202 VALUE *argv;
4203 int argc;
4204 CALLER_SETUP_ARG(cfp, calling, calling->cd->ci, ALLOW_HEAP_ARGV);
4205 if (UNLIKELY(calling->heap_argv)) {
4206 argv = RARRAY_PTR(calling->heap_argv);
4207 cfp->sp -= 2;
4208 }
4209 else {
4210 argc = calling->argc;
4211 argv = ALLOCA_N(VALUE, argc);
4212 MEMCPY(argv, cfp->sp - argc, VALUE, argc);
4213 cfp->sp += - argc - 1;
4214 }
4215
4216 return vm_call_bmethod_body(ec, calling, argv);
4217}
4218
4219static VALUE
4220vm_call_bmethod(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4221{
4222 RB_DEBUG_COUNTER_INC(ccf_bmethod);
4223
4224 const struct rb_callcache *cc = calling->cc;
4225 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4226 VALUE procv = cme->def->body.bmethod.proc;
4227 rb_proc_t *proc;
4228 GetProcPtr(procv, proc);
4229 const struct rb_block *block = &proc->block;
4230
4231 while (vm_block_type(block) == block_type_proc) {
4232 block = vm_proc_block(block->as.proc);
4233 }
4234 if (vm_block_type(block) == block_type_iseq) {
4235 CC_SET_FASTPATH(cc, vm_call_iseq_bmethod, TRUE);
4236 return vm_call_iseq_bmethod(ec, cfp, calling);
4237 }
4238
4239 CC_SET_FASTPATH(cc, vm_call_noniseq_bmethod, TRUE);
4240 return vm_call_noniseq_bmethod(ec, cfp, calling);
4241}
4242
4243 VALUE
4244rb_find_defined_class_by_owner(VALUE current_class, VALUE target_owner)
4245{
4246 VALUE klass = current_class;
4247
4248 /* for prepended Module, then start from cover class */
4249 if (RB_TYPE_P(klass, T_ICLASS) && RICLASS_IS_ORIGIN_P(klass) &&
4250 RB_TYPE_P(RBASIC_CLASS(klass), T_CLASS)) {
4251 klass = RBASIC_CLASS(klass);
4252 }
4253
4254 while (RTEST(klass)) {
4255 VALUE owner = RB_TYPE_P(klass, T_ICLASS) ? RBASIC_CLASS(klass) : klass;
4256 if (owner == target_owner) {
4257 return klass;
4258 }
4259 klass = RCLASS_SUPER(klass);
4260 }
4261
4262 return current_class; /* maybe module function */
4263}
4264
4265static const rb_callable_method_entry_t *
4266aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4267{
4268 const rb_method_entry_t *orig_me = me->def->body.alias.original_me;
4269 const rb_callable_method_entry_t *cme;
4270
4271 if (orig_me->defined_class == 0) {
4272 VALUE defined_class = rb_find_defined_class_by_owner(me->defined_class, orig_me->owner);
4273 VM_ASSERT_TYPE(orig_me->owner, T_MODULE);
4274 cme = rb_method_entry_complement_defined_class(orig_me, me->called_id, defined_class);
4275
4276 if (me->def->reference_count == 1) {
4277 RB_OBJ_WRITE(me, &me->def->body.alias.original_me, cme);
4278 }
4279 else {
4280 rb_method_definition_t *def =
4281 rb_method_definition_create(VM_METHOD_TYPE_ALIAS, me->def->original_id);
4282 rb_method_definition_set((rb_method_entry_t *)me, def, (void *)cme);
4283 }
4284 }
4285 else {
4286 cme = (const rb_callable_method_entry_t *)orig_me;
4287 }
4288
4289 VM_ASSERT(callable_method_entry_p(cme));
4290 return cme;
4291}
4292
4293const rb_callable_method_entry_t *
4294rb_aliased_callable_method_entry(const rb_callable_method_entry_t *me)
4295{
4296 return aliased_callable_method_entry(me);
4297}
4298
4299static VALUE
4300vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4301{
4302 calling->cc = &VM_CC_ON_STACK(Qundef,
4303 vm_call_general,
4304 {{0}},
4305 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
4306
4307 return vm_call_method_each_type(ec, cfp, calling);
4308}
4309
4310static enum method_missing_reason
4311ci_missing_reason(const struct rb_callinfo *ci)
4312{
4313 enum method_missing_reason stat = MISSING_NOENTRY;
4314 if (vm_ci_flag(ci) & VM_CALL_VCALL && !(vm_ci_flag(ci) & VM_CALL_FORWARDING)) stat |= MISSING_VCALL;
4315 if (vm_ci_flag(ci) & VM_CALL_FCALL) stat |= MISSING_FCALL;
4316 if (vm_ci_flag(ci) & VM_CALL_SUPER) stat |= MISSING_SUPER;
4317 return stat;
4318}
4319
4320static VALUE vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling);
4321
4322static VALUE
4323vm_call_symbol(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4324 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE symbol, int flags)
4325{
4326 ASSUME(calling->argc >= 0);
4327
4328 enum method_missing_reason missing_reason = MISSING_NOENTRY;
4329 int argc = calling->argc;
4330 VALUE recv = calling->recv;
4331 VALUE klass = CLASS_OF(recv);
4332 ID mid = rb_check_id(&symbol);
4333 flags |= VM_CALL_OPT_SEND;
4334
4335 if (UNLIKELY(! mid)) {
4336 mid = idMethodMissing;
4337 missing_reason = ci_missing_reason(ci);
4338 ec->method_missing_reason = missing_reason;
4339
4340 VALUE argv_ary;
4341 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4342 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4343 rb_ary_unshift(argv_ary, symbol);
4344
4345 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4346 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4347 VALUE exc = rb_make_no_method_exception(
4348 rb_eNoMethodError, 0, recv, RARRAY_LENINT(argv_ary), RARRAY_CONST_PTR(argv_ary), priv);
4349
4350 rb_exc_raise(exc);
4351 }
4352 rb_ary_unshift(argv_ary, rb_str_intern(symbol));
4353 }
4354 else {
4355 /* E.g. when argc == 2
4356 *
4357 * | | | | TOPN
4358 * | | +------+
4359 * | | +---> | arg1 | 0
4360 * +------+ | +------+
4361 * | arg1 | -+ +-> | arg0 | 1
4362 * +------+ | +------+
4363 * | arg0 | ---+ | sym | 2
4364 * +------+ +------+
4365 * | recv | | recv | 3
4366 * --+------+--------+------+------
4367 */
4368 int i = argc;
4369 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4370 INC_SP(1);
4371 MEMMOVE(&TOPN(i - 1), &TOPN(i), VALUE, i);
4372 argc = ++calling->argc;
4373
4374 if (rb_method_basic_definition_p(klass, idMethodMissing)) {
4375 /* Inadvertent symbol creation shall be forbidden, see [Feature #5112] */
4376 TOPN(i) = symbol;
4377 int priv = vm_ci_flag(ci) & (VM_CALL_FCALL | VM_CALL_VCALL);
4378 const VALUE *argv = STACK_ADDR_FROM_TOP(argc);
4379 VALUE exc = rb_make_no_method_exception(
4380 rb_eNoMethodError, 0, recv, argc, argv, priv);
4381
4382 rb_exc_raise(exc);
4383 }
4384 else {
4385 TOPN(i) = rb_str_intern(symbol);
4386 }
4387 }
4388 }
4389
4390 struct rb_forwarding_call_data new_fcd = {
4391 .cd = {
4392 .ci = &VM_CI_ON_STACK(mid, flags, argc, vm_ci_kwarg(ci)),
4393 .cc = NULL,
4394 },
4395 .caller_ci = NULL,
4396 };
4397
4398 if (!(vm_ci_flag(ci) & VM_CALL_FORWARDING)) {
4399 calling->cd = &new_fcd.cd;
4400 }
4401 else {
4402 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4403 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4404 new_fcd.caller_ci = caller_ci;
4405 calling->cd = (struct rb_call_data *)&new_fcd;
4406 }
4407 calling->cc = &VM_CC_ON_STACK(klass,
4408 vm_call_general,
4409 { .method_missing_reason = missing_reason },
4410 rb_callable_method_entry_with_refinements(klass, mid, NULL));
4411
4412 if (flags & VM_CALL_FCALL) {
4413 return vm_call_method(ec, reg_cfp, calling);
4414 }
4415
4416 const struct rb_callcache *cc = calling->cc;
4417 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
4418
4419 if (vm_cc_cme(cc) != NULL) {
4420 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
4421 case METHOD_VISI_PUBLIC: /* likely */
4422 return vm_call_method_each_type(ec, reg_cfp, calling);
4423 case METHOD_VISI_PRIVATE:
4424 vm_cc_method_missing_reason_set(cc, MISSING_PRIVATE);
4425 break;
4426 case METHOD_VISI_PROTECTED:
4427 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
4428 break;
4429 default:
4430 VM_UNREACHABLE(vm_call_method);
4431 }
4432 return vm_call_method_missing(ec, reg_cfp, calling);
4433 }
4434
4435 return vm_call_method_nome(ec, reg_cfp, calling);
4436}
4437
4438static VALUE
4439vm_call_opt_send0(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, int flags)
4440{
4441 const struct rb_callinfo *ci = calling->cd->ci;
4442 int i;
4443 VALUE sym;
4444
4445 i = calling->argc - 1;
4446
4447 if (calling->argc == 0) {
4448 rb_raise(rb_eArgError, "no method name given");
4449 }
4450
4451 sym = TOPN(i);
4452 /* E.g. when i == 2
4453 *
4454 * | | | | TOPN
4455 * +------+ | |
4456 * | arg1 | ---+ | | 0
4457 * +------+ | +------+
4458 * | arg0 | -+ +-> | arg1 | 1
4459 * +------+ | +------+
4460 * | sym | +---> | arg0 | 2
4461 * +------+ +------+
4462 * | recv | | recv | 3
4463 * --+------+--------+------+------
4464 */
4465 /* shift arguments */
4466 if (i > 0) {
4467 MEMMOVE(&TOPN(i), &TOPN(i-1), VALUE, i);
4468 }
4469 calling->argc -= 1;
4470 DEC_SP(1);
4471
4472 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4473}
4474
4475static VALUE
4476vm_call_opt_send_complex(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4477{
4478 RB_DEBUG_COUNTER_INC(ccf_opt_send_complex);
4479 const struct rb_callinfo *ci = calling->cd->ci;
4480 int flags = VM_CALL_FCALL;
4481 VALUE sym;
4482
4483 VALUE argv_ary;
4484 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
4485 if (UNLIKELY(argv_ary = calling->heap_argv)) {
4486 sym = rb_ary_shift(argv_ary);
4487 flags |= VM_CALL_ARGS_SPLAT;
4488 if (calling->kw_splat) {
4489 VALUE last_hash = rb_ary_last(0, NULL, argv_ary);
4490 ((struct RHash *)last_hash)->basic.flags |= RHASH_PASS_AS_KEYWORDS;
4491 calling->kw_splat = 0;
4492 }
4493 return vm_call_symbol(ec, reg_cfp, calling, ci, sym, flags);
4494 }
4495
4496 if (calling->kw_splat) flags |= VM_CALL_KW_SPLAT;
4497 return vm_call_opt_send0(ec, reg_cfp, calling, flags);
4498}
4499
4500static VALUE
4501vm_call_opt_send_simple(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4502{
4503 RB_DEBUG_COUNTER_INC(ccf_opt_send_simple);
4504 return vm_call_opt_send0(ec, reg_cfp, calling, vm_ci_flag(calling->cd->ci) | VM_CALL_FCALL);
4505}
4506
4507static VALUE
4508vm_call_opt_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4509{
4510 RB_DEBUG_COUNTER_INC(ccf_opt_send);
4511
4512 const struct rb_callinfo *ci = calling->cd->ci;
4513 int flags = vm_ci_flag(ci);
4514
4515 if (UNLIKELY((flags & VM_CALL_FORWARDING) || (!(flags & VM_CALL_ARGS_SIMPLE) &&
4516 ((calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
4517 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
4518 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc)))))) {
4519 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_complex, TRUE);
4520 return vm_call_opt_send_complex(ec, reg_cfp, calling);
4521 }
4522
4523 CC_SET_FASTPATH(calling->cc, vm_call_opt_send_simple, TRUE);
4524 return vm_call_opt_send_simple(ec, reg_cfp, calling);
4525}
4526
4527static VALUE
4528vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling,
4529 const struct rb_callinfo *orig_ci, enum method_missing_reason reason)
4530{
4531 RB_DEBUG_COUNTER_INC(ccf_method_missing);
4532
4533 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4534 unsigned int argc, flag;
4535
4536 flag = VM_CALL_FCALL | VM_CALL_OPT_SEND | vm_ci_flag(orig_ci);
4537 argc = ++calling->argc;
4538
4539 /* shift arguments: m(a, b, c) #=> method_missing(:m, a, b, c) */
4540 CHECK_VM_STACK_OVERFLOW(reg_cfp, 1);
4541 vm_check_canary(ec, reg_cfp->sp);
4542 if (argc > 1) {
4543 MEMMOVE(argv+1, argv, VALUE, argc-1);
4544 }
4545 argv[0] = ID2SYM(vm_ci_mid(orig_ci));
4546 INC_SP(1);
4547
4548 ec->method_missing_reason = reason;
4549
4550 struct rb_forwarding_call_data new_fcd = {
4551 .cd = {
4552 .ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci)),
4553 .cc = NULL,
4554 },
4555 .caller_ci = NULL,
4556 };
4557
4558 if (!(flag & VM_CALL_FORWARDING)) {
4559 calling->cd = &new_fcd.cd;
4560 }
4561 else {
4562 const struct rb_callinfo *caller_ci = ((struct rb_forwarding_call_data *)calling->cd)->caller_ci;
4563 VM_ASSERT((vm_ci_argc(caller_ci), 1));
4564 new_fcd.caller_ci = caller_ci;
4565 calling->cd = (struct rb_call_data *)&new_fcd;
4566 }
4567
4568 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }},
4569 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
4570 return vm_call_method(ec, reg_cfp, calling);
4571}
4572
4573static VALUE
4574vm_call_method_missing(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4575{
4576 return vm_call_method_missing_body(ec, reg_cfp, calling, calling->cd->ci, vm_cc_cmethod_missing_reason(calling->cc));
4577}
4578
4579static const rb_callable_method_entry_t *refined_method_callable_without_refinement(const rb_callable_method_entry_t *me);
4580static VALUE
4581vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling, VALUE klass)
4582{
4583 klass = RCLASS_SUPER(klass);
4584
4585 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, vm_ci_mid(calling->cd->ci)) : NULL;
4586 if (cme == NULL) {
4587 return vm_call_method_nome(ec, cfp, calling);
4588 }
4589 if (cme->def->type == VM_METHOD_TYPE_REFINED &&
4590 cme->def->body.refined.orig_me) {
4591 cme = refined_method_callable_without_refinement(cme);
4592 }
4593
4594 calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, cme);
4595
4596 return vm_call_method_each_type(ec, cfp, calling);
4597}
4598
4599static inline VALUE
4600find_refinement(VALUE refinements, VALUE klass)
4601{
4602 if (NIL_P(refinements)) {
4603 return Qnil;
4604 }
4605 return rb_hash_lookup(refinements, klass);
4606}
4607
4608PUREFUNC(static rb_control_frame_t * current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp));
4609static rb_control_frame_t *
4610current_method_entry(const rb_execution_context_t *ec, rb_control_frame_t *cfp)
4611{
4612 rb_control_frame_t *top_cfp = cfp;
4613
4614 if (cfp->iseq && ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_BLOCK) {
4615 const rb_iseq_t *local_iseq = ISEQ_BODY(cfp->iseq)->local_iseq;
4616
4617 do {
4618 cfp = RUBY_VM_PREVIOUS_CONTROL_FRAME(cfp);
4619 if (RUBY_VM_CONTROL_FRAME_STACK_OVERFLOW_P(ec, cfp)) {
4620 /* TODO: orphan block */
4621 return top_cfp;
4622 }
4623 } while (cfp->iseq != local_iseq);
4624 }
4625 return cfp;
4626}
4627
4628static const rb_callable_method_entry_t *
4629refined_method_callable_without_refinement(const rb_callable_method_entry_t *me)
4630{
4631 const rb_method_entry_t *orig_me = me->def->body.refined.orig_me;
4632 const rb_callable_method_entry_t *cme;
4633
4634 if (orig_me->defined_class == 0) {
4635 cme = NULL;
4636 rb_notimplement();
4637 }
4638 else {
4639 cme = (const rb_callable_method_entry_t *)orig_me;
4640 }
4641
4642 VM_ASSERT(callable_method_entry_p(cme));
4643
4644 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
4645 cme = NULL;
4646 }
4647
4648 return cme;
4649}
4650
4651static const rb_callable_method_entry_t *
4652search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4653{
4654 ID mid = vm_ci_mid(calling->cd->ci);
4655 const rb_cref_t *cref = vm_get_cref(cfp->ep);
4656 const struct rb_callcache * const cc = calling->cc;
4657 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4658
4659 for (; cref; cref = CREF_NEXT(cref)) {
4660 const VALUE refinement = find_refinement(CREF_REFINEMENTS(cref), vm_cc_cme(cc)->owner);
4661 if (NIL_P(refinement)) continue;
4662
4663 const rb_callable_method_entry_t *const ref_me =
4664 rb_callable_method_entry(refinement, mid);
4665
4666 if (ref_me) {
4667 if (vm_cc_call(cc) == vm_call_super_method) {
4668 const rb_control_frame_t *top_cfp = current_method_entry(ec, cfp);
4669 const rb_callable_method_entry_t *top_me = rb_vm_frame_method_entry(top_cfp);
4670 if (top_me && rb_method_definition_eq(ref_me->def, top_me->def)) {
4671 continue;
4672 }
4673 }
4674
4675 if (cme->def->type != VM_METHOD_TYPE_REFINED ||
4676 cme->def != ref_me->def) {
4677 cme = ref_me;
4678 }
4679 if (ref_me->def->type != VM_METHOD_TYPE_REFINED) {
4680 return cme;
4681 }
4682 }
4683 else {
4684 return NULL;
4685 }
4686 }
4687
4688 if (vm_cc_cme(cc)->def->body.refined.orig_me) {
4689 return refined_method_callable_without_refinement(vm_cc_cme(cc));
4690 }
4691 else {
4692 VALUE klass = RCLASS_SUPER(vm_cc_cme(cc)->defined_class);
4693 const rb_callable_method_entry_t *cme = klass ? rb_callable_method_entry(klass, mid) : NULL;
4694 return cme;
4695 }
4696}
4697
4698static VALUE
4699vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4700{
4701 const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling);
4702
4703 if (ref_cme) {
4704 if (calling->cd->cc) {
4705 const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement);
4706 RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc);
4707 return vm_call_method(ec, cfp, calling);
4708 }
4709 else {
4710 struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme);
4711 calling->cc= ref_cc;
4712 return vm_call_method(ec, cfp, calling);
4713 }
4714 }
4715 else {
4716 return vm_call_method_nome(ec, cfp, calling);
4717 }
4718}
4719
4720static inline VALUE vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling, const struct rb_callinfo *ci, bool is_lambda, VALUE block_handler);
4721
4722NOINLINE(static VALUE
4723 vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4724 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler));
4725
4726static VALUE
4727vm_invoke_block_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
4728 struct rb_calling_info *calling, const struct rb_callinfo *ci, VALUE block_handler)
4729{
4730 int argc = calling->argc;
4731
4732 /* remove self */
4733 if (argc > 0) MEMMOVE(&TOPN(argc), &TOPN(argc-1), VALUE, argc);
4734 DEC_SP(1);
4735
4736 return vm_invoke_block(ec, reg_cfp, calling, ci, false, block_handler);
4737}
4738
4739static VALUE
4740vm_call_opt_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4741{
4742 RB_DEBUG_COUNTER_INC(ccf_opt_call);
4743
4744 const struct rb_callinfo *ci = calling->cd->ci;
4745 VALUE procval = calling->recv;
4746 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, VM_BH_FROM_PROC(procval));
4747}
4748
4749static VALUE
4750vm_call_opt_block_call(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4751{
4752 RB_DEBUG_COUNTER_INC(ccf_opt_block_call);
4753
4754 VALUE block_handler = VM_ENV_BLOCK_HANDLER(VM_CF_LEP(reg_cfp));
4755 const struct rb_callinfo *ci = calling->cd->ci;
4756
4757 if (BASIC_OP_UNREDEFINED_P(BOP_CALL, PROC_REDEFINED_OP_FLAG)) {
4758 return vm_invoke_block_opt_call(ec, reg_cfp, calling, ci, block_handler);
4759 }
4760 else {
4761 calling->recv = rb_vm_bh_to_procval(ec, block_handler);
4762 calling->cc = rb_vm_search_method_slowpath(ci, CLASS_OF(calling->recv));
4763 return vm_call_general(ec, reg_cfp, calling);
4764 }
4765}
4766
4767static VALUE
4768vm_call_opt_struct_aref0(rb_execution_context_t *ec, struct rb_calling_info *calling)
4769{
4770 VALUE recv = calling->recv;
4771
4772 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4773 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4774 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_AREF);
4775
4776 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4777 return internal_RSTRUCT_GET(recv, off);
4778}
4779
4780static VALUE
4781vm_call_opt_struct_aref(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4782{
4783 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aref);
4784
4785 VALUE ret = vm_call_opt_struct_aref0(ec, calling);
4786 reg_cfp->sp -= 1;
4787 return ret;
4788}
4789
4790static VALUE
4791vm_call_opt_struct_aset0(rb_execution_context_t *ec, struct rb_calling_info *calling, VALUE val)
4792{
4793 VALUE recv = calling->recv;
4794
4795 VM_ASSERT(RB_TYPE_P(recv, T_STRUCT));
4796 VM_ASSERT(vm_cc_cme(calling->cc)->def->type == VM_METHOD_TYPE_OPTIMIZED);
4797 VM_ASSERT(vm_cc_cme(calling->cc)->def->body.optimized.type == OPTIMIZED_METHOD_TYPE_STRUCT_ASET);
4798
4799 rb_check_frozen(recv);
4800
4801 const unsigned int off = vm_cc_cme(calling->cc)->def->body.optimized.index;
4802 internal_RSTRUCT_SET(recv, off, val);
4803
4804 return val;
4805}
4806
4807static VALUE
4808vm_call_opt_struct_aset(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
4809{
4810 RB_DEBUG_COUNTER_INC(ccf_opt_struct_aset);
4811
4812 VALUE ret = vm_call_opt_struct_aset0(ec, calling, *(reg_cfp->sp - 1));
4813 reg_cfp->sp -= 2;
4814 return ret;
4815}
4816
4817NOINLINE(static VALUE vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4818 const struct rb_callinfo *ci, const struct rb_callcache *cc));
4819
4820#define VM_CALL_METHOD_ATTR(var, func, nohook) \
4821 if (UNLIKELY(ruby_vm_c_events_enabled > 0)) { \
4822 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_CALL, calling->recv, vm_cc_cme(cc)->def->original_id, \
4823 vm_ci_mid(ci), vm_cc_cme(cc)->owner, Qundef); \
4824 var = func; \
4825 EXEC_EVENT_HOOK(ec, RUBY_EVENT_C_RETURN, calling->recv, vm_cc_cme(cc)->def->original_id, \
4826 vm_ci_mid(ci), vm_cc_cme(cc)->owner, (var)); \
4827 } \
4828 else { \
4829 nohook; \
4830 var = func; \
4831 }
4832
4833static VALUE
4834vm_call_optimized(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling,
4835 const struct rb_callinfo *ci, const struct rb_callcache *cc)
4836{
4837 switch (vm_cc_cme(cc)->def->body.optimized.type) {
4838 case OPTIMIZED_METHOD_TYPE_SEND:
4839 CC_SET_FASTPATH(cc, vm_call_opt_send, TRUE);
4840 return vm_call_opt_send(ec, cfp, calling);
4841 case OPTIMIZED_METHOD_TYPE_CALL:
4842 CC_SET_FASTPATH(cc, vm_call_opt_call, TRUE);
4843 return vm_call_opt_call(ec, cfp, calling);
4844 case OPTIMIZED_METHOD_TYPE_BLOCK_CALL:
4845 CC_SET_FASTPATH(cc, vm_call_opt_block_call, TRUE);
4846 return vm_call_opt_block_call(ec, cfp, calling);
4847 case OPTIMIZED_METHOD_TYPE_STRUCT_AREF: {
4848 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4849 rb_check_arity(calling->argc, 0, 0);
4850
4851 VALUE v;
4852 VM_CALL_METHOD_ATTR(v,
4853 vm_call_opt_struct_aref(ec, cfp, calling),
4854 set_vm_cc_ivar(cc); \
4855 CC_SET_FASTPATH(cc, vm_call_opt_struct_aref, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4856 return v;
4857 }
4858 case OPTIMIZED_METHOD_TYPE_STRUCT_ASET: {
4859 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4860 rb_check_arity(calling->argc, 1, 1);
4861
4862 VALUE v;
4863 VM_CALL_METHOD_ATTR(v,
4864 vm_call_opt_struct_aset(ec, cfp, calling),
4865 set_vm_cc_ivar(cc); \
4866 CC_SET_FASTPATH(cc, vm_call_opt_struct_aset, (vm_ci_flag(ci) & VM_CALL_ARGS_SIMPLE)))
4867 return v;
4868 }
4869 default:
4870 rb_bug("vm_call_method: unsupported optimized method type (%d)", vm_cc_cme(cc)->def->body.optimized.type);
4871 }
4872}
4873
4874static VALUE
4875vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4876{
4877 const struct rb_callinfo *ci = calling->cd->ci;
4878 const struct rb_callcache *cc = calling->cc;
4879 const rb_callable_method_entry_t *cme = vm_cc_cme(cc);
4880 VALUE v;
4881
4882 VM_ASSERT(! METHOD_ENTRY_INVALIDATED(cme));
4883
4884 switch (cme->def->type) {
4885 case VM_METHOD_TYPE_ISEQ:
4886 if (ISEQ_BODY(def_iseq_ptr(cme->def))->param.flags.forwardable) {
4887 CC_SET_FASTPATH(cc, vm_call_iseq_fwd_setup, TRUE);
4888 return vm_call_iseq_fwd_setup(ec, cfp, calling);
4889 }
4890 else {
4891 CC_SET_FASTPATH(cc, vm_call_iseq_setup, TRUE);
4892 return vm_call_iseq_setup(ec, cfp, calling);
4893 }
4894
4895 case VM_METHOD_TYPE_NOTIMPLEMENTED:
4896 case VM_METHOD_TYPE_CFUNC:
4897 CC_SET_FASTPATH(cc, vm_call_cfunc, TRUE);
4898 return vm_call_cfunc(ec, cfp, calling);
4899
4900 case VM_METHOD_TYPE_ATTRSET:
4901 CALLER_SETUP_ARG(cfp, calling, ci, 1);
4902
4903 rb_check_arity(calling->argc, 1, 1);
4904
4905 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG | VM_CALL_FORWARDING);
4906
4907 if (vm_cc_markable(cc)) {
4908 vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID);
4909 VM_CALL_METHOD_ATTR(v,
4910 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4911 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4912 }
4913 else {
4914 cc = &((struct rb_callcache) {
4915 .flags = T_IMEMO |
4916 (imemo_callcache << FL_USHIFT) |
4917 VM_CALLCACHE_UNMARKABLE |
4918 VM_CALLCACHE_ON_STACK,
4919 .klass = cc->klass,
4920 .cme_ = cc->cme_,
4921 .call_ = cc->call_,
4922 .aux_ = {
4923 .attr = {
4924 .value = vm_pack_shape_and_index(INVALID_SHAPE_ID, ATTR_INDEX_NOT_SET),
4925 }
4926 },
4927 });
4928
4929 VM_CALL_METHOD_ATTR(v,
4930 vm_call_attrset_direct(ec, cfp, cc, calling->recv),
4931 CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
4932 }
4933 return v;
4934
4935 case VM_METHOD_TYPE_IVAR:
4936 CALLER_SETUP_ARG(cfp, calling, ci, 0);
4937 rb_check_arity(calling->argc, 0, 0);
4938 vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID);
4939 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_FORWARDING);
4940 VM_CALL_METHOD_ATTR(v,
4941 vm_call_ivar(ec, cfp, calling),
4942 CC_SET_FASTPATH(cc, vm_call_ivar, !(vm_ci_flag(ci) & ivar_mask)));
4943 return v;
4944
4945 case VM_METHOD_TYPE_MISSING:
4946 vm_cc_method_missing_reason_set(cc, 0);
4947 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
4948 return vm_call_method_missing(ec, cfp, calling);
4949
4950 case VM_METHOD_TYPE_BMETHOD:
4951 CC_SET_FASTPATH(cc, vm_call_bmethod, TRUE);
4952 return vm_call_bmethod(ec, cfp, calling);
4953
4954 case VM_METHOD_TYPE_ALIAS:
4955 CC_SET_FASTPATH(cc, vm_call_alias, TRUE);
4956 return vm_call_alias(ec, cfp, calling);
4957
4958 case VM_METHOD_TYPE_OPTIMIZED:
4959 return vm_call_optimized(ec, cfp, calling, ci, cc);
4960
4961 case VM_METHOD_TYPE_UNDEF:
4962 break;
4963
4964 case VM_METHOD_TYPE_ZSUPER:
4965 return vm_call_zsuper(ec, cfp, calling, RCLASS_ORIGIN(vm_cc_cme(cc)->defined_class));
4966
4967 case VM_METHOD_TYPE_REFINED:
4968 // CC_SET_FASTPATH(cc, vm_call_refined, TRUE);
4969 // should not set FASTPATH since vm_call_refined assumes cc->call is vm_call_super_method on invokesuper.
4970 return vm_call_refined(ec, cfp, calling);
4971 }
4972
4973 rb_bug("vm_call_method: unsupported method type (%d)", vm_cc_cme(cc)->def->type);
4974}
4975
4976NORETURN(static void vm_raise_method_missing(rb_execution_context_t *ec, int argc, const VALUE *argv, VALUE obj, int call_status));
4977
4978static VALUE
4979vm_call_method_nome(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
4980{
4981 /* method missing */
4982 const struct rb_callinfo *ci = calling->cd->ci;
4983 const int stat = ci_missing_reason(ci);
4984
4985 if (vm_ci_mid(ci) == idMethodMissing) {
4986 if (UNLIKELY(calling->heap_argv)) {
4987 vm_raise_method_missing(ec, RARRAY_LENINT(calling->heap_argv), RARRAY_CONST_PTR(calling->heap_argv), calling->recv, stat);
4988 }
4989 else {
4990 rb_control_frame_t *reg_cfp = cfp;
4991 VALUE *argv = STACK_ADDR_FROM_TOP(calling->argc);
4992 vm_raise_method_missing(ec, calling->argc, argv, calling->recv, stat);
4993 }
4994 }
4995 else {
4996 return vm_call_method_missing_body(ec, cfp, calling, ci, stat);
4997 }
4998}
4999
5000/* Protected method calls and super invocations need to check that the receiver
5001 * (self for super) inherits the module on which the method is defined.
5002 * In the case of refinements, it should consider the original class not the
5003 * refinement.
5004 */
5005static VALUE
5006vm_defined_class_for_protected_call(const rb_callable_method_entry_t *me)
5007{
5008 VALUE defined_class = me->defined_class;
5009 VALUE refined_class = RCLASS_REFINED_CLASS(defined_class);
5010 return NIL_P(refined_class) ? defined_class : refined_class;
5011}
5012
5013static inline VALUE
5014vm_call_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
5015{
5016 const struct rb_callinfo *ci = calling->cd->ci;
5017 const struct rb_callcache *cc = calling->cc;
5018
5019 VM_ASSERT(callable_method_entry_p(vm_cc_cme(cc)));
5020
5021 if (vm_cc_cme(cc) != NULL) {
5022 switch (METHOD_ENTRY_VISI(vm_cc_cme(cc))) {
5023 case METHOD_VISI_PUBLIC: /* likely */
5024 return vm_call_method_each_type(ec, cfp, calling);
5025
5026 case METHOD_VISI_PRIVATE:
5027 if (!(vm_ci_flag(ci) & VM_CALL_FCALL)) {
5028 enum method_missing_reason stat = MISSING_PRIVATE;
5029 if (vm_ci_flag(ci) & VM_CALL_VCALL) stat |= MISSING_VCALL;
5030
5031 vm_cc_method_missing_reason_set(cc, stat);
5032 CC_SET_FASTPATH(cc, vm_call_method_missing, TRUE);
5033 return vm_call_method_missing(ec, cfp, calling);
5034 }
5035 return vm_call_method_each_type(ec, cfp, calling);
5036
5037 case METHOD_VISI_PROTECTED:
5038 if (!(vm_ci_flag(ci) & (VM_CALL_OPT_SEND | VM_CALL_FCALL))) {
5039 VALUE defined_class = vm_defined_class_for_protected_call(vm_cc_cme(cc));
5040 if (!rb_obj_is_kind_of(cfp->self, defined_class)) {
5041 vm_cc_method_missing_reason_set(cc, MISSING_PROTECTED);
5042 return vm_call_method_missing(ec, cfp, calling);
5043 }
5044 else {
5045 /* caching method info to dummy cc */
5046 VM_ASSERT(vm_cc_cme(cc) != NULL);
5047 struct rb_callcache cc_on_stack = *cc;
5048 FL_SET_RAW((VALUE)&cc_on_stack, VM_CALLCACHE_UNMARKABLE);
5049 calling->cc = &cc_on_stack;
5050 return vm_call_method_each_type(ec, cfp, calling);
5051 }
5052 }
5053 return vm_call_method_each_type(ec, cfp, calling);
5054
5055 default:
5056 rb_bug("unreachable");
5057 }
5058 }
5059 else {
5060 return vm_call_method_nome(ec, cfp, calling);
5061 }
5062}
5063
5064static VALUE
5065vm_call_general(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5066{
5067 RB_DEBUG_COUNTER_INC(ccf_general);
5068 return vm_call_method(ec, reg_cfp, calling);
5069}
5070
5071void
5072rb_vm_cc_general(const struct rb_callcache *cc)
5073{
5074 VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache));
5075 VM_ASSERT(cc != vm_cc_empty());
5076
5077 *(vm_call_handler *)&cc->call_ = vm_call_general;
5078}
5079
5080static VALUE
5081vm_call_super_method(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, struct rb_calling_info *calling)
5082{
5083 RB_DEBUG_COUNTER_INC(ccf_super_method);
5084
5085 // This line is introduced to make different from `vm_call_general` because some compilers (VC we found)
5086 // can merge the function and the address of the function becomes same.
5087 // The address of `vm_call_super_method` is used in `search_refined_method`, so it should be different.
5088 if (ec == NULL) rb_bug("unreachable");
5089
5090 /* this check is required to distinguish with other functions. */
5091 VM_ASSERT(vm_cc_call(calling->cc) == vm_call_super_method);
5092 return vm_call_method(ec, reg_cfp, calling);
5093}
5094
5095/* super */
5096
5097static inline VALUE
5098vm_search_normal_superclass(VALUE klass)
5099{
5100 if (BUILTIN_TYPE(klass) == T_ICLASS &&
5101 RB_TYPE_P(RBASIC(klass)->klass, T_MODULE) &&
5102 FL_TEST_RAW(RBASIC(klass)->klass, RMODULE_IS_REFINEMENT)) {
5103 klass = RBASIC(klass)->klass;
5104 }
5105 klass = RCLASS_ORIGIN(klass);
5106 return RCLASS_SUPER(klass);
5107}
5108
5109NORETURN(static void vm_super_outside(void));
5110
5111static void
5112vm_super_outside(void)
5113{
5114 rb_raise(rb_eNoMethodError, "super called outside of method");
5115}
5116
5117static const struct rb_callcache *
5118empty_cc_for_super(void)
5119{
5120 return &vm_empty_cc_for_super;
5121}
5122
5123static const struct rb_callcache *
5124vm_search_super_method(const rb_control_frame_t *reg_cfp, struct rb_call_data *cd, VALUE recv)
5125{
5126 VALUE current_defined_class;
5127 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
5128
5129 if (!me) {
5130 vm_super_outside();
5131 }
5132
5133 current_defined_class = vm_defined_class_for_protected_call(me);
5134
5135 if (BUILTIN_TYPE(current_defined_class) != T_MODULE &&
5136 reg_cfp->iseq != method_entry_iseqptr(me) &&
5137 !rb_obj_is_kind_of(recv, current_defined_class)) {
5138 VALUE m = RB_TYPE_P(current_defined_class, T_ICLASS) ?
5139 RCLASS_INCLUDER(current_defined_class) : current_defined_class;
5140
5141 if (m) { /* not bound UnboundMethod */
5142 rb_raise(rb_eTypeError,
5143 "self has wrong type to call super in this context: "
5144 "%"PRIsVALUE" (expected %"PRIsVALUE")",
5145 rb_obj_class(recv), m);
5146 }
5147 }
5148
5149 if (me->def->type == VM_METHOD_TYPE_BMETHOD && (vm_ci_flag(cd->ci) & VM_CALL_ZSUPER)) {
5150 rb_raise(rb_eRuntimeError,
5151 "implicit argument passing of super from method defined"
5152 " by define_method() is not supported."
5153 " Specify all arguments explicitly.");
5154 }
5155
5156 ID mid = me->def->original_id;
5157
5158 if (!vm_ci_markable(cd->ci)) {
5159 VM_FORCE_WRITE((const VALUE *)&cd->ci->mid, (VALUE)mid);
5160 }
5161 else {
5162 // update iseq. really? (TODO)
5163 cd->ci = vm_ci_new_runtime(mid,
5164 vm_ci_flag(cd->ci),
5165 vm_ci_argc(cd->ci),
5166 vm_ci_kwarg(cd->ci));
5167
5168 RB_OBJ_WRITTEN(reg_cfp->iseq, Qundef, cd->ci);
5169 }
5170
5171 const struct rb_callcache *cc;
5172
5173 VALUE klass = vm_search_normal_superclass(me->defined_class);
5174
5175 if (!klass) {
5176 /* bound instance method of module */
5177 cc = vm_cc_new(Qundef, NULL, vm_call_method_missing, cc_type_super);
5178 RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
5179 }
5180 else {
5181 cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, klass);
5182 const rb_callable_method_entry_t *cached_cme = vm_cc_cme(cc);
5183
5184 // define_method can cache for different method id
5185 if (cached_cme == NULL) {
5186 // empty_cc_for_super is not markable object
5187 cd->cc = empty_cc_for_super();
5188 }
5189 else if (cached_cme->called_id != mid) {
5190 const rb_callable_method_entry_t *cme = rb_callable_method_entry(klass, mid);
5191 if (cme) {
5192 cc = vm_cc_new(klass, cme, vm_call_super_method, cc_type_super);
5193 RB_OBJ_WRITE(reg_cfp->iseq, &cd->cc, cc);
5194 }
5195 else {
5196 cd->cc = cc = empty_cc_for_super();
5197 }
5198 }
5199 else {
5200 switch (cached_cme->def->type) {
5201 // vm_call_refined (search_refined_method) assumes cc->call is vm_call_super_method on invokesuper
5202 case VM_METHOD_TYPE_REFINED:
5203 // cc->klass is superclass of receiver class. Checking cc->klass is not enough to invalidate IVC for the receiver class.
5204 case VM_METHOD_TYPE_ATTRSET:
5205 case VM_METHOD_TYPE_IVAR:
5206 vm_cc_call_set(cc, vm_call_super_method); // invalidate fastpath
5207 break;
5208 default:
5209 break; // use fastpath
5210 }
5211 }
5212 }
5213
5214 VM_ASSERT((vm_cc_cme(cc), true));
5215
5216 return cc;
5217}
5218
5219/* yield */
5220
5221static inline int
5222block_proc_is_lambda(const VALUE procval)
5223{
5224 rb_proc_t *proc;
5225
5226 if (procval) {
5227 GetProcPtr(procval, proc);
5228 return proc->is_lambda;
5229 }
5230 else {
5231 return 0;
5232 }
5233}
5234
5235static VALUE
5236vm_yield_with_cfunc(rb_execution_context_t *ec,
5237 const struct rb_captured_block *captured,
5238 VALUE self, int argc, const VALUE *argv, int kw_splat, VALUE block_handler,
5239 const rb_callable_method_entry_t *me)
5240{
5241 int is_lambda = FALSE; /* TODO */
5242 VALUE val, arg, blockarg;
5243 int frame_flag;
5244 const struct vm_ifunc *ifunc = captured->code.ifunc;
5245
5246 if (is_lambda) {
5247 arg = rb_ary_new4(argc, argv);
5248 }
5249 else if (argc == 0) {
5250 arg = Qnil;
5251 }
5252 else {
5253 arg = argv[0];
5254 }
5255
5256 blockarg = rb_vm_bh_to_procval(ec, block_handler);
5257
5258 frame_flag = VM_FRAME_MAGIC_IFUNC | VM_FRAME_FLAG_CFRAME | (me ? VM_FRAME_FLAG_BMETHOD : 0);
5259 if (kw_splat) {
5260 frame_flag |= VM_FRAME_FLAG_CFRAME_KW;
5261 }
5262
5263 vm_push_frame(ec, (const rb_iseq_t *)captured->code.ifunc,
5264 frame_flag,
5265 self,
5266 VM_GUARDED_PREV_EP(captured->ep),
5267 (VALUE)me,
5268 0, ec->cfp->sp, 0, 0);
5269 val = (*ifunc->func)(arg, (VALUE)ifunc->data, argc, argv, blockarg);
5270 rb_vm_pop_frame(ec);
5271
5272 return val;
5273}
5274
5275 VALUE
5276rb_vm_yield_with_cfunc(rb_execution_context_t *ec, const struct rb_captured_block *captured, int argc, const VALUE *argv)
5277{
5278 return vm_yield_with_cfunc(ec, captured, captured->self, argc, argv, 0, VM_BLOCK_HANDLER_NONE, NULL);
5279}
5280
5281static VALUE
5282vm_yield_with_symbol(rb_execution_context_t *ec, VALUE symbol, int argc, const VALUE *argv, int kw_splat, VALUE block_handler)
5283{
5284 return rb_sym_proc_call(SYM2ID(symbol), argc, argv, kw_splat, rb_vm_bh_to_procval(ec, block_handler));
5285}
5286
5287static inline int
5288vm_callee_setup_block_arg_arg0_splat(rb_control_frame_t *cfp, const rb_iseq_t *iseq, VALUE *argv, VALUE ary)
5289{
5290 int i;
5291 long len = RARRAY_LEN(ary);
5292
5293 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5294
5295 for (i=0; i<len && i<ISEQ_BODY(iseq)->param.lead_num; i++) {
5296 argv[i] = RARRAY_AREF(ary, i);
5297 }
5298
5299 return i;
5300}
5301
5302static inline VALUE
5303vm_callee_setup_block_arg_arg0_check(VALUE *argv)
5304{
5305 VALUE ary, arg0 = argv[0];
5306 ary = rb_check_array_type(arg0);
5307#if 0
5308 argv[0] = arg0;
5309#else
5310 VM_ASSERT(argv[0] == arg0);
5311#endif
5312 return ary;
5313}
5314
5315static int
5316vm_callee_setup_block_arg(rb_execution_context_t *ec, struct rb_calling_info *calling, const struct rb_callinfo *ci, const rb_iseq_t *iseq, VALUE *argv, const enum arg_setup_type arg_setup_type)
5317{
5318 if (rb_simple_iseq_p(iseq)) {
5319 rb_control_frame_t *cfp = ec->cfp;
5320 VALUE arg0;
5321
5322 CALLER_SETUP_ARG(cfp, calling, ci, ISEQ_BODY(iseq)->param.lead_num);
5323
5324 if (arg_setup_type == arg_setup_block &&
5325 calling->argc == 1 &&
5326 ISEQ_BODY(iseq)->param.flags.has_lead &&
5327 !ISEQ_BODY(iseq)->param.flags.ambiguous_param0 &&
5328 !NIL_P(arg0 = vm_callee_setup_block_arg_arg0_check(argv))) {
5329 calling->argc = vm_callee_setup_block_arg_arg0_splat(cfp, iseq, argv, arg0);
5330 }
5331
5332 if (calling->argc != ISEQ_BODY(iseq)->param.lead_num) {
5333 if (arg_setup_type == arg_setup_block) {
5334 if (calling->argc < ISEQ_BODY(iseq)->param.lead_num) {
5335 int i;
5336 CHECK_VM_STACK_OVERFLOW(cfp, ISEQ_BODY(iseq)->param.lead_num);
5337 for (i=calling->argc; i<ISEQ_BODY(iseq)->param.lead_num; i++) argv[i] = Qnil;
5338 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* fill rest parameters */
5339 }
5340 else if (calling->argc > ISEQ_BODY(iseq)->param.lead_num) {
5341 calling->argc = ISEQ_BODY(iseq)->param.lead_num; /* simply truncate arguments */
5342 }
5343 }
5344 else {
5345 argument_arity_error(ec, iseq, NULL, calling->argc, ISEQ_BODY(iseq)->param.lead_num, ISEQ_BODY(iseq)->param.lead_num);
5346 }
5347 }
5348
5349 return 0;
5350 }
5351 else {
5352 return setup_parameters_complex(ec, iseq, calling, ci, argv, arg_setup_type);
5353 }
5354}
5355
5356static int
5357vm_yield_setup_args(rb_execution_context_t *ec, const rb_iseq_t *iseq, const int argc, VALUE *argv, int flags, VALUE block_handler, enum arg_setup_type arg_setup_type)
5358{
5359 struct rb_calling_info calling_entry, *calling;
5360
5361 calling = &calling_entry;
5362 calling->argc = argc;
5363 calling->block_handler = block_handler;
5364 calling->kw_splat = (flags & VM_CALL_KW_SPLAT) ? 1 : 0;
5365 calling->recv = Qundef;
5366 calling->heap_argv = 0;
5367 calling->cc = NULL;
5368 struct rb_callinfo dummy_ci = VM_CI_ON_STACK(0, flags, 0, 0);
5369
5370 return vm_callee_setup_block_arg(ec, calling, &dummy_ci, iseq, argv, arg_setup_type);
5371}
5372
5373/* ruby iseq -> ruby block */
5374
5375static VALUE
5376vm_invoke_iseq_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5377 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5378 bool is_lambda, VALUE block_handler)
5379{
5380 const struct rb_captured_block *captured = VM_BH_TO_ISEQ_BLOCK(block_handler);
5381 const rb_iseq_t *iseq = rb_iseq_check(captured->code.iseq);
5382 const int arg_size = ISEQ_BODY(iseq)->param.size;
5383 VALUE * const rsp = GET_SP() - calling->argc;
5384 VALUE * const argv = rsp;
5385 int opt_pc = vm_callee_setup_block_arg(ec, calling, ci, iseq, argv, is_lambda ? arg_setup_method : arg_setup_block);
5386 int frame_flag = VM_FRAME_MAGIC_BLOCK | (is_lambda ? VM_FRAME_FLAG_LAMBDA : 0);
5387
5388 SET_SP(rsp);
5389
5390 vm_push_frame(ec, iseq,
5391 frame_flag,
5392 captured->self,
5393 VM_GUARDED_PREV_EP(captured->ep), 0,
5394 ISEQ_BODY(iseq)->iseq_encoded + opt_pc,
5395 rsp + arg_size,
5396 ISEQ_BODY(iseq)->local_table_size - arg_size, ISEQ_BODY(iseq)->stack_max);
5397
5398 return Qundef;
5399}
5400
5401static VALUE
5402vm_invoke_symbol_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5403 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5404 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5405{
5406 VALUE symbol = VM_BH_TO_SYMBOL(block_handler);
5407 int flags = vm_ci_flag(ci);
5408
5409 if (UNLIKELY(!(flags & VM_CALL_ARGS_SIMPLE) &&
5410 ((calling->argc == 0) ||
5411 (calling->argc == 1 && (flags & (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT))) ||
5412 (calling->argc == 2 && (flags & VM_CALL_ARGS_SPLAT) && (flags & VM_CALL_KW_SPLAT)) ||
5413 ((flags & VM_CALL_KWARG) && (vm_ci_kwarg(ci)->keyword_len == calling->argc))))) {
5414 CALLER_SETUP_ARG(reg_cfp, calling, ci, ALLOW_HEAP_ARGV);
5415 flags = 0;
5416 if (UNLIKELY(calling->heap_argv)) {
5417#if VM_ARGC_STACK_MAX < 0
5418 if (RARRAY_LEN(calling->heap_argv) < 1) {
5419 rb_raise(rb_eArgError, "no receiver given");
5420 }
5421#endif
5422 calling->recv = rb_ary_shift(calling->heap_argv);
5423 // Modify stack to avoid cfp consistency error
5424 reg_cfp->sp++;
5425 reg_cfp->sp[-1] = reg_cfp->sp[-2];
5426 reg_cfp->sp[-2] = calling->recv;
5427 flags |= VM_CALL_ARGS_SPLAT;
5428 }
5429 else {
5430 if (calling->argc < 1) {
5431 rb_raise(rb_eArgError, "no receiver given");
5432 }
5433 calling->recv = TOPN(--calling->argc);
5434 }
5435 if (calling->kw_splat) {
5436 flags |= VM_CALL_KW_SPLAT;
5437 }
5438 }
5439 else {
5440 if (calling->argc < 1) {
5441 rb_raise(rb_eArgError, "no receiver given");
5442 }
5443 calling->recv = TOPN(--calling->argc);
5444 }
5445
5446 return vm_call_symbol(ec, reg_cfp, calling, ci, symbol, flags);
5447}
5448
5449static VALUE
5450vm_invoke_ifunc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5451 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5452 MAYBE_UNUSED(bool is_lambda), VALUE block_handler)
5453{
5454 VALUE val;
5455 int argc;
5456 const struct rb_captured_block *captured = VM_BH_TO_IFUNC_BLOCK(block_handler);
5457 CALLER_SETUP_ARG(ec->cfp, calling, ci, ALLOW_HEAP_ARGV_KEEP_KWSPLAT);
5458 argc = calling->argc;
5459 val = vm_yield_with_cfunc(ec, captured, captured->self, CALLING_ARGC(calling), calling->heap_argv ? RARRAY_CONST_PTR(calling->heap_argv) : STACK_ADDR_FROM_TOP(argc), calling->kw_splat, calling->block_handler, NULL);
5460 POPN(argc); /* TODO: should put before C/yield? */
5461 return val;
5462}
5463
5464static VALUE
5465vm_proc_to_block_handler(VALUE procval)
5466{
5467 const struct rb_block *block = vm_proc_block(procval);
5468
5469 switch (vm_block_type(block)) {
5470 case block_type_iseq:
5471 return VM_BH_FROM_ISEQ_BLOCK(&block->as.captured);
5472 case block_type_ifunc:
5473 return VM_BH_FROM_IFUNC_BLOCK(&block->as.captured);
5474 case block_type_symbol:
5475 return VM_BH_FROM_SYMBOL(block->as.symbol);
5476 case block_type_proc:
5477 return VM_BH_FROM_PROC(block->as.proc);
5478 }
5479 VM_UNREACHABLE(vm_yield_with_proc);
5480 return Qundef;
5481}
5482
5483static VALUE
5484vm_invoke_proc_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5485 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5486 bool is_lambda, VALUE block_handler)
5487{
5488 while (vm_block_handler_type(block_handler) == block_handler_type_proc) {
5489 VALUE proc = VM_BH_TO_PROC(block_handler);
5490 is_lambda = block_proc_is_lambda(proc);
5491 block_handler = vm_proc_to_block_handler(proc);
5492 }
5493
5494 return vm_invoke_block(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5495}
5496
5497static inline VALUE
5498vm_invoke_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5499 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5500 bool is_lambda, VALUE block_handler)
5501{
5502 VALUE (*func)(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp,
5503 struct rb_calling_info *calling, const struct rb_callinfo *ci,
5504 bool is_lambda, VALUE block_handler);
5505
5506 switch (vm_block_handler_type(block_handler)) {
5507 case block_handler_type_iseq: func = vm_invoke_iseq_block; break;
5508 case block_handler_type_ifunc: func = vm_invoke_ifunc_block; break;
5509 case block_handler_type_proc: func = vm_invoke_proc_block; break;
5510 case block_handler_type_symbol: func = vm_invoke_symbol_block; break;
5511 default: rb_bug("vm_invoke_block: unreachable");
5512 }
5513
5514 return func(ec, reg_cfp, calling, ci, is_lambda, block_handler);
5515}
5516
5517static VALUE
5518vm_make_proc_with_iseq(const rb_iseq_t *blockiseq)
5519{
5520 const rb_execution_context_t *ec = GET_EC();
5521 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
5522 struct rb_captured_block *captured;
5523
5524 if (cfp == 0) {
5525 rb_bug("vm_make_proc_with_iseq: unreachable");
5526 }
5527
5528 captured = VM_CFP_TO_CAPTURED_BLOCK(cfp);
5529 captured->code.iseq = blockiseq;
5530
5531 return rb_vm_make_proc(ec, captured, rb_cProc);
5532}
5533
5534static VALUE
5535vm_once_exec(VALUE iseq)
5536{
5537 VALUE proc = vm_make_proc_with_iseq((rb_iseq_t *)iseq);
5538 return rb_proc_call_with_block(proc, 0, 0, Qnil);
5539}
5540
5541static VALUE
5542vm_once_clear(VALUE data)
5543{
5544 union iseq_inline_storage_entry *is = (union iseq_inline_storage_entry *)data;
5545 is->once.running_thread = NULL;
5546 return Qnil;
5547}
5548
5549/* defined insn */
5550
5551static bool
5552check_respond_to_missing(VALUE obj, VALUE v)
5553{
5554 VALUE args[2];
5555 VALUE r;
5556
5557 args[0] = obj; args[1] = Qfalse;
5558 r = rb_check_funcall(v, idRespond_to_missing, 2, args);
5559 if (!UNDEF_P(r) && RTEST(r)) {
5560 return true;
5561 }
5562 else {
5563 return false;
5564 }
5565}
5566
5567static bool
5568vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5569{
5570 VALUE klass;
5571 enum defined_type type = (enum defined_type)op_type;
5572
5573 switch (type) {
5574 case DEFINED_IVAR:
5575 return rb_ivar_defined(GET_SELF(), SYM2ID(obj));
5576 break;
5577 case DEFINED_GVAR:
5578 return rb_gvar_defined(SYM2ID(obj));
5579 break;
5580 case DEFINED_CVAR: {
5581 const rb_cref_t *cref = vm_get_cref(GET_EP());
5582 klass = vm_get_cvar_base(cref, GET_CFP(), 0);
5583 return rb_cvar_defined(klass, SYM2ID(obj));
5584 break;
5585 }
5586 case DEFINED_CONST:
5587 case DEFINED_CONST_FROM: {
5588 bool allow_nil = type == DEFINED_CONST;
5589 klass = v;
5590 return vm_get_ev_const(ec, klass, SYM2ID(obj), allow_nil, true);
5591 break;
5592 }
5593 case DEFINED_FUNC:
5594 klass = CLASS_OF(v);
5595 return rb_ec_obj_respond_to(ec, v, SYM2ID(obj), TRUE);
5596 break;
5597 case DEFINED_METHOD:{
5598 VALUE klass = CLASS_OF(v);
5599 const rb_callable_method_entry_t *cme = rb_callable_method_entry_with_refinements(klass, SYM2ID(obj), NULL);
5600
5601 if (cme) {
5602 switch (METHOD_ENTRY_VISI(cme)) {
5603 case METHOD_VISI_PRIVATE:
5604 break;
5605 case METHOD_VISI_PROTECTED:
5606 if (!rb_obj_is_kind_of(GET_SELF(), vm_defined_class_for_protected_call(cme))) {
5607 break;
5608 }
5609 case METHOD_VISI_PUBLIC:
5610 return true;
5611 break;
5612 default:
5613 rb_bug("vm_defined: unreachable: %u", (unsigned int)METHOD_ENTRY_VISI(cme));
5614 }
5615 }
5616 else {
5617 return check_respond_to_missing(obj, v);
5618 }
5619 break;
5620 }
5621 case DEFINED_YIELD:
5622 if (GET_BLOCK_HANDLER() != VM_BLOCK_HANDLER_NONE) {
5623 return true;
5624 }
5625 break;
5626 case DEFINED_ZSUPER:
5627 {
5628 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(GET_CFP());
5629
5630 if (me) {
5631 VALUE klass = vm_search_normal_superclass(me->defined_class);
5632 if (!klass) return false;
5633
5634 ID id = me->def->original_id;
5635
5636 return rb_method_boundp(klass, id, 0);
5637 }
5638 }
5639 break;
5640 case DEFINED_REF:
5641 return RTEST(vm_backref_defined(ec, GET_LEP(), FIX2INT(obj)));
5642 default:
5643 rb_bug("unimplemented defined? type (VM)");
5644 break;
5645 }
5646
5647 return false;
5648}
5649
5650bool
5651rb_vm_defined(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, rb_num_t op_type, VALUE obj, VALUE v)
5652{
5653 return vm_defined(ec, reg_cfp, op_type, obj, v);
5654}
5655
5656static const VALUE *
5657vm_get_ep(const VALUE *const reg_ep, rb_num_t lv)
5658{
5659 rb_num_t i;
5660 const VALUE *ep = reg_ep;
5661 for (i = 0; i < lv; i++) {
5662 ep = GET_PREV_EP(ep);
5663 }
5664 return ep;
5665}
5666
5667static VALUE
5668vm_get_special_object(const VALUE *const reg_ep,
5669 enum vm_special_object_type type)
5670{
5671 switch (type) {
5672 case VM_SPECIAL_OBJECT_VMCORE:
5673 return rb_mRubyVMFrozenCore;
5674 case VM_SPECIAL_OBJECT_CBASE:
5675 return vm_get_cbase(reg_ep);
5676 case VM_SPECIAL_OBJECT_CONST_BASE:
5677 return vm_get_const_base(reg_ep);
5678 default:
5679 rb_bug("putspecialobject insn: unknown value_type %d", type);
5680 }
5681}
5682
5683// ZJIT implementation is using the C function
5684// and needs to call a non-static function
5685 VALUE
5686rb_vm_get_special_object(const VALUE *reg_ep, enum vm_special_object_type type)
5687{
5688 return vm_get_special_object(reg_ep, type);
5689}
5690
5691static VALUE
5692vm_concat_array(VALUE ary1, VALUE ary2st)
5693{
5694 const VALUE ary2 = ary2st;
5695 VALUE tmp1 = rb_check_to_array(ary1);
5696 VALUE tmp2 = rb_check_to_array(ary2);
5697
5698 if (NIL_P(tmp1)) {
5699 tmp1 = rb_ary_new3(1, ary1);
5700 }
5701 if (tmp1 == ary1) {
5702 tmp1 = rb_ary_dup(ary1);
5703 }
5704
5705 if (NIL_P(tmp2)) {
5706 return rb_ary_push(tmp1, ary2);
5707 }
5708 else {
5709 return rb_ary_concat(tmp1, tmp2);
5710 }
5711}
5712
5713static VALUE
5714vm_concat_to_array(VALUE ary1, VALUE ary2st)
5715{
5716 /* ary1 must be a newly created array */
5717 const VALUE ary2 = ary2st;
5718
5719 if (NIL_P(ary2)) return ary1;
5720
5721 VALUE tmp2 = rb_check_to_array(ary2);
5722
5723 if (NIL_P(tmp2)) {
5724 return rb_ary_push(ary1, ary2);
5725 }
5726 else {
5727 return rb_ary_concat(ary1, tmp2);
5728 }
5729}
5730
5731// YJIT implementation is using the C function
5732// and needs to call a non-static function
5733 VALUE
5734rb_vm_concat_array(VALUE ary1, VALUE ary2st)
5735{
5736 return vm_concat_array(ary1, ary2st);
5737}
5738
5739 VALUE
5740rb_vm_concat_to_array(VALUE ary1, VALUE ary2st)
5741{
5742 return vm_concat_to_array(ary1, ary2st);
5743}
5744
5745static VALUE
5746vm_splat_array(VALUE flag, VALUE ary)
5747{
5748 if (NIL_P(ary)) {
5749 return RTEST(flag) ? rb_ary_new() : rb_cArray_empty_frozen;
5750 }
5751 VALUE tmp = rb_check_to_array(ary);
5752 if (NIL_P(tmp)) {
5753 return rb_ary_new3(1, ary);
5754 }
5755 else if (RTEST(flag)) {
5756 return rb_ary_dup(tmp);
5757 }
5758 else {
5759 return tmp;
5760 }
5761}
5762
5763// YJIT implementation is using the C function
5764// and needs to call a non-static function
5765 VALUE
5766rb_vm_splat_array(VALUE flag, VALUE ary)
5767{
5768 return vm_splat_array(flag, ary);
5769}
5770
5771static VALUE
5772vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5773{
5774 enum vm_check_match_type type = ((int)flag) & VM_CHECKMATCH_TYPE_MASK;
5775
5776 if (flag & VM_CHECKMATCH_ARRAY) {
5777 long i;
5778 const long n = RARRAY_LEN(pattern);
5779
5780 for (i = 0; i < n; i++) {
5781 VALUE v = RARRAY_AREF(pattern, i);
5782 VALUE c = check_match(ec, v, target, type);
5783
5784 if (RTEST(c)) {
5785 return c;
5786 }
5787 }
5788 return Qfalse;
5789 }
5790 else {
5791 return check_match(ec, pattern, target, type);
5792 }
5793}
5794
5795 VALUE
5796rb_vm_check_match(rb_execution_context_t *ec, VALUE target, VALUE pattern, rb_num_t flag)
5797{
5798 return vm_check_match(ec, target, pattern, flag);
5799}
5800
5801static VALUE
5802vm_check_keyword(lindex_t bits, lindex_t idx, const VALUE *ep)
5803{
5804 const VALUE kw_bits = *(ep - bits);
5805
5806 if (FIXNUM_P(kw_bits)) {
5807 unsigned int b = (unsigned int)FIX2ULONG(kw_bits);
5808 if ((idx < VM_KW_SPECIFIED_BITS_MAX) && (b & (0x01 << idx)))
5809 return Qfalse;
5810 }
5811 else {
5812 VM_ASSERT(RB_TYPE_P(kw_bits, T_HASH));
5813 if (rb_hash_has_key(kw_bits, INT2FIX(idx))) return Qfalse;
5814 }
5815 return Qtrue;
5816}
5817
5818static void
5819vm_dtrace(rb_event_flag_t flag, rb_execution_context_t *ec)
5820{
5821 if (RUBY_DTRACE_METHOD_ENTRY_ENABLED() ||
5822 RUBY_DTRACE_METHOD_RETURN_ENABLED() ||
5823 RUBY_DTRACE_CMETHOD_ENTRY_ENABLED() ||
5824 RUBY_DTRACE_CMETHOD_RETURN_ENABLED()) {
5825
5826 switch (flag) {
5827 case RUBY_EVENT_CALL:
5828 RUBY_DTRACE_METHOD_ENTRY_HOOK(ec, 0, 0);
5829 return;
5830 case RUBY_EVENT_C_CALL:
5831 RUBY_DTRACE_CMETHOD_ENTRY_HOOK(ec, 0, 0);
5832 return;
5833 case RUBY_EVENT_RETURN:
5834 RUBY_DTRACE_METHOD_RETURN_HOOK(ec, 0, 0);
5835 return;
5836 case RUBY_EVENT_C_RETURN:
5837 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, 0, 0);
5838 return;
5839 }
5840 }
5841}
5842
5843static VALUE
5844vm_const_get_under(ID id, rb_num_t flags, VALUE cbase)
5845{
5846 if (!rb_const_defined_at(cbase, id)) {
5847 return 0;
5848 }
5849 else if (VM_DEFINECLASS_SCOPED_P(flags)) {
5850 return rb_public_const_get_at(cbase, id);
5851 }
5852 else {
5853 return rb_const_get_at(cbase, id);
5854 }
5855}
5856
5857static VALUE
5858vm_check_if_class(ID id, rb_num_t flags, VALUE super, VALUE klass)
5859{
5860 if (!RB_TYPE_P(klass, T_CLASS)) {
5861 return 0;
5862 }
5863 else if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags)) {
5864 VALUE tmp = rb_class_real(RCLASS_SUPER(klass));
5865
5866 if (tmp != super) {
5867 rb_raise(rb_eTypeError,
5868 "superclass mismatch for class %"PRIsVALUE"",
5869 rb_id2str(id));
5870 }
5871 else {
5872 return klass;
5873 }
5874 }
5875 else {
5876 return klass;
5877 }
5878}
5879
5880static VALUE
5881vm_check_if_module(ID id, VALUE mod)
5882{
5883 if (!RB_TYPE_P(mod, T_MODULE)) {
5884 return 0;
5885 }
5886 else {
5887 return mod;
5888 }
5889}
5890
5891static VALUE
5892declare_under(ID id, VALUE cbase, VALUE c)
5893{
5894 rb_set_class_path_string(c, cbase, rb_id2str(id));
5895 rb_const_set(cbase, id, c);
5896 return c;
5897}
5898
5899static VALUE
5900vm_declare_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5901{
5902 /* new class declaration */
5903 VALUE s = VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) ? super : rb_cObject;
5904 VALUE c = declare_under(id, cbase, rb_define_class_id(id, s));
5905 rb_define_alloc_func(c, rb_get_alloc_func(c));
5906 rb_class_inherited(s, c);
5907 return c;
5908}
5909
5910static VALUE
5911vm_declare_module(ID id, VALUE cbase)
5912{
5913 /* new module declaration */
5914 return declare_under(id, cbase, rb_module_new());
5915}
5916
5917NORETURN(static void unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old));
5918static void
5919unmatched_redefinition(const char *type, VALUE cbase, ID id, VALUE old)
5920{
5921 VALUE name = rb_id2str(id);
5922 VALUE message = rb_sprintf("%"PRIsVALUE" is not a %s",
5923 name, type);
5924 VALUE location = rb_const_source_location_at(cbase, id);
5925 if (!NIL_P(location)) {
5926 rb_str_catf(message, "\n%"PRIsVALUE":%"PRIsVALUE":"
5927 " previous definition of %"PRIsVALUE" was here",
5928 rb_ary_entry(location, 0), rb_ary_entry(location, 1), name);
5929 }
5930 rb_exc_raise(rb_exc_new_str(rb_eTypeError, message));
5931}
5932
5933static VALUE
5934vm_define_class(ID id, rb_num_t flags, VALUE cbase, VALUE super)
5935{
5936 VALUE klass;
5937
5938 if (VM_DEFINECLASS_HAS_SUPERCLASS_P(flags) && !RB_TYPE_P(super, T_CLASS)) {
5939 rb_raise(rb_eTypeError,
5940 "superclass must be an instance of Class (given an instance of %"PRIsVALUE")",
5941 rb_obj_class(super));
5942 }
5943
5944 vm_check_if_namespace(cbase);
5945
5946 /* find klass */
5947 rb_autoload_load(cbase, id);
5948
5949 if ((klass = vm_const_get_under(id, flags, cbase)) != 0) {
5950 if (!vm_check_if_class(id, flags, super, klass))
5951 unmatched_redefinition("class", cbase, id, klass);
5952 return klass;
5953 }
5954 else {
5955 return vm_declare_class(id, flags, cbase, super);
5956 }
5957}
5958
5959static VALUE
5960vm_define_module(ID id, rb_num_t flags, VALUE cbase)
5961{
5962 VALUE mod;
5963
5964 vm_check_if_namespace(cbase);
5965 if ((mod = vm_const_get_under(id, flags, cbase)) != 0) {
5966 if (!vm_check_if_module(id, mod))
5967 unmatched_redefinition("module", cbase, id, mod);
5968 return mod;
5969 }
5970 else {
5971 return vm_declare_module(id, cbase);
5972 }
5973}
5974
5975static VALUE
5976vm_find_or_create_class_by_id(ID id,
5977 rb_num_t flags,
5978 VALUE cbase,
5979 VALUE super)
5980{
5981 rb_vm_defineclass_type_t type = VM_DEFINECLASS_TYPE(flags);
5982
5983 switch (type) {
5984 case VM_DEFINECLASS_TYPE_CLASS:
5985 /* classdef returns class scope value */
5986 return vm_define_class(id, flags, cbase, super);
5987
5988 case VM_DEFINECLASS_TYPE_SINGLETON_CLASS:
5989 /* classdef returns class scope value */
5990 return rb_singleton_class(cbase);
5991
5992 case VM_DEFINECLASS_TYPE_MODULE:
5993 /* classdef returns class scope value */
5994 return vm_define_module(id, flags, cbase);
5995
5996 default:
5997 rb_bug("unknown defineclass type: %d", (int)type);
5998 }
5999}
6000
6001static rb_method_visibility_t
6002vm_scope_visibility_get(const rb_execution_context_t *ec)
6003{
6004 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6005
6006 if (!vm_env_cref_by_cref(cfp->ep)) {
6007 return METHOD_VISI_PUBLIC;
6008 }
6009 else {
6010 return CREF_SCOPE_VISI(vm_ec_cref(ec))->method_visi;
6011 }
6012}
6013
6014static int
6015vm_scope_module_func_check(const rb_execution_context_t *ec)
6016{
6017 const rb_control_frame_t *cfp = rb_vm_get_ruby_level_next_cfp(ec, ec->cfp);
6018
6019 if (!vm_env_cref_by_cref(cfp->ep)) {
6020 return FALSE;
6021 }
6022 else {
6023 return CREF_SCOPE_VISI(vm_ec_cref(ec))->module_func;
6024 }
6025}
6026
6027static void
6028vm_define_method(const rb_execution_context_t *ec, VALUE obj, ID id, VALUE iseqval, int is_singleton)
6029{
6030 VALUE klass;
6031 rb_method_visibility_t visi;
6032 rb_cref_t *cref = vm_ec_cref(ec);
6033
6034 if (is_singleton) {
6035 klass = rb_singleton_class(obj); /* class and frozen checked in this API */
6036 visi = METHOD_VISI_PUBLIC;
6037 }
6038 else {
6039 klass = CREF_CLASS_FOR_DEFINITION(cref);
6040 visi = vm_scope_visibility_get(ec);
6041 }
6042
6043 if (NIL_P(klass)) {
6044 rb_raise(rb_eTypeError, "no class/module to add method");
6045 }
6046
6047 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, visi);
6048 // Set max_iv_count on klasses based on number of ivar sets that are in the initialize method
6049 if (id == idInitialize && klass != rb_cObject && RB_TYPE_P(klass, T_CLASS) && (rb_get_alloc_func(klass) == rb_class_allocate_instance)) {
6050 RCLASS_SET_MAX_IV_COUNT(klass, rb_estimate_iv_count(klass, (const rb_iseq_t *)iseqval));
6051 }
6052
6053 if (!is_singleton && vm_scope_module_func_check(ec)) {
6054 klass = rb_singleton_class(klass);
6055 rb_add_method_iseq(klass, id, (const rb_iseq_t *)iseqval, cref, METHOD_VISI_PUBLIC);
6056 }
6057}
6058
6059// Return the untagged block handler:
6060// * If it's VM_BLOCK_HANDLER_NONE, return nil
6061// * If it's an ISEQ or an IFUNC, fetch it from its rb_captured_block
6062// * If it's a PROC or SYMBOL, return it as is
6063static VALUE
6064rb_vm_untag_block_handler(VALUE block_handler)
6065{
6066 if (VM_BLOCK_HANDLER_NONE == block_handler) return Qnil;
6067
6068 switch (vm_block_handler_type(block_handler)) {
6069 case block_handler_type_iseq:
6070 case block_handler_type_ifunc: {
6071 struct rb_captured_block *captured = VM_TAGGED_PTR_REF(block_handler, 0x03);
6072 return captured->code.val;
6073 }
6074 case block_handler_type_proc:
6075 case block_handler_type_symbol:
6076 return block_handler;
6077 default:
6078 rb_bug("rb_vm_untag_block_handler: unreachable");
6079 }
6080}
6081
6082 VALUE
6083rb_vm_get_untagged_block_handler(rb_control_frame_t *reg_cfp)
6084{
6085 return rb_vm_untag_block_handler(VM_CF_BLOCK_HANDLER(reg_cfp));
6086}
6087
6088static VALUE
6089vm_invokeblock_i(struct rb_execution_context_struct *ec,
6090 struct rb_control_frame_struct *reg_cfp,
6091 struct rb_calling_info *calling)
6092{
6093 const struct rb_callinfo *ci = calling->cd->ci;
6094 VALUE block_handler = VM_CF_BLOCK_HANDLER(GET_CFP());
6095
6096 if (block_handler == VM_BLOCK_HANDLER_NONE) {
6097 rb_vm_localjump_error("no block given (yield)", Qnil, 0);
6098 }
6099 else {
6100 return vm_invoke_block(ec, GET_CFP(), calling, ci, false, block_handler);
6101 }
6102}
6103
6104enum method_explorer_type {
6105 mexp_search_method,
6106 mexp_search_invokeblock,
6107 mexp_search_super,
6108};
6109
6110static inline VALUE
6111vm_sendish(
6112 struct rb_execution_context_struct *ec,
6113 struct rb_control_frame_struct *reg_cfp,
6114 struct rb_call_data *cd,
6115 VALUE block_handler,
6116 enum method_explorer_type method_explorer
6117) {
6118 VALUE val = Qundef;
6119 const struct rb_callinfo *ci = cd->ci;
6120 const struct rb_callcache *cc;
6121 int argc = vm_ci_argc(ci);
6122 VALUE recv = TOPN(argc);
6123 struct rb_calling_info calling = {
6124 .block_handler = block_handler,
6125 .kw_splat = IS_ARGS_KW_SPLAT(ci) > 0,
6126 .recv = recv,
6127 .argc = argc,
6128 .cd = cd,
6129 };
6130
6131 switch (method_explorer) {
6132 case mexp_search_method:
6133 calling.cc = cc = vm_search_method_fastpath((VALUE)reg_cfp->iseq, cd, CLASS_OF(recv));
6134 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6135 break;
6136 case mexp_search_super:
6137 calling.cc = cc = vm_search_super_method(reg_cfp, cd, recv);
6138 val = vm_cc_call(cc)(ec, GET_CFP(), &calling);
6139 break;
6140 case mexp_search_invokeblock:
6141 val = vm_invokeblock_i(ec, GET_CFP(), &calling);
6142 break;
6143 }
6144 return val;
6145}
6146
6147 VALUE
6148rb_vm_send(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6149{
6150 stack_check(ec);
6151 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), cd->ci, blockiseq, false);
6152 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6153 VM_EXEC(ec, val);
6154 return val;
6155}
6156
6157 VALUE
6158rb_vm_sendforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6159{
6160 stack_check(ec);
6161
6162 struct rb_forwarding_call_data adjusted_cd;
6163 struct rb_callinfo adjusted_ci;
6164
6165 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, false, &adjusted_cd, &adjusted_ci);
6166
6167 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_method);
6168
6169 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6170 RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
6171 }
6172
6173 VM_EXEC(ec, val);
6174 return val;
6175}
6176
6177 VALUE
6178rb_vm_opt_send_without_block(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6179{
6180 stack_check(ec);
6181 VALUE bh = VM_BLOCK_HANDLER_NONE;
6182 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_method);
6183 VM_EXEC(ec, val);
6184 return val;
6185}
6186
6187 VALUE
6188rb_vm_invokesuper(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6189{
6190 stack_check(ec);
6191
6192 struct rb_callinfo adjusted_ci = VM_CI_ON_STACK(vm_ci_mid(cd->ci),
6193 vm_ci_flag(cd->ci),
6194 vm_ci_argc(cd->ci),
6195 vm_ci_kwarg(cd->ci));
6196 const struct rb_callcache *original_cc = rbimpl_atomic_ptr_load((void **)&cd->cc, RBIMPL_ATOMIC_ACQUIRE);
6197 struct rb_call_data adjusted_cd = {
6198 .ci = &adjusted_ci,
6199 .cc = original_cc,
6200 };
6201
6202 VALUE bh = vm_caller_setup_arg_block(ec, GET_CFP(), adjusted_cd.ci, blockiseq, true);
6203 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd, bh, mexp_search_super);
6204
6205 if (original_cc != adjusted_cd.cc && vm_cc_markable(adjusted_cd.cc)) {
6206 rbimpl_atomic_ptr_store((volatile void **)&cd->cc, (void *)adjusted_cd.cc, RBIMPL_ATOMIC_RELEASE);
6207 RB_OBJ_WRITTEN(GET_ISEQ(), Qundef, adjusted_cd.cc);
6208 }
6209
6210 VM_EXEC(ec, val);
6211 return val;
6212}
6213
6214 VALUE
6215rb_vm_invokesuperforward(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd, ISEQ blockiseq)
6216{
6217 stack_check(ec);
6218 struct rb_forwarding_call_data adjusted_cd;
6219 struct rb_callinfo adjusted_ci;
6220
6221 VALUE bh = vm_caller_setup_fwd_args(GET_EC(), GET_CFP(), cd, blockiseq, true, &adjusted_cd, &adjusted_ci);
6222
6223 VALUE val = vm_sendish(ec, GET_CFP(), &adjusted_cd.cd, bh, mexp_search_super);
6224
6225 if (cd->cc != adjusted_cd.cd.cc && vm_cc_markable(adjusted_cd.cd.cc)) {
6226 RB_OBJ_WRITE(GET_ISEQ(), &cd->cc, adjusted_cd.cd.cc);
6227 }
6228
6229 VM_EXEC(ec, val);
6230 return val;
6231}
6232
6233 VALUE
6234rb_vm_invokeblock(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, CALL_DATA cd)
6235{
6236 stack_check(ec);
6237 VALUE bh = VM_BLOCK_HANDLER_NONE;
6238 VALUE val = vm_sendish(ec, GET_CFP(), cd, bh, mexp_search_invokeblock);
6239 VM_EXEC(ec, val);
6240 return val;
6241}
6242
6243/* object.c */
6244 VALUE rb_nil_to_s(VALUE);
6245 VALUE rb_true_to_s(VALUE);
6246 VALUE rb_false_to_s(VALUE);
6247/* numeric.c */
6248 VALUE rb_int_to_s(int argc, VALUE *argv, VALUE x);
6249 VALUE rb_fix_to_s(VALUE);
6250/* variable.c */
6251 VALUE rb_mod_to_s(VALUE);
6252 VALUE rb_mod_name(VALUE);
6253
6254static VALUE
6255vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd)
6256{
6257 int type = TYPE(recv);
6258 if (type == T_STRING) {
6259 return recv;
6260 }
6261
6262 const struct rb_callable_method_entry_struct *cme = vm_search_method((VALUE)iseq, cd, recv);
6263
6264 switch (type) {
6265 case T_SYMBOL:
6266 if (check_method_basic_definition(cme)) {
6267 // rb_sym_to_s() allocates a mutable string, but since we are only
6268 // going to use this string for interpolation, it's fine to use the
6269 // frozen string.
6270 return rb_sym2str(recv);
6271 }
6272 break;
6273 case T_MODULE:
6274 case T_CLASS:
6275 if (check_cfunc(cme, rb_mod_to_s)) {
6276 // rb_mod_to_s() allocates a mutable string, but since we are only
6277 // going to use this string for interpolation, it's fine to use the
6278 // frozen string.
6279 VALUE val = rb_mod_name(recv);
6280 if (NIL_P(val)) {
6281 val = rb_mod_to_s(recv);
6282 }
6283 return val;
6284 }
6285 break;
6286 case T_NIL:
6287 if (check_cfunc(cme, rb_nil_to_s)) {
6288 return rb_nil_to_s(recv);
6289 }
6290 break;
6291 case T_TRUE:
6292 if (check_cfunc(cme, rb_true_to_s)) {
6293 return rb_true_to_s(recv);
6294 }
6295 break;
6296 case T_FALSE:
6297 if (check_cfunc(cme, rb_false_to_s)) {
6298 return rb_false_to_s(recv);
6299 }
6300 break;
6301 case T_FIXNUM:
6302 if (check_cfunc(cme, rb_int_to_s)) {
6303 return rb_fix_to_s(recv);
6304 }
6305 break;
6306 }
6307 return Qundef;
6308}
6309
6310// ZJIT implementation is using the C function
6311// and needs to call a non-static function
6312 VALUE
6313rb_vm_objtostring(const rb_iseq_t *iseq, VALUE recv, CALL_DATA cd)
6314{
6315 return vm_objtostring(iseq, recv, cd);
6316}
6317
6318static VALUE
6319vm_opt_ary_freeze(VALUE ary, int bop, ID id)
6320{
6321 if (BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
6322 return ary;
6323 }
6324 else {
6325 return Qundef;
6326 }
6327}
6328
6329static VALUE
6330vm_opt_hash_freeze(VALUE hash, int bop, ID id)
6331{
6332 if (BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
6333 return hash;
6334 }
6335 else {
6336 return Qundef;
6337 }
6338}
6339
6340static VALUE
6341vm_opt_str_freeze(VALUE str, int bop, ID id)
6342{
6343 if (BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
6344 return str;
6345 }
6346 else {
6347 return Qundef;
6348 }
6349}
6350
6351/* this macro is mandatory to use OPTIMIZED_CMP. What a design! */
6352#define id_cmp idCmp
6353
6354static VALUE
6355vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6356{
6357 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6358 return rb_ary_includes(ary, target);
6359 }
6360 else {
6361 VALUE args[1] = {target};
6362
6363 // duparray
6364 RUBY_DTRACE_CREATE_HOOK(ARRAY, RARRAY_LEN(ary));
6365 VALUE dupary = rb_ary_resurrect(ary);
6366
6367 return rb_vm_call_with_refinements(ec, dupary, idIncludeP, 1, args, RB_NO_KEYWORDS);
6368 }
6369}
6370
6371 VALUE
6372rb_vm_opt_duparray_include_p(rb_execution_context_t *ec, const VALUE ary, VALUE target)
6373{
6374 return vm_opt_duparray_include_p(ec, ary, target);
6375}
6376
6377static VALUE
6378vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6379{
6380 if (BASIC_OP_UNREDEFINED_P(BOP_MAX, ARRAY_REDEFINED_OP_FLAG)) {
6381 if (array_len == 0) {
6382 return Qnil;
6383 }
6384 else {
6385 VALUE result = *ptr;
6386 rb_snum_t i = array_len - 1;
6387 while (i-- > 0) {
6388 const VALUE v = *++ptr;
6389 if (OPTIMIZED_CMP(v, result) > 0) {
6390 result = v;
6391 }
6392 }
6393 return result;
6394 }
6395 }
6396 else {
6397 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMax, 0, NULL, RB_NO_KEYWORDS);
6398 }
6399}
6400
6401 VALUE
6402rb_vm_opt_newarray_max(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6403{
6404 return vm_opt_newarray_max(ec, array_len, ptr);
6405}
6406
6407static VALUE
6408vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6409{
6410 if (BASIC_OP_UNREDEFINED_P(BOP_MIN, ARRAY_REDEFINED_OP_FLAG)) {
6411 if (array_len == 0) {
6412 return Qnil;
6413 }
6414 else {
6415 VALUE result = *ptr;
6416 rb_snum_t i = array_len - 1;
6417 while (i-- > 0) {
6418 const VALUE v = *++ptr;
6419 if (OPTIMIZED_CMP(v, result) < 0) {
6420 result = v;
6421 }
6422 }
6423 return result;
6424 }
6425 }
6426 else {
6427 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idMin, 0, NULL, RB_NO_KEYWORDS);
6428 }
6429}
6430
6431 VALUE
6432rb_vm_opt_newarray_min(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6433{
6434 return vm_opt_newarray_min(ec, array_len, ptr);
6435}
6436
6437static VALUE
6438vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6439{
6440 // If Array#hash is _not_ monkeypatched, use the optimized call
6441 if (BASIC_OP_UNREDEFINED_P(BOP_HASH, ARRAY_REDEFINED_OP_FLAG)) {
6442 return rb_ary_hash_values(array_len, ptr);
6443 }
6444 else {
6445 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idHash, 0, NULL, RB_NO_KEYWORDS);
6446 }
6447}
6448
6449 VALUE
6450rb_vm_opt_newarray_hash(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr)
6451{
6452 return vm_opt_newarray_hash(ec, array_len, ptr);
6453}
6454
6455 VALUE rb_setup_fake_ary(struct RArray *fake_ary, const VALUE *list, long len);
6456 VALUE rb_ec_pack_ary(rb_execution_context_t *ec, VALUE ary, VALUE fmt, VALUE buffer);
6457
6458static VALUE
6459vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6460{
6461 if (BASIC_OP_UNREDEFINED_P(BOP_INCLUDE_P, ARRAY_REDEFINED_OP_FLAG)) {
6462 struct RArray fake_ary = {RBASIC_INIT};
6463 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6464 return rb_ary_includes(ary, target);
6465 }
6466 else {
6467 VALUE args[1] = {target};
6468 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idIncludeP, 1, args, RB_NO_KEYWORDS);
6469 }
6470}
6471
6472 VALUE
6473rb_vm_opt_newarray_include_p(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE target)
6474{
6475 return vm_opt_newarray_include_p(ec, array_len, ptr, target);
6476}
6477
6478static VALUE
6479vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6480{
6481 if (BASIC_OP_UNREDEFINED_P(BOP_PACK, ARRAY_REDEFINED_OP_FLAG)) {
6482 struct RArray fake_ary = {RBASIC_INIT};
6483 VALUE ary = rb_setup_fake_ary(&fake_ary, ptr, array_len);
6484 return rb_ec_pack_ary(ec, ary, fmt, (UNDEF_P(buffer) ? Qnil : buffer));
6485 }
6486 else {
6487 // The opt_newarray_send insn drops the keyword args so we need to rebuild them.
6488 // Setup an array with room for keyword hash.
6489 VALUE args[2];
6490 args[0] = fmt;
6491 int kw_splat = RB_NO_KEYWORDS;
6492 int argc = 1;
6493
6494 if (!UNDEF_P(buffer)) {
6495 args[1] = rb_hash_new_with_size(1);
6496 rb_hash_aset(args[1], ID2SYM(idBuffer), buffer);
6497 kw_splat = RB_PASS_KEYWORDS;
6498 argc++;
6499 }
6500
6501 return rb_vm_call_with_refinements(ec, rb_ary_new4(array_len, ptr), idPack, argc, args, kw_splat);
6502 }
6503}
6504
6505 VALUE
6506rb_vm_opt_newarray_pack_buffer(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt, VALUE buffer)
6507{
6508 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, buffer);
6509}
6510
6511 VALUE
6512rb_vm_opt_newarray_pack(rb_execution_context_t *ec, rb_num_t array_len, const VALUE *ptr, VALUE fmt)
6513{
6514 return vm_opt_newarray_pack_buffer(ec, array_len, ptr, fmt, Qundef);
6515}
6516
6517#undef id_cmp
6518
6519static void
6520vm_track_constant_cache(ID id, void *ic)
6521{
6522 rb_vm_t *vm = GET_VM();
6523 struct rb_id_table *const_cache = vm->constant_cache;
6524 VALUE lookup_result;
6525 set_table *ics;
6526
6527 if (rb_id_table_lookup(const_cache, id, &lookup_result)) {
6528 ics = (set_table *)lookup_result;
6529 }
6530 else {
6531 ics = set_init_numtable();
6532 rb_id_table_insert(const_cache, id, (VALUE)ics);
6533 }
6534
6535 /* The call below to st_insert could allocate which could trigger a GC.
6536 * If it triggers a GC, it may free an iseq that also holds a cache to this
6537 * constant. If that iseq is the last iseq with a cache to this constant, then
6538 * it will free this ST table, which would cause an use-after-free during this
6539 * st_insert.
6540 *
6541 * So to fix this issue, we store the ID that is currently being inserted
6542 * and, in remove_from_constant_cache, we don't free the ST table for ID
6543 * equal to this one.
6544 *
6545 * See [Bug #20921].
6546 */
6547 vm->inserting_constant_cache_id = id;
6548
6549 set_insert(ics, (st_data_t)ic);
6550
6551 vm->inserting_constant_cache_id = (ID)0;
6552}
6553
6554static void
6555vm_ic_track_const_chain(rb_control_frame_t *cfp, IC ic, const ID *segments)
6556{
6557 RB_VM_LOCKING() {
6558 for (int i = 0; segments[i]; i++) {
6559 ID id = segments[i];
6560 if (id == idNULL) continue;
6561 vm_track_constant_cache(id, ic);
6562 }
6563 }
6564}
6565
6566// For JIT inlining
6567static inline bool
6568vm_inlined_ic_hit_p(VALUE flags, VALUE value, const rb_cref_t *ic_cref, const VALUE *reg_ep)
6569{
6570 if ((flags & IMEMO_CONST_CACHE_SHAREABLE) || rb_ractor_main_p()) {
6571 VM_ASSERT(ractor_incidental_shareable_p(flags & IMEMO_CONST_CACHE_SHAREABLE, value));
6572
6573 return (ic_cref == NULL || // no need to check CREF
6574 ic_cref == vm_get_cref(reg_ep));
6575 }
6576 return false;
6577}
6578
6579static bool
6580vm_ic_hit_p(const struct iseq_inline_constant_cache_entry *ice, const VALUE *reg_ep)
6581{
6582 VM_ASSERT(IMEMO_TYPE_P(ice, imemo_constcache));
6583 return vm_inlined_ic_hit_p(ice->flags, ice->value, ice->ic_cref, reg_ep);
6584}
6585
6586// YJIT needs this function to never allocate and never raise
6587bool
6588rb_vm_ic_hit_p(IC ic, const VALUE *reg_ep)
6589{
6590 return ic->entry && vm_ic_hit_p(ic->entry, reg_ep);
6591}
6592
6593static void
6594vm_ic_update(const rb_iseq_t *iseq, IC ic, VALUE val, const VALUE *reg_ep, const VALUE *pc)
6595{
6596 if (ruby_vm_const_missing_count > 0) {
6597 ruby_vm_const_missing_count = 0;
6598 ic->entry = NULL;
6599 return;
6600 }
6601
6602 struct iseq_inline_constant_cache_entry *ice = SHAREABLE_IMEMO_NEW(struct iseq_inline_constant_cache_entry, imemo_constcache, 0);
6603 RB_OBJ_WRITE(ice, &ice->value, val);
6604 ice->ic_cref = vm_get_const_key_cref(reg_ep);
6605
6606 if (rb_ractor_shareable_p(val)) {
6607 RUBY_ASSERT((rb_gc_verify_shareable(val), 1));
6608 ice->flags |= IMEMO_CONST_CACHE_SHAREABLE;
6609 }
6610 RB_OBJ_WRITE(iseq, &ic->entry, ice);
6611 RUBY_ASSERT(pc >= ISEQ_BODY(iseq)->iseq_encoded);
6612 unsigned pos = (unsigned)(pc - ISEQ_BODY(iseq)->iseq_encoded);
6613 rb_yjit_constant_ic_update(iseq, ic, pos);
6614}
6615
6616 VALUE
6617rb_vm_opt_getconstant_path(rb_execution_context_t *ec, rb_control_frame_t *const reg_cfp, IC ic)
6618{
6619 VALUE val;
6620 const ID *segments = ic->segments;
6621 struct iseq_inline_constant_cache_entry *ice = ic->entry;
6622
6623 if (ice && vm_ic_hit_p(ice, GET_EP())) {
6624 val = ice->value;
6625
6626 VM_ASSERT(val == vm_get_ev_const_chain(ec, segments));
6627 }
6628 else {
6629 ruby_vm_constant_cache_misses++;
6630 val = vm_get_ev_const_chain(ec, segments);
6631 vm_ic_track_const_chain(GET_CFP(), ic, segments);
6632 // Undo the PC increment to get the address to this instruction
6633 // INSN_ATTR(width) == 2
6634 vm_ic_update(GET_ISEQ(), ic, val, GET_EP(), GET_PC() - 2);
6635 }
6636 return val;
6637}
6638
6639static VALUE
6640vm_once_dispatch(rb_execution_context_t *ec, ISEQ iseq, ISE is)
6641{
6642 rb_thread_t *th = rb_ec_thread_ptr(ec);
6643 rb_thread_t *const RUNNING_THREAD_ONCE_DONE = (rb_thread_t *)(0x1);
6644
6645 again:
6646 if (is->once.running_thread == RUNNING_THREAD_ONCE_DONE) {
6647 return is->once.value;
6648 }
6649 else if (is->once.running_thread == NULL) {
6650 VALUE val;
6651 is->once.running_thread = th;
6652 val = rb_ensure(vm_once_exec, (VALUE)iseq, vm_once_clear, (VALUE)is);
6653 // TODO: confirm that it is shareable
6654
6655 if (RB_FL_ABLE(val)) {
6656 RB_OBJ_SET_SHAREABLE(val);
6657 }
6658
6659 RB_OBJ_WRITE(ec->cfp->iseq, &is->once.value, val);
6660
6661 /* is->once.running_thread is cleared by vm_once_clear() */
6662 is->once.running_thread = RUNNING_THREAD_ONCE_DONE; /* success */
6663 return val;
6664 }
6665 else if (is->once.running_thread == th) {
6666 /* recursive once */
6667 return vm_once_exec((VALUE)iseq);
6668 }
6669 else {
6670 /* waiting for finish */
6671 RUBY_VM_CHECK_INTS(ec);
6672 rb_thread_schedule();
6673 goto again;
6674 }
6675}
6676
6677static OFFSET
6678vm_case_dispatch(CDHASH hash, OFFSET else_offset, VALUE key)
6679{
6680 switch (OBJ_BUILTIN_TYPE(key)) {
6681 case -1:
6682 case T_FLOAT:
6683 case T_SYMBOL:
6684 case T_BIGNUM:
6685 case T_STRING:
6686 if (BASIC_OP_UNREDEFINED_P(BOP_EQQ,
6687 SYMBOL_REDEFINED_OP_FLAG |
6688 INTEGER_REDEFINED_OP_FLAG |
6689 FLOAT_REDEFINED_OP_FLAG |
6690 NIL_REDEFINED_OP_FLAG |
6691 TRUE_REDEFINED_OP_FLAG |
6692 FALSE_REDEFINED_OP_FLAG |
6693 STRING_REDEFINED_OP_FLAG)) {
6694 st_data_t val;
6695 if (RB_FLOAT_TYPE_P(key)) {
6696 double kval = RFLOAT_VALUE(key);
6697 if (!isinf(kval) && modf(kval, &kval) == 0.0) {
6698 key = FIXABLE(kval) ? LONG2FIX((long)kval) : rb_dbl2big(kval);
6699 }
6700 }
6701 if (rb_hash_stlike_lookup(hash, key, &val)) {
6702 return FIX2LONG((VALUE)val);
6703 }
6704 else {
6705 return else_offset;
6706 }
6707 }
6708 }
6709 return 0;
6710}
6711
6712NORETURN(static void
6713 vm_stack_consistency_error(const rb_execution_context_t *ec,
6714 const rb_control_frame_t *,
6715 const VALUE *));
6716static void
6717vm_stack_consistency_error(const rb_execution_context_t *ec,
6718 const rb_control_frame_t *cfp,
6719 const VALUE *bp)
6720{
6721 const ptrdiff_t nsp = VM_SP_CNT(ec, cfp->sp);
6722 const ptrdiff_t nbp = VM_SP_CNT(ec, bp);
6723 static const char stack_consistency_error[] =
6724 "Stack consistency error (sp: %"PRIdPTRDIFF", bp: %"PRIdPTRDIFF")";
6725#if defined RUBY_DEVEL
6726 VALUE mesg = rb_sprintf(stack_consistency_error, nsp, nbp);
6727 rb_str_cat_cstr(mesg, "\n");
6728 rb_str_append(mesg, rb_iseq_disasm(cfp->iseq));
6729 rb_exc_fatal(rb_exc_new3(rb_eFatal, mesg));
6730#else
6731 rb_bug(stack_consistency_error, nsp, nbp);
6732#endif
6733}
6734
6735static VALUE
6736vm_opt_plus(VALUE recv, VALUE obj)
6737{
6738 if (FIXNUM_2_P(recv, obj) &&
6739 BASIC_OP_UNREDEFINED_P(BOP_PLUS, INTEGER_REDEFINED_OP_FLAG)) {
6740 return rb_fix_plus_fix(recv, obj);
6741 }
6742 else if (FLONUM_2_P(recv, obj) &&
6743 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6744 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6745 }
6746 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6747 return Qundef;
6748 }
6749 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6750 RBASIC_CLASS(obj) == rb_cFloat &&
6751 BASIC_OP_UNREDEFINED_P(BOP_PLUS, FLOAT_REDEFINED_OP_FLAG)) {
6752 return DBL2NUM(RFLOAT_VALUE(recv) + RFLOAT_VALUE(obj));
6753 }
6754 else if (RBASIC_CLASS(recv) == rb_cString &&
6755 RBASIC_CLASS(obj) == rb_cString &&
6756 BASIC_OP_UNREDEFINED_P(BOP_PLUS, STRING_REDEFINED_OP_FLAG)) {
6757 return rb_str_opt_plus(recv, obj);
6758 }
6759 else if (RBASIC_CLASS(recv) == rb_cArray &&
6760 RBASIC_CLASS(obj) == rb_cArray &&
6761 BASIC_OP_UNREDEFINED_P(BOP_PLUS, ARRAY_REDEFINED_OP_FLAG)) {
6762 return rb_ary_plus(recv, obj);
6763 }
6764 else {
6765 return Qundef;
6766 }
6767}
6768
6769static VALUE
6770vm_opt_minus(VALUE recv, VALUE obj)
6771{
6772 if (FIXNUM_2_P(recv, obj) &&
6773 BASIC_OP_UNREDEFINED_P(BOP_MINUS, INTEGER_REDEFINED_OP_FLAG)) {
6774 return rb_fix_minus_fix(recv, obj);
6775 }
6776 else if (FLONUM_2_P(recv, obj) &&
6777 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6778 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6779 }
6780 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6781 return Qundef;
6782 }
6783 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6784 RBASIC_CLASS(obj) == rb_cFloat &&
6785 BASIC_OP_UNREDEFINED_P(BOP_MINUS, FLOAT_REDEFINED_OP_FLAG)) {
6786 return DBL2NUM(RFLOAT_VALUE(recv) - RFLOAT_VALUE(obj));
6787 }
6788 else {
6789 return Qundef;
6790 }
6791}
6792
6793static VALUE
6794vm_opt_mult(VALUE recv, VALUE obj)
6795{
6796 if (FIXNUM_2_P(recv, obj) &&
6797 BASIC_OP_UNREDEFINED_P(BOP_MULT, INTEGER_REDEFINED_OP_FLAG)) {
6798 return rb_fix_mul_fix(recv, obj);
6799 }
6800 else if (FLONUM_2_P(recv, obj) &&
6801 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6802 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6803 }
6804 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6805 return Qundef;
6806 }
6807 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6808 RBASIC_CLASS(obj) == rb_cFloat &&
6809 BASIC_OP_UNREDEFINED_P(BOP_MULT, FLOAT_REDEFINED_OP_FLAG)) {
6810 return DBL2NUM(RFLOAT_VALUE(recv) * RFLOAT_VALUE(obj));
6811 }
6812 else {
6813 return Qundef;
6814 }
6815}
6816
6817static VALUE
6818vm_opt_div(VALUE recv, VALUE obj)
6819{
6820 if (FIXNUM_2_P(recv, obj) &&
6821 BASIC_OP_UNREDEFINED_P(BOP_DIV, INTEGER_REDEFINED_OP_FLAG)) {
6822 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_div_fix(recv, obj);
6823 }
6824 else if (FLONUM_2_P(recv, obj) &&
6825 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6826 return rb_flo_div_flo(recv, obj);
6827 }
6828 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6829 return Qundef;
6830 }
6831 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6832 RBASIC_CLASS(obj) == rb_cFloat &&
6833 BASIC_OP_UNREDEFINED_P(BOP_DIV, FLOAT_REDEFINED_OP_FLAG)) {
6834 return rb_flo_div_flo(recv, obj);
6835 }
6836 else {
6837 return Qundef;
6838 }
6839}
6840
6841static VALUE
6842vm_opt_mod(VALUE recv, VALUE obj)
6843{
6844 if (FIXNUM_2_P(recv, obj) &&
6845 BASIC_OP_UNREDEFINED_P(BOP_MOD, INTEGER_REDEFINED_OP_FLAG)) {
6846 return (FIX2LONG(obj) == 0) ? Qundef : rb_fix_mod_fix(recv, obj);
6847 }
6848 else if (FLONUM_2_P(recv, obj) &&
6849 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6850 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6851 }
6852 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6853 return Qundef;
6854 }
6855 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6856 RBASIC_CLASS(obj) == rb_cFloat &&
6857 BASIC_OP_UNREDEFINED_P(BOP_MOD, FLOAT_REDEFINED_OP_FLAG)) {
6858 return DBL2NUM(ruby_float_mod(RFLOAT_VALUE(recv), RFLOAT_VALUE(obj)));
6859 }
6860 else {
6861 return Qundef;
6862 }
6863}
6864
6865static VALUE
6866vm_opt_neq(const rb_iseq_t *iseq, CALL_DATA cd, CALL_DATA cd_eq, VALUE recv, VALUE obj)
6867{
6868 if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not_equal)) {
6869 VALUE val = opt_equality(iseq, recv, obj, cd_eq);
6870
6871 if (!UNDEF_P(val)) {
6872 return RBOOL(!RTEST(val));
6873 }
6874 }
6875
6876 return Qundef;
6877}
6878
6879static VALUE
6880vm_opt_lt(VALUE recv, VALUE obj)
6881{
6882 if (FIXNUM_2_P(recv, obj) &&
6883 BASIC_OP_UNREDEFINED_P(BOP_LT, INTEGER_REDEFINED_OP_FLAG)) {
6884 return RBOOL((SIGNED_VALUE)recv < (SIGNED_VALUE)obj);
6885 }
6886 else if (FLONUM_2_P(recv, obj) &&
6887 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6888 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6889 }
6890 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6891 return Qundef;
6892 }
6893 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6894 RBASIC_CLASS(obj) == rb_cFloat &&
6895 BASIC_OP_UNREDEFINED_P(BOP_LT, FLOAT_REDEFINED_OP_FLAG)) {
6896 return RBOOL(RFLOAT_VALUE(recv) < RFLOAT_VALUE(obj));
6897 }
6898 else {
6899 return Qundef;
6900 }
6901}
6902
6903static VALUE
6904vm_opt_le(VALUE recv, VALUE obj)
6905{
6906 if (FIXNUM_2_P(recv, obj) &&
6907 BASIC_OP_UNREDEFINED_P(BOP_LE, INTEGER_REDEFINED_OP_FLAG)) {
6908 return RBOOL((SIGNED_VALUE)recv <= (SIGNED_VALUE)obj);
6909 }
6910 else if (FLONUM_2_P(recv, obj) &&
6911 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6912 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6913 }
6914 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6915 return Qundef;
6916 }
6917 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6918 RBASIC_CLASS(obj) == rb_cFloat &&
6919 BASIC_OP_UNREDEFINED_P(BOP_LE, FLOAT_REDEFINED_OP_FLAG)) {
6920 return RBOOL(RFLOAT_VALUE(recv) <= RFLOAT_VALUE(obj));
6921 }
6922 else {
6923 return Qundef;
6924 }
6925}
6926
6927static VALUE
6928vm_opt_gt(VALUE recv, VALUE obj)
6929{
6930 if (FIXNUM_2_P(recv, obj) &&
6931 BASIC_OP_UNREDEFINED_P(BOP_GT, INTEGER_REDEFINED_OP_FLAG)) {
6932 return RBOOL((SIGNED_VALUE)recv > (SIGNED_VALUE)obj);
6933 }
6934 else if (FLONUM_2_P(recv, obj) &&
6935 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6936 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6937 }
6938 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6939 return Qundef;
6940 }
6941 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6942 RBASIC_CLASS(obj) == rb_cFloat &&
6943 BASIC_OP_UNREDEFINED_P(BOP_GT, FLOAT_REDEFINED_OP_FLAG)) {
6944 return RBOOL(RFLOAT_VALUE(recv) > RFLOAT_VALUE(obj));
6945 }
6946 else {
6947 return Qundef;
6948 }
6949}
6950
6951static VALUE
6952vm_opt_ge(VALUE recv, VALUE obj)
6953{
6954 if (FIXNUM_2_P(recv, obj) &&
6955 BASIC_OP_UNREDEFINED_P(BOP_GE, INTEGER_REDEFINED_OP_FLAG)) {
6956 return RBOOL((SIGNED_VALUE)recv >= (SIGNED_VALUE)obj);
6957 }
6958 else if (FLONUM_2_P(recv, obj) &&
6959 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6960 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6961 }
6962 else if (SPECIAL_CONST_P(recv) || SPECIAL_CONST_P(obj)) {
6963 return Qundef;
6964 }
6965 else if (RBASIC_CLASS(recv) == rb_cFloat &&
6966 RBASIC_CLASS(obj) == rb_cFloat &&
6967 BASIC_OP_UNREDEFINED_P(BOP_GE, FLOAT_REDEFINED_OP_FLAG)) {
6968 return RBOOL(RFLOAT_VALUE(recv) >= RFLOAT_VALUE(obj));
6969 }
6970 else {
6971 return Qundef;
6972 }
6973}
6974
6975
6976static VALUE
6977vm_opt_ltlt(VALUE recv, VALUE obj)
6978{
6979 if (SPECIAL_CONST_P(recv)) {
6980 return Qundef;
6981 }
6982 else if (RBASIC_CLASS(recv) == rb_cString &&
6983 BASIC_OP_UNREDEFINED_P(BOP_LTLT, STRING_REDEFINED_OP_FLAG)) {
6984 if (LIKELY(RB_TYPE_P(obj, T_STRING))) {
6985 return rb_str_buf_append(recv, obj);
6986 }
6987 else {
6988 return rb_str_concat(recv, obj);
6989 }
6990 }
6991 else if (RBASIC_CLASS(recv) == rb_cArray &&
6992 BASIC_OP_UNREDEFINED_P(BOP_LTLT, ARRAY_REDEFINED_OP_FLAG)) {
6993 return rb_ary_push(recv, obj);
6994 }
6995 else {
6996 return Qundef;
6997 }
6998}
6999
7000static VALUE
7001vm_opt_and(VALUE recv, VALUE obj)
7002{
7003 // If recv and obj are both fixnums, then the bottom tag bit
7004 // will be 1 on both. 1 & 1 == 1, so the result value will also
7005 // be a fixnum. If either side is *not* a fixnum, then the tag bit
7006 // will be 0, and we return Qundef.
7007 VALUE ret = ((SIGNED_VALUE) recv) & ((SIGNED_VALUE) obj);
7008
7009 if (FIXNUM_P(ret) &&
7010 BASIC_OP_UNREDEFINED_P(BOP_AND, INTEGER_REDEFINED_OP_FLAG)) {
7011 return ret;
7012 }
7013 else {
7014 return Qundef;
7015 }
7016}
7017
7018static VALUE
7019vm_opt_or(VALUE recv, VALUE obj)
7020{
7021 if (FIXNUM_2_P(recv, obj) &&
7022 BASIC_OP_UNREDEFINED_P(BOP_OR, INTEGER_REDEFINED_OP_FLAG)) {
7023 return recv | obj;
7024 }
7025 else {
7026 return Qundef;
7027 }
7028}
7029
7030static VALUE
7031vm_opt_aref(VALUE recv, VALUE obj)
7032{
7033 if (SPECIAL_CONST_P(recv)) {
7034 if (FIXNUM_2_P(recv, obj) &&
7035 BASIC_OP_UNREDEFINED_P(BOP_AREF, INTEGER_REDEFINED_OP_FLAG)) {
7036 return rb_fix_aref(recv, obj);
7037 }
7038 return Qundef;
7039 }
7040 else if (RBASIC_CLASS(recv) == rb_cArray &&
7041 BASIC_OP_UNREDEFINED_P(BOP_AREF, ARRAY_REDEFINED_OP_FLAG)) {
7042 if (FIXNUM_P(obj)) {
7043 return rb_ary_entry_internal(recv, FIX2LONG(obj));
7044 }
7045 else {
7046 return rb_ary_aref1(recv, obj);
7047 }
7048 }
7049 else if (RBASIC_CLASS(recv) == rb_cHash &&
7050 BASIC_OP_UNREDEFINED_P(BOP_AREF, HASH_REDEFINED_OP_FLAG)) {
7051 return rb_hash_aref(recv, obj);
7052 }
7053 else {
7054 return Qundef;
7055 }
7056}
7057
7058static VALUE
7059vm_opt_aset(VALUE recv, VALUE obj, VALUE set)
7060{
7061 if (SPECIAL_CONST_P(recv)) {
7062 return Qundef;
7063 }
7064 else if (RBASIC_CLASS(recv) == rb_cArray &&
7065 BASIC_OP_UNREDEFINED_P(BOP_ASET, ARRAY_REDEFINED_OP_FLAG) &&
7066 FIXNUM_P(obj)) {
7067 rb_ary_store(recv, FIX2LONG(obj), set);
7068 return set;
7069 }
7070 else if (RBASIC_CLASS(recv) == rb_cHash &&
7071 BASIC_OP_UNREDEFINED_P(BOP_ASET, HASH_REDEFINED_OP_FLAG)) {
7072 rb_hash_aset(recv, obj, set);
7073 return set;
7074 }
7075 else {
7076 return Qundef;
7077 }
7078}
7079
7080static VALUE
7081vm_opt_length(VALUE recv, int bop)
7082{
7083 if (SPECIAL_CONST_P(recv)) {
7084 return Qundef;
7085 }
7086 else if (RBASIC_CLASS(recv) == rb_cString &&
7087 BASIC_OP_UNREDEFINED_P(bop, STRING_REDEFINED_OP_FLAG)) {
7088 if (bop == BOP_EMPTY_P) {
7089 return LONG2NUM(RSTRING_LEN(recv));
7090 }
7091 else {
7092 return rb_str_length(recv);
7093 }
7094 }
7095 else if (RBASIC_CLASS(recv) == rb_cArray &&
7096 BASIC_OP_UNREDEFINED_P(bop, ARRAY_REDEFINED_OP_FLAG)) {
7097 return LONG2NUM(RARRAY_LEN(recv));
7098 }
7099 else if (RBASIC_CLASS(recv) == rb_cHash &&
7100 BASIC_OP_UNREDEFINED_P(bop, HASH_REDEFINED_OP_FLAG)) {
7101 return INT2FIX(RHASH_SIZE(recv));
7102 }
7103 else {
7104 return Qundef;
7105 }
7106}
7107
7108static VALUE
7109vm_opt_empty_p(VALUE recv)
7110{
7111 switch (vm_opt_length(recv, BOP_EMPTY_P)) {
7112 case Qundef: return Qundef;
7113 case INT2FIX(0): return Qtrue;
7114 default: return Qfalse;
7115 }
7116}
7117
7118 VALUE rb_false(VALUE obj);
7119
7120static VALUE
7121vm_opt_nil_p(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv)
7122{
7123 if (NIL_P(recv) &&
7124 BASIC_OP_UNREDEFINED_P(BOP_NIL_P, NIL_REDEFINED_OP_FLAG)) {
7125 return Qtrue;
7126 }
7127 else if (vm_method_cfunc_is(iseq, cd, recv, rb_false)) {
7128 return Qfalse;
7129 }
7130 else {
7131 return Qundef;
7132 }
7133}
7134
7135static VALUE
7136fix_succ(VALUE x)
7137{
7138 switch (x) {
7139 case ~0UL:
7140 /* 0xFFFF_FFFF == INT2FIX(-1)
7141 * `-1.succ` is of course 0. */
7142 return INT2FIX(0);
7143 case RSHIFT(~0UL, 1):
7144 /* 0x7FFF_FFFF == LONG2FIX(0x3FFF_FFFF)
7145 * 0x3FFF_FFFF + 1 == 0x4000_0000, which is a Bignum. */
7146 return rb_uint2big(1UL << (SIZEOF_LONG * CHAR_BIT - 2));
7147 default:
7148 /* LONG2FIX(FIX2LONG(x)+FIX2LONG(y))
7149 * == ((lx*2+1)/2 + (ly*2+1)/2)*2+1
7150 * == lx*2 + ly*2 + 1
7151 * == (lx*2+1) + (ly*2+1) - 1
7152 * == x + y - 1
7153 *
7154 * Here, if we put y := INT2FIX(1):
7155 *
7156 * == x + INT2FIX(1) - 1
7157 * == x + 2 .
7158 */
7159 return x + 2;
7160 }
7161}
7162
7163static VALUE
7164vm_opt_succ(VALUE recv)
7165{
7166 if (FIXNUM_P(recv) &&
7167 BASIC_OP_UNREDEFINED_P(BOP_SUCC, INTEGER_REDEFINED_OP_FLAG)) {
7168 return fix_succ(recv);
7169 }
7170 else if (SPECIAL_CONST_P(recv)) {
7171 return Qundef;
7172 }
7173 else if (RBASIC_CLASS(recv) == rb_cString &&
7174 BASIC_OP_UNREDEFINED_P(BOP_SUCC, STRING_REDEFINED_OP_FLAG)) {
7175 return rb_str_succ(recv);
7176 }
7177 else {
7178 return Qundef;
7179 }
7180}
7181
7182static VALUE
7183vm_opt_not(const rb_iseq_t *iseq, CALL_DATA cd, VALUE recv)
7184{
7185 if (vm_method_cfunc_is(iseq, cd, recv, rb_obj_not)) {
7186 return RBOOL(!RTEST(recv));
7187 }
7188 else {
7189 return Qundef;
7190 }
7191}
7192
7193static VALUE
7194vm_opt_regexpmatch2(VALUE recv, VALUE obj)
7195{
7196 if (SPECIAL_CONST_P(recv)) {
7197 return Qundef;
7198 }
7199 else if (RBASIC_CLASS(recv) == rb_cString &&
7200 CLASS_OF(obj) == rb_cRegexp &&
7201 BASIC_OP_UNREDEFINED_P(BOP_MATCH, STRING_REDEFINED_OP_FLAG)) {
7202 return rb_reg_match(obj, recv);
7203 }
7204 else if (RBASIC_CLASS(recv) == rb_cRegexp &&
7205 BASIC_OP_UNREDEFINED_P(BOP_MATCH, REGEXP_REDEFINED_OP_FLAG)) {
7206 return rb_reg_match(recv, obj);
7207 }
7208 else {
7209 return Qundef;
7210 }
7211}
7212
7213 rb_event_flag_t rb_iseq_event_flags(const rb_iseq_t *iseq, size_t pos);
7214
7215NOINLINE(static void vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp));
7216
7217static inline void
7218vm_trace_hook(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const VALUE *pc,
7219 rb_event_flag_t pc_events, rb_event_flag_t target_event,
7220 rb_hook_list_t *global_hooks, rb_hook_list_t *local_hooks, VALUE val)
7221{
7222 rb_event_flag_t event = pc_events & target_event;
7223 VALUE self = GET_SELF();
7224
7225 VM_ASSERT(rb_popcount64((uint64_t)event) == 1);
7226
7227 if (local_hooks) local_hooks->running++; // make sure they don't get deleted while global hooks run
7228
7229 if (event & global_hooks->events) {
7230 /* increment PC because source line is calculated with PC-1 */
7231 reg_cfp->pc++;
7232 vm_dtrace(event, ec);
7233 rb_exec_event_hook_orig(ec, global_hooks, event, self, 0, 0, 0 , val, 0);
7234 reg_cfp->pc--;
7235 }
7236
7237 if (local_hooks) local_hooks->running--;
7238 if (local_hooks != NULL) {
7239 if (event & local_hooks->events) {
7240 /* increment PC because source line is calculated with PC-1 */
7241 reg_cfp->pc++;
7242 rb_exec_event_hook_orig(ec, local_hooks, event, self, 0, 0, 0 , val, 0);
7243 reg_cfp->pc--;
7244 }
7245 }
7246}
7247
7248#define VM_TRACE_HOOK(target_event, val) do { \
7249 if ((pc_events & (target_event)) & enabled_flags) { \
7250 vm_trace_hook(ec, reg_cfp, pc, pc_events, (target_event), global_hooks, local_hooks, (val)); \
7251 } \
7252} while (0)
7253
7254static VALUE
7255rescue_errinfo(rb_execution_context_t *ec, rb_control_frame_t *cfp)
7256{
7257 VM_ASSERT(VM_FRAME_RUBYFRAME_P(cfp));
7258 VM_ASSERT(ISEQ_BODY(cfp->iseq)->type == ISEQ_TYPE_RESCUE);
7259 return cfp->ep[VM_ENV_INDEX_LAST_LVAR];
7260}
7261
7262static void
7263vm_trace(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp)
7264{
7265 const VALUE *pc = reg_cfp->pc;
7266 rb_ractor_t *r = rb_ec_ractor_ptr(ec);
7267 rb_event_flag_t enabled_flags = r->pub.hooks.events & ISEQ_TRACE_EVENTS;
7268 rb_event_flag_t ractor_events = enabled_flags;
7269
7270 if (enabled_flags == 0 && rb_ractor_targeted_hooks_cnt(r) == 0) {
7271 return;
7272 }
7273 else {
7274 const rb_iseq_t *iseq = reg_cfp->iseq;
7275 size_t pos = pc - ISEQ_BODY(iseq)->iseq_encoded;
7276 rb_event_flag_t pc_events = rb_iseq_event_flags(iseq, pos);
7277 unsigned int local_hooks_cnt = iseq->aux.exec.local_hooks_cnt;
7278 rb_hook_list_t *local_hooks = NULL;
7279 if (RB_UNLIKELY(local_hooks_cnt > 0)) {
7280 st_data_t val;
7281 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)iseq, &val)) {
7282 local_hooks = (rb_hook_list_t*)val;
7283 }
7284 }
7285 rb_event_flag_t iseq_local_events = local_hooks != NULL ? local_hooks->events : 0;
7286
7287 rb_hook_list_t *bmethod_local_hooks = NULL;
7288 rb_event_flag_t bmethod_local_events = 0;
7289 const bool bmethod_frame = VM_FRAME_BMETHOD_P(reg_cfp);
7290 enabled_flags |= iseq_local_events;
7291
7292 VM_ASSERT((iseq_local_events & ~ISEQ_TRACE_EVENTS) == 0);
7293
7294 if (bmethod_frame) {
7295 const rb_callable_method_entry_t *me = rb_vm_frame_method_entry(reg_cfp);
7296 VM_ASSERT(me->def->type == VM_METHOD_TYPE_BMETHOD);
7297 unsigned int bmethod_hooks_cnt = me->def->body.bmethod.local_hooks_cnt;
7298 if (RB_UNLIKELY(bmethod_hooks_cnt > 0)) {
7299 st_data_t val;
7300 if (st_lookup(rb_ractor_targeted_hooks(r), (st_data_t)me->def, &val)) {
7301 bmethod_local_hooks = (rb_hook_list_t*)val;
7302 }
7303 if (bmethod_local_hooks) {
7304 bmethod_local_events = bmethod_local_hooks->events;
7305 }
7306 }
7307 }
7308
7309 if ((pc_events & enabled_flags) == 0 && !bmethod_frame) {
7310#if 0
7311 /* disable trace */
7312 /* TODO: incomplete */
7313 rb_iseq_trace_set(iseq, vm_event_flags & ISEQ_TRACE_EVENTS);
7314#else
7315 /* do not disable trace because of performance problem
7316 * (re-enable overhead)
7317 */
7318#endif
7319 return;
7320 }
7321 else if (ec->trace_arg != NULL) {
7322 /* already tracing */
7323 return;
7324 }
7325 else {
7326 rb_hook_list_t *global_hooks = rb_ec_ractor_hooks(ec);
7327 /* Note, not considering iseq local events here since the same
7328 * iseq could be used in multiple bmethods. */
7329 rb_event_flag_t bmethod_events = ractor_events | bmethod_local_events;
7330
7331 if (0) {
7332 ruby_debug_printf("vm_trace>>%4d (%4x) - %s:%d %s\n",
7333 (int)pos,
7334 (int)pc_events,
7335 RSTRING_PTR(rb_iseq_path(iseq)),
7336 (int)rb_iseq_line_no(iseq, pos),
7337 RSTRING_PTR(rb_iseq_label(iseq)));
7338 }
7339 VM_ASSERT(reg_cfp->pc == pc);
7340 VM_ASSERT(pc_events != 0);
7341
7342 /* check traces */
7343 if ((pc_events & RUBY_EVENT_B_CALL) && bmethod_frame && (bmethod_events & RUBY_EVENT_CALL)) {
7344 /* b_call instruction running as a method. Fire call event. */
7345 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_CALL, RUBY_EVENT_CALL, global_hooks, bmethod_local_hooks, Qundef);
7346 }
7347 VM_TRACE_HOOK(RUBY_EVENT_CLASS | RUBY_EVENT_CALL | RUBY_EVENT_B_CALL, Qundef);
7348 VM_TRACE_HOOK(RUBY_EVENT_RESCUE, rescue_errinfo(ec, reg_cfp));
7349 VM_TRACE_HOOK(RUBY_EVENT_LINE, Qundef);
7350 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_LINE, Qundef);
7351 VM_TRACE_HOOK(RUBY_EVENT_COVERAGE_BRANCH, Qundef);
7352 VM_TRACE_HOOK(RUBY_EVENT_END | RUBY_EVENT_RETURN | RUBY_EVENT_B_RETURN, TOPN(0));
7353 if ((pc_events & RUBY_EVENT_B_RETURN) && bmethod_frame && (bmethod_events & RUBY_EVENT_RETURN)) {
7354 /* b_return instruction running as a method. Fire return event. */
7355 vm_trace_hook(ec, reg_cfp, pc, RUBY_EVENT_RETURN, RUBY_EVENT_RETURN, global_hooks, bmethod_local_hooks, TOPN(0));
7356 }
7357 }
7358 }
7359}
7360#undef VM_TRACE_HOOK
7361
7362#if VM_CHECK_MODE > 0
7363NORETURN( NOINLINE( COLDFUNC
7364void rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)));
7365
7366void
7367Init_vm_stack_canary(void)
7368{
7369 /* This has to be called _after_ our PRNG is properly set up. */
7370 int n = ruby_fill_random_bytes(&vm_stack_canary, sizeof vm_stack_canary, false);
7371 vm_stack_canary |= 0x01; // valid VALUE (Fixnum)
7372
7373 vm_stack_canary_was_born = true;
7374 VM_ASSERT(n == 0);
7375}
7376
7377void
7378rb_vm_canary_is_found_dead(enum ruby_vminsn_type i, VALUE c)
7379{
7380 /* Because a method has already been called, why not call
7381 * another one. */
7382 const char *insn = rb_insns_name(i);
7383 VALUE inspection = rb_inspect(c);
7384 const char *str = StringValueCStr(inspection);
7385
7386 rb_bug("dead canary found at %s: %s", insn, str);
7387}
7388
7389#else
7390void Init_vm_stack_canary(void) { /* nothing to do */ }
7391#endif
7392
7393
7394/* a part of the following code is generated by this ruby script:
7395
739616.times{|i|
7397 typedef_args = (0...i).map{|j| "VALUE v#{j+1}"}.join(", ")
7398 typedef_args.prepend(", ") if i != 0
7399 call_args = (0...i).map{|j| "argv[#{j}]"}.join(", ")
7400 call_args.prepend(", ") if i != 0
7401 puts %Q{
7402static VALUE
7403builtin_invoker#{i}(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7404{
7405 typedef VALUE (*rb_invoke_funcptr#{i}_t)(rb_execution_context_t *ec, VALUE self#{typedef_args});
7406 return (*(rb_invoke_funcptr#{i}_t)funcptr)(ec, self#{call_args});
7407}}
7408}
7409
7410puts
7411puts "static VALUE (* const cfunc_invokers[])(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr) = {"
741216.times{|i|
7413 puts " builtin_invoker#{i},"
7414}
7415puts "};"
7416*/
7417
7418static VALUE
7419builtin_invoker0(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7420{
7421 typedef VALUE (*rb_invoke_funcptr0_t)(rb_execution_context_t *ec, VALUE self);
7422 return (*(rb_invoke_funcptr0_t)funcptr)(ec, self);
7423}
7424
7425static VALUE
7426builtin_invoker1(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7427{
7428 typedef VALUE (*rb_invoke_funcptr1_t)(rb_execution_context_t *ec, VALUE self, VALUE v1);
7429 return (*(rb_invoke_funcptr1_t)funcptr)(ec, self, argv[0]);
7430}
7431
7432static VALUE
7433builtin_invoker2(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7434{
7435 typedef VALUE (*rb_invoke_funcptr2_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2);
7436 return (*(rb_invoke_funcptr2_t)funcptr)(ec, self, argv[0], argv[1]);
7437}
7438
7439static VALUE
7440builtin_invoker3(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7441{
7442 typedef VALUE (*rb_invoke_funcptr3_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3);
7443 return (*(rb_invoke_funcptr3_t)funcptr)(ec, self, argv[0], argv[1], argv[2]);
7444}
7445
7446static VALUE
7447builtin_invoker4(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7448{
7449 typedef VALUE (*rb_invoke_funcptr4_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4);
7450 return (*(rb_invoke_funcptr4_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3]);
7451}
7452
7453static VALUE
7454builtin_invoker5(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7455{
7456 typedef VALUE (*rb_invoke_funcptr5_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5);
7457 return (*(rb_invoke_funcptr5_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4]);
7458}
7459
7460static VALUE
7461builtin_invoker6(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7462{
7463 typedef VALUE (*rb_invoke_funcptr6_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6);
7464 return (*(rb_invoke_funcptr6_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5]);
7465}
7466
7467static VALUE
7468builtin_invoker7(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7469{
7470 typedef VALUE (*rb_invoke_funcptr7_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7);
7471 return (*(rb_invoke_funcptr7_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6]);
7472}
7473
7474static VALUE
7475builtin_invoker8(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7476{
7477 typedef VALUE (*rb_invoke_funcptr8_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8);
7478 return (*(rb_invoke_funcptr8_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7]);
7479}
7480
7481static VALUE
7482builtin_invoker9(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7483{
7484 typedef VALUE (*rb_invoke_funcptr9_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9);
7485 return (*(rb_invoke_funcptr9_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8]);
7486}
7487
7488static VALUE
7489builtin_invoker10(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7490{
7491 typedef VALUE (*rb_invoke_funcptr10_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10);
7492 return (*(rb_invoke_funcptr10_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9]);
7493}
7494
7495static VALUE
7496builtin_invoker11(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7497{
7498 typedef VALUE (*rb_invoke_funcptr11_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11);
7499 return (*(rb_invoke_funcptr11_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10]);
7500}
7501
7502static VALUE
7503builtin_invoker12(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7504{
7505 typedef VALUE (*rb_invoke_funcptr12_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12);
7506 return (*(rb_invoke_funcptr12_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11]);
7507}
7508
7509static VALUE
7510builtin_invoker13(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7511{
7512 typedef VALUE (*rb_invoke_funcptr13_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13);
7513 return (*(rb_invoke_funcptr13_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12]);
7514}
7515
7516static VALUE
7517builtin_invoker14(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7518{
7519 typedef VALUE (*rb_invoke_funcptr14_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14);
7520 return (*(rb_invoke_funcptr14_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13]);
7521}
7522
7523static VALUE
7524builtin_invoker15(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr)
7525{
7526 typedef VALUE (*rb_invoke_funcptr15_t)(rb_execution_context_t *ec, VALUE self, VALUE v1, VALUE v2, VALUE v3, VALUE v4, VALUE v5, VALUE v6, VALUE v7, VALUE v8, VALUE v9, VALUE v10, VALUE v11, VALUE v12, VALUE v13, VALUE v14, VALUE v15);
7527 return (*(rb_invoke_funcptr15_t)funcptr)(ec, self, argv[0], argv[1], argv[2], argv[3], argv[4], argv[5], argv[6], argv[7], argv[8], argv[9], argv[10], argv[11], argv[12], argv[13], argv[14]);
7528}
7529
7530typedef VALUE (*builtin_invoker)(rb_execution_context_t *ec, VALUE self, const VALUE *argv, rb_insn_func_t funcptr);
7531
7532static builtin_invoker
7533lookup_builtin_invoker(int argc)
7534{
7535 static const builtin_invoker invokers[] = {
7536 builtin_invoker0,
7537 builtin_invoker1,
7538 builtin_invoker2,
7539 builtin_invoker3,
7540 builtin_invoker4,
7541 builtin_invoker5,
7542 builtin_invoker6,
7543 builtin_invoker7,
7544 builtin_invoker8,
7545 builtin_invoker9,
7546 builtin_invoker10,
7547 builtin_invoker11,
7548 builtin_invoker12,
7549 builtin_invoker13,
7550 builtin_invoker14,
7551 builtin_invoker15,
7552 };
7553
7554 return invokers[argc];
7555}
7556
7557static inline VALUE
7558invoke_bf(rb_execution_context_t *ec, rb_control_frame_t *reg_cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7559{
7560 const bool canary_p = ISEQ_BODY(reg_cfp->iseq)->builtin_attrs & BUILTIN_ATTR_LEAF; // Verify an assumption of `Primitive.attr! :leaf`
7561 SETUP_CANARY(canary_p);
7562 rb_insn_func_t func_ptr = (rb_insn_func_t)(uintptr_t)bf->func_ptr;
7563 VALUE ret = (*lookup_builtin_invoker(bf->argc))(ec, reg_cfp->self, argv, func_ptr);
7564 CHECK_CANARY(canary_p, BIN(invokebuiltin));
7565 return ret;
7566}
7567
7568static VALUE
7569vm_invoke_builtin(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function* bf, const VALUE *argv)
7570{
7571 return invoke_bf(ec, cfp, bf, argv);
7572}
7573
7574static VALUE
7575vm_invoke_builtin_delegate(rb_execution_context_t *ec, rb_control_frame_t *cfp, const struct rb_builtin_function *bf, unsigned int start_index)
7576{
7577 if (0) { // debug print
7578 fputs("vm_invoke_builtin_delegate: passing -> ", stderr);
7579 for (int i=0; i<bf->argc; i++) {
7580 ruby_debug_printf(":%s ", rb_id2name(ISEQ_BODY(cfp->iseq)->local_table[i+start_index]));
7581 }
7582 ruby_debug_printf("\n" "%s %s(%d):%p\n", RUBY_FUNCTION_NAME_STRING, bf->name, bf->argc,
7583 (void *)(uintptr_t)bf->func_ptr);
7584 }
7585
7586 if (bf->argc == 0) {
7587 return invoke_bf(ec, cfp, bf, NULL);
7588 }
7589 else {
7590 const VALUE *argv = cfp->ep - ISEQ_BODY(cfp->iseq)->local_table_size - VM_ENV_DATA_SIZE + 1 + start_index;
7591 return invoke_bf(ec, cfp, bf, argv);
7592 }
7593}
7594
7595// for __builtin_inline!()
7596
7597 VALUE
7598rb_vm_lvar_exposed(rb_execution_context_t *ec, int index)
7599{
7600 const rb_control_frame_t *cfp = ec->cfp;
7601 return cfp->ep[index];
7602}
RUBY_ASSERT
#define RUBY_ASSERT(...)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
Definition assert.h:219
RUBY_EVENT_END
#define RUBY_EVENT_END
Encountered an end of a class clause.
Definition event.h:40
RUBY_EVENT_C_CALL
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
Definition event.h:43
RUBY_EVENT_B_RETURN
#define RUBY_EVENT_B_RETURN
Encountered a next statement.
Definition event.h:56
RUBY_EVENT_CLASS
#define RUBY_EVENT_CLASS
Encountered a new class.
Definition event.h:39
RUBY_EVENT_LINE
#define RUBY_EVENT_LINE
Encountered a new line.
Definition event.h:38
RUBY_EVENT_RETURN
#define RUBY_EVENT_RETURN
Encountered a return statement.
Definition event.h:42
RUBY_EVENT_C_RETURN
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
Definition event.h:44
RUBY_EVENT_B_CALL
#define RUBY_EVENT_B_CALL
Encountered an yield statement.
Definition event.h:55
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
RUBY_EVENT_CALL
#define RUBY_EVENT_CALL
A method, written in Ruby, is called.
Definition event.h:41
RUBY_EVENT_RESCUE
#define RUBY_EVENT_RESCUE
Encountered a rescue statement.
Definition event.h:61
RB_FL_ABLE
static bool RB_FL_ABLE(VALUE obj)
Checks if the object is flaggable.
Definition fl_type.h:440
rb_singleton_class
VALUE rb_singleton_class(VALUE obj)
Finds or creates the singleton class of the passed object.
Definition class.c:2799
rb_module_new
VALUE rb_module_new(void)
Creates a new, anonymous module.
Definition class.c:1571
rb_class_inherited
VALUE rb_class_inherited(VALUE super, VALUE klass)
Calls Class::inherited.
Definition class.c:1468
rb_define_class_id
VALUE rb_define_class_id(ID id, VALUE super)
This is a very badly designed API that creates an anonymous class.
Definition class.c:1447
TYPE
#define TYPE(_)
Old name of rb_type.
Definition value_type.h:108
RFLOAT_VALUE
#define RFLOAT_VALUE
Old name of rb_float_value.
Definition double.h:28
T_STRING
#define T_STRING
Old name of RUBY_T_STRING.
Definition value_type.h:78
Qundef
#define Qundef
Old name of RUBY_Qundef.
Definition special_consts.h:62
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
T_NIL
#define T_NIL
Old name of RUBY_T_NIL.
Definition value_type.h:72
T_FLOAT
#define T_FLOAT
Old name of RUBY_T_FLOAT.
Definition value_type.h:64
T_IMEMO
#define T_IMEMO
Old name of RUBY_T_IMEMO.
Definition value_type.h:67
ID2SYM
#define ID2SYM
Old name of RB_ID2SYM.
Definition symbol.h:44
T_BIGNUM
#define T_BIGNUM
Old name of RUBY_T_BIGNUM.
Definition value_type.h:57
SPECIAL_CONST_P
#define SPECIAL_CONST_P
Old name of RB_SPECIAL_CONST_P.
Definition special_consts.h:56
T_STRUCT
#define T_STRUCT
Old name of RUBY_T_STRUCT.
Definition value_type.h:79
T_FIXNUM
#define T_FIXNUM
Old name of RUBY_T_FIXNUM.
Definition value_type.h:63
SYM2ID
#define SYM2ID
Old name of RB_SYM2ID.
Definition symbol.h:45
CLASS_OF
#define CLASS_OF
Old name of rb_class_of.
Definition globals.h:205
rb_ary_new4
#define rb_ary_new4
Old name of rb_ary_new_from_values.
Definition array.h:659
FIXABLE
#define FIXABLE
Old name of RB_FIXABLE.
Definition fixnum.h:25
LONG2FIX
#define LONG2FIX
Old name of RB_INT2FIX.
Definition long.h:49
FIX2INT
#define FIX2INT
Old name of RB_FIX2INT.
Definition int.h:41
T_MODULE
#define T_MODULE
Old name of RUBY_T_MODULE.
Definition value_type.h:70
STATIC_SYM_P
#define STATIC_SYM_P
Old name of RB_STATIC_SYM_P.
Definition special_consts.h:57
ASSUME
#define ASSUME
Old name of RBIMPL_ASSUME.
Definition assume.h:27
FIX2ULONG
#define FIX2ULONG
Old name of RB_FIX2ULONG.
Definition long.h:47
T_TRUE
#define T_TRUE
Old name of RUBY_T_TRUE.
Definition value_type.h:81
T_ICLASS
#define T_ICLASS
Old name of RUBY_T_ICLASS.
Definition value_type.h:66
T_HASH
#define T_HASH
Old name of RUBY_T_HASH.
Definition value_type.h:65
FL_TEST_RAW
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
Definition fl_type.h:131
rb_ary_new3
#define rb_ary_new3
Old name of rb_ary_new_from_args.
Definition array.h:658
LONG2NUM
#define LONG2NUM
Old name of RB_LONG2NUM.
Definition long.h:50
rb_exc_new3
#define rb_exc_new3
Old name of rb_exc_new_str.
Definition error.h:38
T_FALSE
#define T_FALSE
Old name of RUBY_T_FALSE.
Definition value_type.h:61
Qtrue
#define Qtrue
Old name of RUBY_Qtrue.
Definition special_consts.h:61
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
Qfalse
#define Qfalse
Old name of RUBY_Qfalse.
Definition special_consts.h:59
FIX2LONG
#define FIX2LONG
Old name of RB_FIX2LONG.
Definition long.h:46
T_ARRAY
#define T_ARRAY
Old name of RUBY_T_ARRAY.
Definition value_type.h:56
T_OBJECT
#define T_OBJECT
Old name of RUBY_T_OBJECT.
Definition value_type.h:75
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
T_SYMBOL
#define T_SYMBOL
Old name of RUBY_T_SYMBOL.
Definition value_type.h:80
DBL2NUM
#define DBL2NUM
Old name of rb_float_new.
Definition double.h:29
T_CLASS
#define T_CLASS
Old name of RUBY_T_CLASS.
Definition value_type.h:58
BUILTIN_TYPE
#define BUILTIN_TYPE
Old name of RB_BUILTIN_TYPE.
Definition value_type.h:85
FL_TEST
#define FL_TEST
Old name of RB_FL_TEST.
Definition fl_type.h:130
FIXNUM_P
#define FIXNUM_P
Old name of RB_FIXNUM_P.
Definition special_consts.h:53
FL_USHIFT
#define FL_USHIFT
Old name of RUBY_FL_USHIFT.
Definition fl_type.h:68
FL_SET_RAW
#define FL_SET_RAW
Old name of RB_FL_SET_RAW.
Definition fl_type.h:129
SYMBOL_P
#define SYMBOL_P
Old name of RB_SYMBOL_P.
Definition value_type.h:88
rb_notimplement
void rb_notimplement(void)
Definition error.c:3840
rb_exc_raise
void rb_exc_raise(VALUE mesg)
Raises an exception in the current thread.
Definition eval.c:664
ruby_verbose
#define ruby_verbose
This variable controls whether the interpreter is in debug mode.
Definition error.h:475
rb_eTypeError
VALUE rb_eTypeError
TypeError exception.
Definition error.c:1431
rb_eFatal
VALUE rb_eFatal
fatal exception.
Definition error.c:1427
rb_eNoMethodError
VALUE rb_eNoMethodError
NoMethodError exception.
Definition error.c:1439
rb_exc_fatal
void rb_exc_fatal(VALUE mesg)
Raises a fatal error in the current thread.
Definition eval.c:677
rb_eRuntimeError
VALUE rb_eRuntimeError
RuntimeError exception.
Definition error.c:1429
rb_warn
void rb_warn(const char *fmt,...)
Identical to rb_warning(), except it reports unless $VERBOSE is nil.
Definition error.c:466
rb_error_frozen_object
void rb_error_frozen_object(VALUE frozen_obj)
Identical to rb_error_frozen(), except it takes arbitrary Ruby object instead of C's string.
Definition error.c:4161
rb_exc_new_str
VALUE rb_exc_new_str(VALUE etype, VALUE str)
Identical to rb_exc_new_cstr(), except it takes a Ruby's string instead of C's.
Definition error.c:1482
RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
@ RB_WARN_CATEGORY_STRICT_UNUSED_BLOCK
Warning is for checking unused block strictly.
Definition error.h:57
rb_cClass
VALUE rb_cClass
Class class.
Definition object.c:63
rb_cArray
VALUE rb_cArray
Array class.
rb_obj_alloc
VALUE rb_obj_alloc(VALUE klass)
Allocates an instance of the given class.
Definition object.c:2208
rb_cRegexp
VALUE rb_cRegexp
Regexp class.
Definition re.c:2657
rb_obj_frozen_p
VALUE rb_obj_frozen_p(VALUE obj)
Just calls RB_OBJ_FROZEN() inside.
Definition object.c:1354
rb_cHash
VALUE rb_cHash
Hash class.
Definition hash.c:109
rb_obj_class
VALUE rb_obj_class(VALUE obj)
Queries the class of an object.
Definition object.c:264
rb_inspect
VALUE rb_inspect(VALUE obj)
Generates a human-readable textual representation of the given object.
Definition object.c:686
rb_cBasicObject
VALUE rb_cBasicObject
BasicObject class.
Definition object.c:59
rb_cModule
VALUE rb_cModule
Module class.
Definition object.c:62
rb_class_real
VALUE rb_class_real(VALUE klass)
Finds a "real" class.
Definition object.c:255
rb_obj_is_kind_of
VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass)
Queries if the given object is an instance (of possibly descendants) of the given class.
Definition object.c:923
rb_cFloat
VALUE rb_cFloat
Float class.
Definition numeric.c:199
rb_cProc
VALUE rb_cProc
Proc class.
Definition proc.c:45
rb_cString
VALUE rb_cString
String class.
Definition string.c:84
RB_OBJ_WRITTEN
#define RB_OBJ_WRITTEN(old, oldv, young)
Identical to RB_OBJ_WRITE(), except it doesn't write any values, but only a WB declaration.
Definition gc.h:615
RB_OBJ_WRITE
#define RB_OBJ_WRITE(old, slot, young)
Declaration of a "back" pointer.
Definition gc.h:603
rb_during_gc
int rb_during_gc(void)
Queries if the GC is busy.
Definition gc.c:4276
rb_ary_concat
VALUE rb_ary_concat(VALUE lhs, VALUE rhs)
Destructively appends the contents of latter into the end of former.
rb_ary_shift
VALUE rb_ary_shift(VALUE ary)
Destructively deletes an element from the beginning of the passed array and returns what was deleted.
rb_ary_resurrect
VALUE rb_ary_resurrect(VALUE ary)
I guess there is no use case of this function in extension libraries, but this is a routine identical...
rb_ary_dup
VALUE rb_ary_dup(VALUE ary)
Duplicates an array.
rb_ary_includes
VALUE rb_ary_includes(VALUE ary, VALUE elem)
Queries if the passed array has the passed entry.
rb_ary_plus
VALUE rb_ary_plus(VALUE lhs, VALUE rhs)
Creates a new array, concatenating the former to the latter.
rb_ary_cat
VALUE rb_ary_cat(VALUE ary, const VALUE *train, long len)
Destructively appends multiple elements at the end of the array.
rb_check_array_type
VALUE rb_check_array_type(VALUE obj)
Try converting an object to its array representation using its to_ary method, if any.
rb_ary_new
VALUE rb_ary_new(void)
Allocates a new, empty array.
rb_ary_pop
VALUE rb_ary_pop(VALUE ary)
Destructively deletes an element from the end of the passed array and returns what was deleted.
rb_ary_hidden_new
VALUE rb_ary_hidden_new(long capa)
Allocates a hidden (no class) empty array.
rb_ary_push
VALUE rb_ary_push(VALUE ary, VALUE elem)
Special case of rb_ary_cat() that it adds only one element.
rb_ary_entry
VALUE rb_ary_entry(VALUE ary, long off)
Queries an element of an array.
rb_ary_store
void rb_ary_store(VALUE ary, long key, VALUE val)
Destructively stores the passed value to the passed array's passed index.
UNLIMITED_ARGUMENTS
#define UNLIMITED_ARGUMENTS
This macro is used in conjunction with rb_check_arity().
Definition error.h:35
rb_check_arity
static int rb_check_arity(int argc, int min, int max)
Ensures that the passed integer is in the passed range.
Definition error.h:284
rb_proc_call_with_block
VALUE rb_proc_call_with_block(VALUE recv, int argc, const VALUE *argv, VALUE proc)
Identical to rb_proc_call(), except you can additionally pass another proc object,...
Definition proc.c:1169
rb_reg_last_match
VALUE rb_reg_last_match(VALUE md)
This just returns the argument, stringified.
Definition re.c:1947
rb_reg_match
VALUE rb_reg_match(VALUE re, VALUE str)
This is the match operator.
Definition re.c:3720
rb_reg_nth_match
VALUE rb_reg_nth_match(int n, VALUE md)
Queries the nth captured substring.
Definition re.c:1922
rb_reg_match_post
VALUE rb_reg_match_post(VALUE md)
The portion of the original string after the given match.
Definition re.c:2004
rb_reg_nth_defined
VALUE rb_reg_nth_defined(int n, VALUE md)
Identical to rb_reg_nth_match(), except it just returns Boolean.
Definition re.c:1905
rb_reg_match_pre
VALUE rb_reg_match_pre(VALUE md)
The portion of the original string before the given match.
Definition re.c:1971
rb_reg_match_last
VALUE rb_reg_match_last(VALUE md)
The portion of the original string that captured at the very last.
Definition re.c:2037
rb_str_append
VALUE rb_str_append(VALUE dst, VALUE src)
Identical to rb_str_buf_append(), except it converts the right hand side before concatenating.
Definition string.c:3799
rb_str_succ
VALUE rb_str_succ(VALUE orig)
Searches for the "successor" of a string.
Definition string.c:5341
rb_str_buf_append
VALUE rb_str_buf_append(VALUE dst, VALUE src)
Identical to rb_str_cat_cstr(), except it takes Ruby's string instead of C's.
Definition string.c:3765
rb_str_concat
VALUE rb_str_concat(VALUE dst, VALUE src)
Identical to rb_str_append(), except it also accepts an integer as a codepoint.
Definition string.c:4036
rb_str_cat_cstr
#define rb_str_cat_cstr(buf, str)
Identical to rb_str_cat(), except it assumes the passed pointer is a pointer to a C string.
Definition string.h:1657
rb_str_length
VALUE rb_str_length(VALUE)
Identical to rb_str_strlen(), except it returns the value in rb_cInteger.
Definition string.c:2437
rb_str_intern
VALUE rb_str_intern(VALUE str)
Identical to rb_to_symbol(), except it assumes the receiver being an instance of RString.
Definition symbol.c:937
rb_thread_schedule
void rb_thread_schedule(void)
Tries to switch to another thread.
Definition thread.c:1513
rb_const_get
VALUE rb_const_get(VALUE space, ID name)
Identical to rb_const_defined(), except it returns the actual defined value.
Definition variable.c:3505
rb_ivar_set
VALUE rb_ivar_set(VALUE obj, ID name, VALUE val)
Identical to rb_iv_set(), except it accepts the name as an ID instead of a C string.
Definition variable.c:2030
rb_cvar_set
void rb_cvar_set(VALUE klass, ID name, VALUE val)
Assigns a value to a class variable.
Definition variable.c:4305
rb_cvar_find
VALUE rb_cvar_find(VALUE klass, ID name, VALUE *front)
Identical to rb_cvar_get(), except it takes additional "front" pointer.
Definition variable.c:4368
rb_ivar_get
VALUE rb_ivar_get(VALUE obj, ID name)
Identical to rb_iv_get(), except it accepts the name as an ID instead of a C string.
Definition variable.c:1505
rb_const_set
void rb_const_set(VALUE space, ID name, VALUE val)
Names a constant.
Definition variable.c:3983
rb_autoload_load
VALUE rb_autoload_load(VALUE space, ID name)
Kicks the autoload procedure as if it was "touched".
Definition variable.c:3335
rb_mod_name
VALUE rb_mod_name(VALUE mod)
Queries the name of a module.
Definition variable.c:136
rb_const_get_at
VALUE rb_const_get_at(VALUE space, ID name)
Identical to rb_const_defined_at(), except it returns the actual defined value.
Definition variable.c:3511
rb_set_class_path_string
void rb_set_class_path_string(VALUE klass, VALUE space, VALUE name)
Identical to rb_set_class_path(), except it accepts the name as Ruby's string instead of C's.
Definition variable.c:423
rb_ivar_defined
VALUE rb_ivar_defined(VALUE obj, ID name)
Queries if the instance variable is defined at the object.
Definition variable.c:2109
rb_const_defined_at
int rb_const_defined_at(VALUE space, ID name)
Identical to rb_const_defined(), except it doesn't look for parent classes.
Definition variable.c:3843
rb_cvar_defined
VALUE rb_cvar_defined(VALUE klass, ID name)
Queries if the given class has the given class variable.
Definition variable.c:4390
rb_class_path
VALUE rb_class_path(VALUE mod)
Identical to rb_mod_name(), except it returns #<Class: ...> style inspection for anonymous modules.
Definition variable.c:380
rb_const_defined
int rb_const_defined(VALUE space, ID name)
Queries if the constant is defined at the namespace.
Definition variable.c:3837
rb_check_funcall
VALUE rb_check_funcall(VALUE recv, ID mid, int argc, const VALUE *argv)
Identical to rb_funcallv(), except it returns RUBY_Qundef instead of raising rb_eNoMethodError.
Definition vm_eval.c:686
rb_get_alloc_func
rb_alloc_func_t rb_get_alloc_func(VALUE klass)
Queries the allocator function of a class.
Definition vm_method.c:1717
rb_define_alloc_func
void rb_define_alloc_func(VALUE klass, rb_alloc_func_t func)
Sets the allocator function of a class.
rb_method_boundp
int rb_method_boundp(VALUE klass, ID id, int ex)
Queries if the klass has this method.
Definition vm_method.c:2299
rb_check_id
ID rb_check_id(volatile VALUE *namep)
Detects if the given name is already interned or not.
Definition symbol.c:1133
rb_sym2str
VALUE rb_sym2str(VALUE symbol)
Obtain a frozen string representation of a symbol (not including the leading colon).
Definition symbol.c:993
off
int off
Offset inside of ptr.
Definition io.h:5
len
int len
Length of the buffer.
Definition io.h:8
rb_ractor_shareable_p
static bool rb_ractor_shareable_p(VALUE obj)
Queries if multiple Ractors can share the passed object or not.
Definition ractor.h:249
RB_OBJ_SHAREABLE_P
#define RB_OBJ_SHAREABLE_P(obj)
Queries if the passed object has previously classified as shareable or not.
Definition ractor.h:235
MEMCPY
#define MEMCPY(p1, p2, type, n)
Handy macro to call memcpy.
Definition memory.h:372
ALLOCA_N
#define ALLOCA_N(type, n)
Definition memory.h:292
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
MEMMOVE
#define MEMMOVE(p1, p2, type, n)
Handy macro to call memmove.
Definition memory.h:384
ruby::backward::cxxanyargs::type
VALUE type(ANYARGS)
ANYARGS-ed function type.
Definition cxxanyargs.hpp:56
ruby::backward::cxxanyargs::rb_ensure
VALUE rb_ensure(type *q, VALUE w, type *e, VALUE r)
An equivalent of ensure clause.
Definition cxxanyargs.hpp:310
RARRAY_LEN
#define RARRAY_LEN
Just another name of rb_array_len.
Definition rarray.h:51
RARRAY_LENINT
static int RARRAY_LENINT(VALUE ary)
Identical to rb_array_len(), except it differs for the return type.
Definition rarray.h:281
RARRAY_PTR
static VALUE * RARRAY_PTR(VALUE ary)
Wild use of a C pointer.
Definition rarray.h:366
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RARRAY_CONST_PTR
#define RARRAY_CONST_PTR
Just another name of rb_array_const_ptr.
Definition rarray.h:52
RBASIC_CLASS
static VALUE RBASIC_CLASS(VALUE obj)
Queries the class of an object.
Definition rbasic.h:166
RBASIC
#define RBASIC(obj)
Convenient casting macro.
Definition rbasic.h:40
RCLASS_SUPER
#define RCLASS_SUPER
Just another name of rb_class_get_superclass.
Definition rclass.h:44
RHASH_SIZE
#define RHASH_SIZE(h)
Queries the size of the hash.
Definition rhash.h:69
RHASH_EMPTY_P
#define RHASH_EMPTY_P(h)
Checks if the hash is empty.
Definition rhash.h:79
ROBJECT_FIELDS
static VALUE * ROBJECT_FIELDS(VALUE obj)
Queries the instance variables.
Definition robject.h:128
StringValueCStr
#define StringValueCStr(v)
Identical to StringValuePtr, except it additionally checks for the contents for viability as a C stri...
Definition rstring.h:89
RB_PASS_KEYWORDS
#define RB_PASS_KEYWORDS
Pass keywords, final argument should be a hash of keywords.
Definition scan_args.h:72
RB_NO_KEYWORDS
#define RB_NO_KEYWORDS
Do not pass keywords.
Definition scan_args.h:69
RB_SPECIAL_CONST_P
static bool RB_SPECIAL_CONST_P(VALUE obj)
Checks if the given object is of enum ruby_special_consts.
Definition special_consts.h:327
RTEST
#define RTEST
This is an old name of RB_TEST.
Definition special_consts.h:51
ANYARGS
#define ANYARGS
Functions declared using this macro take arbitrary arguments, including void.
Definition stdarg.h:64
RArray
Ruby's array.
Definition rarray.h:128
RArray::ary
const VALUE ary[1]
Embedded elements.
Definition rarray.h:188
RHash
Definition hash.h:53
iseq_catch_table_entry
Definition iseq.h:288
iseq_catch_table
Definition iseq.h:299
iseq_inline_constant_cache_entry
Definition vm_core.h:261
iseq_inline_constant_cache::segments
const ID * segments
A null-terminated list of ids, used to represent a constant's path idNULL is used to represent the ::...
Definition vm_core.h:285
rb_block
Definition vm_core.h:903
rb_builtin_function
Definition builtin.h:6
rb_call_data
Definition vm_callinfo.h:604
rb_callable_method_entry_struct
Definition method.h:63
rb_callcache
Definition vm_callinfo.h:292
rb_callinfo_kwarg
Definition vm_callinfo.h:48
rb_callinfo
Definition vm_callinfo.h:79
rb_calling_info
Definition vm_core.h:306
rb_captured_block
Definition vm_core.h:879
rb_class_cc_entries
Definition vm_callinfo.h:609
rb_control_frame_struct
Definition vm_core.h:912
rb_cvar_class_tbl_entry
Definition class.h:37
rb_env_t
Definition vm_core.h:1301
rb_execution_context_struct
Definition vm_core.h:1034
rb_forwarding_call_data
Definition vm_insnhelper.h:69
rb_id_table
Definition id_table.c:41
rb_iseq_struct
Definition vm_core.h:572
rb_method_iseq_struct::cref
rb_cref_t * cref
class reference, should be marked
Definition method.h:144
rb_method_iseq_struct::iseqptr
const rb_iseq_t * iseqptr
iseq pointer, should be separated from iseqval
Definition method.h:143
rb_proc_t
Definition vm_core.h:1288
set_table
Definition set_table.h:10
vm_ifunc
IFUNC (Internal FUNCtion).
Definition imemo.h:86
vm_svar
SVAR (Special VARiable).
Definition imemo.h:50
vm_svar::cref_or_me
const VALUE cref_or_me
class reference or rb_method_entry_t
Definition imemo.h:52
vm_throw_data
THROW_DATA.
Definition imemo.h:59
__attribute__
Definition vm_insnhelper.c:2410
iseq_inline_storage_entry
Definition vm_core.h:297
SIGNED_VALUE
intptr_t SIGNED_VALUE
A signed integer type that has the same width with VALUE.
Definition value.h:63
ID
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
Definition value.h:52
SIZEOF_VALUE
#define SIZEOF_VALUE
Identical to sizeof(VALUE), except it is a macro that can also be used inside of preprocessor directi...
Definition value.h:69
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
RB_BUILTIN_TYPE
static enum ruby_value_type RB_BUILTIN_TYPE(VALUE obj)
Queries the type of the object.
Definition value_type.h:182
RB_FLOAT_TYPE_P
static bool RB_FLOAT_TYPE_P(VALUE obj)
Queries if the object is an instance of rb_cFloat.
Definition value_type.h:264
RB_TYPE_P
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
Definition value_type.h:376