Ruby: thread_pthread.c Source File

Ruby 4.0.6p0 (2026-07-14 revision 03b6d3f8898a28604fe6cb00eae3226b821168f4)

thread_pthread.c
1/* -*-c-*- */
2/**********************************************************************
3
4 thread_pthread.c -
5
6 $Author$
7
8 Copyright (C) 2004-2007 Koichi Sasada
9
10**********************************************************************/
11
12#ifdef THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION
13
14#include "internal/gc.h"
15#include "internal/sanitizers.h"
16
17#ifdef HAVE_SYS_RESOURCE_H
18#include <sys/resource.h>
19#endif
20#ifdef HAVE_THR_STKSEGMENT
21#include <thread.h>
22#endif
23#if defined(HAVE_FCNTL_H)
24#include <fcntl.h>
25#elif defined(HAVE_SYS_FCNTL_H)
26#include <sys/fcntl.h>
27#endif
28#ifdef HAVE_SYS_PRCTL_H
29#include <sys/prctl.h>
30#endif
31#if defined(HAVE_SYS_TIME_H)
32#include <sys/time.h>
33#endif
34#if defined(__HAIKU__)
35#include <kernel/OS.h>
36#endif
37#ifdef __linux__
38#include <sys/syscall.h> /* for SYS_gettid */
39#endif
40#include <time.h>
41#include <signal.h>
42
43#if defined __APPLE__
44# include <AvailabilityMacros.h>
45#endif
46
47#if defined(HAVE_SYS_EVENTFD_H) && defined(HAVE_EVENTFD)
48# define USE_EVENTFD (1)
49# include <sys/eventfd.h>
50#else
51# define USE_EVENTFD (0)
52#endif
53
54#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK) && \
55 defined(CLOCK_REALTIME) && defined(CLOCK_MONOTONIC) && \
56 defined(HAVE_CLOCK_GETTIME)
57static pthread_condattr_t condattr_mono;
58static pthread_condattr_t *condattr_monotonic = &condattr_mono;
59#else
60static const void *const condattr_monotonic = NULL;
61#endif
62
63#include COROUTINE_H
64
65#ifndef HAVE_SYS_EVENT_H
66#define HAVE_SYS_EVENT_H 0
67#endif
68
69#ifndef HAVE_SYS_EPOLL_H
70#define HAVE_SYS_EPOLL_H 0
71#else
72// force setting for debug
73// #undef HAVE_SYS_EPOLL_H
74// #define HAVE_SYS_EPOLL_H 0
75#endif
76
77#ifndef USE_MN_THREADS
78 #if defined(__EMSCRIPTEN__) || defined(COROUTINE_PTHREAD_CONTEXT)
79 // on __EMSCRIPTEN__ provides epoll* declarations, but no implementations.
80 // on COROUTINE_PTHREAD_CONTEXT, it doesn't worth to use it.
81 #define USE_MN_THREADS 0
82 #elif HAVE_SYS_EPOLL_H
83 #include <sys/epoll.h>
84 #define USE_MN_THREADS 1
85 #elif HAVE_SYS_EVENT_H
86 #include <sys/event.h>
87 #define USE_MN_THREADS 1
88 #else
89 #define USE_MN_THREADS 0
90 #endif
91#endif
92
93#ifdef HAVE_SCHED_YIELD
94#define native_thread_yield() (void)sched_yield()
95#else
96#define native_thread_yield() ((void)0)
97#endif
98
99// native thread wrappers
100
101#define NATIVE_MUTEX_LOCK_DEBUG 0
102#define NATIVE_MUTEX_LOCK_DEBUG_YIELD 0
103
104static void
105mutex_debug(const char *msg, void *lock)
106{
107 if (NATIVE_MUTEX_LOCK_DEBUG) {
108 int r;
109 static pthread_mutex_t dbglock = PTHREAD_MUTEX_INITIALIZER;
110
111 if ((r = pthread_mutex_lock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
112 fprintf(stdout, "%s: %p\n", msg, lock);
113 if ((r = pthread_mutex_unlock(&dbglock)) != 0) {exit(EXIT_FAILURE);}
114 }
115}
116
117void
118 rb_native_mutex_lock(pthread_mutex_t *lock)
119{
120 int r;
121#if NATIVE_MUTEX_LOCK_DEBUG_YIELD
122 native_thread_yield();
123#endif
124 mutex_debug("lock", lock);
125 if ((r = pthread_mutex_lock(lock)) != 0) {
126 rb_bug_errno("pthread_mutex_lock", r);
127 }
128}
129
130void
131 rb_native_mutex_unlock(pthread_mutex_t *lock)
132{
133 int r;
134 mutex_debug("unlock", lock);
135 if ((r = pthread_mutex_unlock(lock)) != 0) {
136 rb_bug_errno("pthread_mutex_unlock", r);
137 }
138}
139
140int
141 rb_native_mutex_trylock(pthread_mutex_t *lock)
142{
143 int r;
144 mutex_debug("trylock", lock);
145 if ((r = pthread_mutex_trylock(lock)) != 0) {
146 if (r == EBUSY) {
147 return EBUSY;
148 }
149 else {
150 rb_bug_errno("pthread_mutex_trylock", r);
151 }
152 }
153 return 0;
154}
155
156void
157 rb_native_mutex_initialize(pthread_mutex_t *lock)
158{
159 int r = pthread_mutex_init(lock, 0);
160 mutex_debug("init", lock);
161 if (r != 0) {
162 rb_bug_errno("pthread_mutex_init", r);
163 }
164}
165
166void
167 rb_native_mutex_destroy(pthread_mutex_t *lock)
168{
169 int r = pthread_mutex_destroy(lock);
170 mutex_debug("destroy", lock);
171 if (r != 0) {
172 rb_bug_errno("pthread_mutex_destroy", r);
173 }
174}
175
176void
177 rb_native_cond_initialize(rb_nativethread_cond_t *cond)
178{
179 int r = pthread_cond_init(cond, condattr_monotonic);
180 if (r != 0) {
181 rb_bug_errno("pthread_cond_init", r);
182 }
183}
184
185void
186 rb_native_cond_destroy(rb_nativethread_cond_t *cond)
187{
188 int r = pthread_cond_destroy(cond);
189 if (r != 0) {
190 rb_bug_errno("pthread_cond_destroy", r);
191 }
192}
193
194/*
195 * In OS X 10.7 (Lion), pthread_cond_signal and pthread_cond_broadcast return
196 * EAGAIN after retrying 8192 times. You can see them in the following page:
197 *
198 * http://www.opensource.apple.com/source/Libc/Libc-763.11/pthreads/pthread_cond.c
199 *
200 * The following rb_native_cond_signal and rb_native_cond_broadcast functions
201 * need to retrying until pthread functions don't return EAGAIN.
202 */
203
204void
205 rb_native_cond_signal(rb_nativethread_cond_t *cond)
206{
207 int r;
208 do {
209 r = pthread_cond_signal(cond);
210 } while (r == EAGAIN);
211 if (r != 0) {
212 rb_bug_errno("pthread_cond_signal", r);
213 }
214}
215
216void
217 rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
218{
219 int r;
220 do {
221 r = pthread_cond_broadcast(cond);
222 } while (r == EAGAIN);
223 if (r != 0) {
224 rb_bug_errno("rb_native_cond_broadcast", r);
225 }
226}
227
228void
229 rb_native_cond_wait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex)
230{
231 int r = pthread_cond_wait(cond, mutex);
232 if (r != 0) {
233 rb_bug_errno("pthread_cond_wait", r);
234 }
235}
236
237static int
238native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, const rb_hrtime_t *abs)
239{
240 int r;
241 struct timespec ts;
242
243 /*
244 * An old Linux may return EINTR. Even though POSIX says
245 * "These functions shall not return an error code of [EINTR]".
246 * http://pubs.opengroup.org/onlinepubs/009695399/functions/pthread_cond_timedwait.html
247 * Let's hide it from arch generic code.
248 */
249 do {
250 rb_hrtime2timespec(&ts, abs);
251 r = pthread_cond_timedwait(cond, mutex, &ts);
252 } while (r == EINTR);
253
254 if (r != 0 && r != ETIMEDOUT) {
255 rb_bug_errno("pthread_cond_timedwait", r);
256 }
257
258 return r;
259}
260
261static rb_hrtime_t
262native_cond_timeout(rb_nativethread_cond_t *cond, const rb_hrtime_t rel)
263{
264 if (condattr_monotonic) {
265 return rb_hrtime_add(rb_hrtime_now(), rel);
266 }
267 else {
268 struct timespec ts;
269
270 rb_timespec_now(&ts);
271 return rb_hrtime_add(rb_timespec2hrtime(&ts), rel);
272 }
273}
274
275void
276 rb_native_cond_timedwait(rb_nativethread_cond_t *cond, pthread_mutex_t *mutex, unsigned long msec)
277{
278 rb_hrtime_t hrmsec = native_cond_timeout(cond, RB_HRTIME_PER_MSEC * msec);
279 native_cond_timedwait(cond, mutex, &hrmsec);
280}
281
282// thread scheduling
283
284static rb_internal_thread_event_hook_t *rb_internal_thread_event_hooks = NULL;
285static void rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th);
286
287#if 0
288static const char *
289event_name(rb_event_flag_t event)
290{
291 switch (event) {
292 case RUBY_INTERNAL_THREAD_EVENT_STARTED:
293 return "STARTED";
294 case RUBY_INTERNAL_THREAD_EVENT_READY:
295 return "READY";
296 case RUBY_INTERNAL_THREAD_EVENT_RESUMED:
297 return "RESUMED";
298 case RUBY_INTERNAL_THREAD_EVENT_SUSPENDED:
299 return "SUSPENDED";
300 case RUBY_INTERNAL_THREAD_EVENT_EXITED:
301 return "EXITED";
302 }
303 return "no-event";
304}
305
306#define RB_INTERNAL_THREAD_HOOK(event, th) \
307 if (UNLIKELY(rb_internal_thread_event_hooks)) { \
308 fprintf(stderr, "[thread=%"PRIxVALUE"] %s in %s (%s:%d)\n", th->self, event_name(event), __func__, __FILE__, __LINE__); \
309 rb_thread_execute_hooks(event, th); \
310 }
311#else
312#define RB_INTERNAL_THREAD_HOOK(event, th) if (UNLIKELY(rb_internal_thread_event_hooks)) { rb_thread_execute_hooks(event, th); }
313#endif
314
315static rb_serial_t current_fork_gen = 1; /* We can't use GET_VM()->fork_gen */
316
317#if defined(SIGVTALRM) && !defined(__EMSCRIPTEN__)
318# define USE_UBF_LIST 1
319#endif
320
321static void threadptr_trap_interrupt(rb_thread_t *);
322
323static void native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
324static void native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt);
325static void native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th);
326
327static void ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r);
328static void timer_thread_wakeup(void);
329static void timer_thread_wakeup_locked(rb_vm_t *vm);
330static void timer_thread_wakeup_force(void);
331static void thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th);
332static void coroutine_transfer0(struct coroutine_context *transfer_from,
333 struct coroutine_context *transfer_to, bool to_dead);
334
335#define thread_sched_dump(s) thread_sched_dump_(__FILE__, __LINE__, s)
336
337static bool
338th_has_dedicated_nt(const rb_thread_t *th)
339{
340 // TODO: th->has_dedicated_nt
341 return th->nt->dedicated > 0;
342}
343
344 RBIMPL_ATTR_MAYBE_UNUSED()
345static void
346thread_sched_dump_(const char *file, int line, struct rb_thread_sched *sched)
347{
348 fprintf(stderr, "@%s:%d running:%d\n", file, line, sched->running ? (int)sched->running->serial : -1);
349 rb_thread_t *th;
350 int i = 0;
351 ccan_list_for_each(&sched->readyq, th, sched.node.readyq) {
352 i++; if (i>10) rb_bug("too many");
353 fprintf(stderr, " ready:%d (%sNT:%d)\n", th->serial,
354 th->nt ? (th->nt->dedicated ? "D" : "S") : "x",
355 th->nt ? (int)th->nt->serial : -1);
356 }
357}
358
359#define ractor_sched_dump(s) ractor_sched_dump_(__FILE__, __LINE__, s)
360
361 RBIMPL_ATTR_MAYBE_UNUSED()
362static void
363ractor_sched_dump_(const char *file, int line, rb_vm_t *vm)
364{
365 rb_ractor_t *r;
366
367 fprintf(stderr, "ractor_sched_dump %s:%d\n", file, line);
368
369 int i = 0;
370 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
371 i++;
372 if (i>10) rb_bug("!!");
373 fprintf(stderr, " %d ready:%d\n", i, rb_ractor_id(r));
374 }
375}
376
377#define thread_sched_lock(a, b) thread_sched_lock_(a, b, __FILE__, __LINE__)
378#define thread_sched_unlock(a, b) thread_sched_unlock_(a, b, __FILE__, __LINE__)
379
380static void
381thread_sched_set_locked(struct rb_thread_sched *sched, rb_thread_t *th)
382{
383#if VM_CHECK_MODE > 0
384 VM_ASSERT(sched->lock_owner == NULL);
385
386 sched->lock_owner = th;
387#endif
388}
389
390static void
391thread_sched_set_unlocked(struct rb_thread_sched *sched, rb_thread_t *th)
392{
393#if VM_CHECK_MODE > 0
394 VM_ASSERT(sched->lock_owner == th);
395
396 sched->lock_owner = NULL;
397#endif
398}
399
400static void
401thread_sched_lock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
402{
403 rb_native_mutex_lock(&sched->lock_);
404
405#if VM_CHECK_MODE
406 RUBY_DEBUG_LOG2(file, line, "r:%d th:%u", th ? (int)rb_ractor_id(th->ractor) : -1, rb_th_serial(th));
407#else
408 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
409#endif
410
411 thread_sched_set_locked(sched, th);
412}
413
414static void
415thread_sched_unlock_(struct rb_thread_sched *sched, rb_thread_t *th, const char *file, int line)
416{
417 RUBY_DEBUG_LOG2(file, line, "th:%u", rb_th_serial(th));
418
419 thread_sched_set_unlocked(sched, th);
420
421 rb_native_mutex_unlock(&sched->lock_);
422}
423
424static void
425ASSERT_thread_sched_locked(struct rb_thread_sched *sched, rb_thread_t *th)
426{
427 VM_ASSERT(rb_native_mutex_trylock(&sched->lock_) == EBUSY);
428
429#if VM_CHECK_MODE
430 if (th) {
431 VM_ASSERT(sched->lock_owner == th);
432 }
433 else {
434 VM_ASSERT(sched->lock_owner != NULL);
435 }
436#endif
437}
438
439#define ractor_sched_lock(a, b) ractor_sched_lock_(a, b, __FILE__, __LINE__)
440#define ractor_sched_unlock(a, b) ractor_sched_unlock_(a, b, __FILE__, __LINE__)
441
442 RBIMPL_ATTR_MAYBE_UNUSED()
443static unsigned int
444rb_ractor_serial(const rb_ractor_t *r)
445{
446 if (r) {
447 return rb_ractor_id(r);
448 }
449 else {
450 return 0;
451 }
452}
453
454static void
455ractor_sched_set_locked(rb_vm_t *vm, rb_ractor_t *cr)
456{
457#if VM_CHECK_MODE > 0
458 VM_ASSERT(vm->ractor.sched.lock_owner == NULL);
459 VM_ASSERT(vm->ractor.sched.locked == false);
460
461 vm->ractor.sched.lock_owner = cr;
462 vm->ractor.sched.locked = true;
463#endif
464}
465
466static void
467ractor_sched_set_unlocked(rb_vm_t *vm, rb_ractor_t *cr)
468{
469#if VM_CHECK_MODE > 0
470 VM_ASSERT(vm->ractor.sched.locked);
471 VM_ASSERT(vm->ractor.sched.lock_owner == cr);
472
473 vm->ractor.sched.locked = false;
474 vm->ractor.sched.lock_owner = NULL;
475#endif
476}
477
478static void
479ractor_sched_lock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
480{
481 rb_native_mutex_lock(&vm->ractor.sched.lock);
482
483#if VM_CHECK_MODE
484 RUBY_DEBUG_LOG2(file, line, "cr:%u prev_owner:%u", rb_ractor_serial(cr), rb_ractor_serial(vm->ractor.sched.lock_owner));
485#else
486 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
487#endif
488
489 ractor_sched_set_locked(vm, cr);
490}
491
492static void
493ractor_sched_unlock_(rb_vm_t *vm, rb_ractor_t *cr, const char *file, int line)
494{
495 RUBY_DEBUG_LOG2(file, line, "cr:%u", rb_ractor_serial(cr));
496
497 ractor_sched_set_unlocked(vm, cr);
498 rb_native_mutex_unlock(&vm->ractor.sched.lock);
499}
500
501static void
502ASSERT_ractor_sched_locked(rb_vm_t *vm, rb_ractor_t *cr)
503{
504 VM_ASSERT(rb_native_mutex_trylock(&vm->ractor.sched.lock) == EBUSY);
505 VM_ASSERT(vm->ractor.sched.locked);
506 VM_ASSERT(cr == NULL || vm->ractor.sched.lock_owner == cr);
507}
508
509 RBIMPL_ATTR_MAYBE_UNUSED()
510static bool
511ractor_sched_running_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
512{
513 rb_thread_t *rth;
514 ccan_list_for_each(&vm->ractor.sched.running_threads, rth, sched.node.running_threads) {
515 if (rth == th) return true;
516 }
517 return false;
518}
519
520 RBIMPL_ATTR_MAYBE_UNUSED()
521static unsigned int
522ractor_sched_running_threads_size(rb_vm_t *vm)
523{
524 rb_thread_t *th;
525 unsigned int i = 0;
526 ccan_list_for_each(&vm->ractor.sched.running_threads, th, sched.node.running_threads) {
527 i++;
528 }
529 return i;
530}
531
532 RBIMPL_ATTR_MAYBE_UNUSED()
533static unsigned int
534ractor_sched_timeslice_threads_size(rb_vm_t *vm)
535{
536 rb_thread_t *th;
537 unsigned int i = 0;
538 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
539 i++;
540 }
541 return i;
542}
543
544 RBIMPL_ATTR_MAYBE_UNUSED()
545static bool
546ractor_sched_timeslice_threads_contain_p(rb_vm_t *vm, rb_thread_t *th)
547{
548 rb_thread_t *rth;
549 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, rth, sched.node.timeslice_threads) {
550 if (rth == th) return true;
551 }
552 return false;
553}
554
555static void ractor_sched_barrier_join_signal_locked(rb_vm_t *vm);
556
557// setup timeslice signals by the timer thread.
558static void
559thread_sched_setup_running_threads(struct rb_thread_sched *sched, rb_ractor_t *cr, rb_vm_t *vm,
560 rb_thread_t *add_th, rb_thread_t *del_th, rb_thread_t *add_timeslice_th)
561{
562#if USE_RUBY_DEBUG_LOG
563 unsigned int prev_running_cnt = vm->ractor.sched.running_cnt;
564#endif
565
566 rb_thread_t *del_timeslice_th;
567
568 if (del_th && sched->is_running_timeslice) {
569 del_timeslice_th = del_th;
570 sched->is_running_timeslice = false;
571 }
572 else {
573 del_timeslice_th = NULL;
574 }
575
576 RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u",
577 rb_th_serial(add_th), rb_th_serial(del_th),
578 rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th));
579
580 ractor_sched_lock(vm, cr);
581 {
582 // update running_threads
583 if (del_th) {
584 VM_ASSERT(ractor_sched_running_threads_contain_p(vm, del_th));
585 VM_ASSERT(del_timeslice_th != NULL ||
586 !ractor_sched_timeslice_threads_contain_p(vm, del_th));
587
588 ccan_list_del_init(&del_th->sched.node.running_threads);
589 vm->ractor.sched.running_cnt--;
590
591 if (UNLIKELY(vm->ractor.sched.barrier_waiting)) {
592 ractor_sched_barrier_join_signal_locked(vm);
593 }
594 sched->is_running = false;
595 }
596
597 if (add_th) {
598 if (vm->ractor.sched.barrier_waiting) {
599 // TODO: GC barrier check?
600 RUBY_DEBUG_LOG("barrier_waiting");
601 RUBY_VM_SET_VM_BARRIER_INTERRUPT(add_th->ec);
602 }
603
604 VM_ASSERT(!ractor_sched_running_threads_contain_p(vm, add_th));
605 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_th));
606
607 ccan_list_add(&vm->ractor.sched.running_threads, &add_th->sched.node.running_threads);
608 vm->ractor.sched.running_cnt++;
609 sched->is_running = true;
610 }
611
612 if (add_timeslice_th) {
613 // update timeslice threads
614 int was_empty = ccan_list_empty(&vm->ractor.sched.timeslice_threads);
615 VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(vm, add_timeslice_th));
616 ccan_list_add(&vm->ractor.sched.timeslice_threads, &add_timeslice_th->sched.node.timeslice_threads);
617 sched->is_running_timeslice = true;
618 if (was_empty) {
619 timer_thread_wakeup_locked(vm);
620 }
621 }
622
623 if (del_timeslice_th) {
624 VM_ASSERT(ractor_sched_timeslice_threads_contain_p(vm, del_timeslice_th));
625 ccan_list_del_init(&del_timeslice_th->sched.node.timeslice_threads);
626 }
627
628 VM_ASSERT(ractor_sched_running_threads_size(vm) == vm->ractor.sched.running_cnt);
629 VM_ASSERT(ractor_sched_timeslice_threads_size(vm) <= vm->ractor.sched.running_cnt);
630 }
631 ractor_sched_unlock(vm, cr);
632
633 //RUBY_DEBUG_LOG("+:%u -:%u +ts:%u -ts:%u run:%u->%u",
634 // rb_th_serial(add_th), rb_th_serial(del_th),
635 // rb_th_serial(add_timeslice_th), rb_th_serial(del_timeslice_th),
636 RUBY_DEBUG_LOG("run:%u->%u", prev_running_cnt, vm->ractor.sched.running_cnt);
637}
638
639static void
640thread_sched_add_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
641{
642 ASSERT_thread_sched_locked(sched, th);
643 VM_ASSERT(sched->running == th);
644
645 rb_vm_t *vm = th->vm;
646 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, ccan_list_empty(&sched->readyq) ? NULL : th);
647}
648
649static void
650thread_sched_del_running_thread(struct rb_thread_sched *sched, rb_thread_t *th)
651{
652 ASSERT_thread_sched_locked(sched, th);
653
654 rb_vm_t *vm = th->vm;
655 thread_sched_setup_running_threads(sched, th->ractor, vm, NULL, th, NULL);
656}
657
658void
659rb_add_running_thread(rb_thread_t *th)
660{
661 struct rb_thread_sched *sched = TH_SCHED(th);
662
663 thread_sched_lock(sched, th);
664 {
665 thread_sched_add_running_thread(sched, th);
666 }
667 thread_sched_unlock(sched, th);
668}
669
670void
671rb_del_running_thread(rb_thread_t *th)
672{
673 struct rb_thread_sched *sched = TH_SCHED(th);
674
675 thread_sched_lock(sched, th);
676 {
677 thread_sched_del_running_thread(sched, th);
678 }
679 thread_sched_unlock(sched, th);
680}
681
682// setup current or next running thread
683// sched->running should be set only on this function.
684//
685// if th is NULL, there is no running threads.
686static void
687thread_sched_set_running(struct rb_thread_sched *sched, rb_thread_t *th)
688{
689 RUBY_DEBUG_LOG("th:%u->th:%u", rb_th_serial(sched->running), rb_th_serial(th));
690 VM_ASSERT(sched->running != th);
691
692 sched->running = th;
693}
694
695 RBIMPL_ATTR_MAYBE_UNUSED()
696static bool
697thread_sched_readyq_contain_p(struct rb_thread_sched *sched, rb_thread_t *th)
698{
699 rb_thread_t *rth;
700 ccan_list_for_each(&sched->readyq, rth, sched.node.readyq) {
701 if (rth == th) {
702 VM_ASSERT(th->sched.node.is_ready);
703 return true;
704 }
705 }
706 VM_ASSERT(!th->sched.node.is_ready);
707 return false;
708}
709
710// deque thread from the ready queue.
711// if the ready queue is empty, return NULL.
712//
713// return deque'ed running thread (or NULL).
714static rb_thread_t *
715thread_sched_deq(struct rb_thread_sched *sched)
716{
717 ASSERT_thread_sched_locked(sched, NULL);
718 rb_thread_t *next_th;
719
720 VM_ASSERT(sched->running != NULL);
721
722 if (ccan_list_empty(&sched->readyq)) {
723 next_th = NULL;
724 }
725 else {
726 next_th = ccan_list_pop(&sched->readyq, rb_thread_t, sched.node.readyq);
727 VM_ASSERT(next_th->sched.node.is_ready);
728 next_th->sched.node.is_ready = false;
729
730 VM_ASSERT(sched->readyq_cnt > 0);
731 sched->readyq_cnt--;
732 ccan_list_node_init(&next_th->sched.node.readyq);
733 }
734
735 RUBY_DEBUG_LOG("next_th:%u readyq_cnt:%d", rb_th_serial(next_th), sched->readyq_cnt);
736
737 return next_th;
738}
739
740// enqueue ready thread to the ready queue.
741static void
742thread_sched_enq(struct rb_thread_sched *sched, rb_thread_t *ready_th)
743{
744 ASSERT_thread_sched_locked(sched, NULL);
745 RUBY_DEBUG_LOG("ready_th:%u readyq_cnt:%d", rb_th_serial(ready_th), sched->readyq_cnt);
746
747 VM_ASSERT(sched->running != NULL);
748 VM_ASSERT(!thread_sched_readyq_contain_p(sched, ready_th));
749
750 if (sched->is_running) {
751 if (ccan_list_empty(&sched->readyq)) {
752 // add sched->running to timeslice
753 thread_sched_setup_running_threads(sched, ready_th->ractor, ready_th->vm, NULL, NULL, sched->running);
754 }
755 }
756 else {
757 // ractor_sched lock is needed
758 // VM_ASSERT(!ractor_sched_timeslice_threads_contain_p(ready_th->vm, sched->running));
759 }
760
761 ccan_list_add_tail(&sched->readyq, &ready_th->sched.node.readyq);
762 ready_th->sched.node.is_ready = true;
763 sched->readyq_cnt++;
764}
765
766// DNT: kick condvar
767// SNT: TODO
768static void
769thread_sched_wakeup_running_thread(struct rb_thread_sched *sched, rb_thread_t *next_th, bool will_switch)
770{
771 ASSERT_thread_sched_locked(sched, NULL);
772 VM_ASSERT(sched->running == next_th);
773
774 if (next_th) {
775 if (next_th->nt) {
776 if (th_has_dedicated_nt(next_th)) {
777 RUBY_DEBUG_LOG("pinning th:%u", next_th->serial);
778 rb_native_cond_signal(&next_th->nt->cond.readyq);
779 }
780 else {
781 // TODO
782 RUBY_DEBUG_LOG("th:%u is already running.", next_th->serial);
783 }
784 }
785 else {
786 if (will_switch) {
787 RUBY_DEBUG_LOG("th:%u (do nothing)", rb_th_serial(next_th));
788 }
789 else {
790 RUBY_DEBUG_LOG("th:%u (enq)", rb_th_serial(next_th));
791 ractor_sched_enq(next_th->vm, next_th->ractor);
792 }
793 }
794 }
795 else {
796 RUBY_DEBUG_LOG("no waiting threads%s", "");
797 }
798}
799
800// waiting -> ready (locked)
801static void
802thread_sched_to_ready_common(struct rb_thread_sched *sched, rb_thread_t *th, bool wakeup, bool will_switch)
803{
804 RUBY_DEBUG_LOG("th:%u running:%u redyq_cnt:%d", rb_th_serial(th), rb_th_serial(sched->running), sched->readyq_cnt);
805
806 VM_ASSERT(sched->running != th);
807 VM_ASSERT(!thread_sched_readyq_contain_p(sched, th));
808 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_READY, th);
809
810 if (sched->running == NULL) {
811 thread_sched_set_running(sched, th);
812 if (wakeup) thread_sched_wakeup_running_thread(sched, th, will_switch);
813 }
814 else {
815 thread_sched_enq(sched, th);
816 }
817}
818
819// waiting -> ready
820//
821// `th` had became "waiting" state by `thread_sched_to_waiting`
822// and `thread_sched_to_ready` enqueue `th` to the thread ready queue.
823 RBIMPL_ATTR_MAYBE_UNUSED()
824static void
825thread_sched_to_ready(struct rb_thread_sched *sched, rb_thread_t *th)
826{
827 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
828
829 thread_sched_lock(sched, th);
830 {
831 thread_sched_to_ready_common(sched, th, true, false);
832 }
833 thread_sched_unlock(sched, th);
834}
835
836// wait until sched->running is `th`.
837static void
838thread_sched_wait_running_turn(struct rb_thread_sched *sched, rb_thread_t *th, bool can_direct_transfer)
839{
840 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
841
842 ASSERT_thread_sched_locked(sched, th);
843 VM_ASSERT(th == rb_ec_thread_ptr(rb_current_ec_noinline()));
844
845 if (th != sched->running) {
846 // TODO: This optimization should also be made to work for MN_THREADS
847 if (th->has_dedicated_nt && th == sched->runnable_hot_th && (sched->running == NULL || sched->running->has_dedicated_nt)) {
848 RUBY_DEBUG_LOG("(nt) stealing: hot-th:%u. running:%u", rb_th_serial(th), rb_th_serial(sched->running));
849
850 // If there is a thread set to run, move it back to the front of the readyq
851 if (sched->running != NULL) {
852 rb_thread_t *running = sched->running;
853 VM_ASSERT(!thread_sched_readyq_contain_p(sched, running));
854 running->sched.node.is_ready = true;
855 ccan_list_add(&sched->readyq, &running->sched.node.readyq);
856 sched->readyq_cnt++;
857 }
858
859 // Pull off the ready queue and start running.
860 if (th->sched.node.is_ready) {
861 VM_ASSERT(thread_sched_readyq_contain_p(sched, th));
862 ccan_list_del_init(&th->sched.node.readyq);
863 th->sched.node.is_ready = false;
864 sched->readyq_cnt--;
865 }
866 thread_sched_set_running(sched, th);
867 rb_ractor_thread_switch(th->ractor, th, false);
868 }
869
870 // already deleted from running threads
871 // VM_ASSERT(!ractor_sched_running_threads_contain_p(th->vm, th)); // need locking
872
873 // wait for execution right
874 rb_thread_t *next_th;
875 while((next_th = sched->running) != th) {
876 if (th_has_dedicated_nt(th)) {
877 RUBY_DEBUG_LOG("(nt) sleep th:%u running:%u", rb_th_serial(th), rb_th_serial(sched->running));
878
879 thread_sched_set_unlocked(sched, th);
880 {
881 RUBY_DEBUG_LOG("nt:%d cond:%p", th->nt->serial, &th->nt->cond.readyq);
882 rb_native_cond_wait(&th->nt->cond.readyq, &sched->lock_);
883 }
884 thread_sched_set_locked(sched, th);
885
886 if (sched->runnable_hot_th != NULL && sched->runnable_hot_th_waiting) {
887 VM_ASSERT(sched->runnable_hot_th != th);
888 // Give the hot thread a chance to preempt, if it's actively spinning.
889 // On multicore, this reduces the rate of core-switching. On single-core it
890 // should mostly be a nop, since the other thread can't be concurrently spinning.
891 thread_sched_unlock(sched, th);
892 thread_sched_lock(sched, th);
893 }
894
895 RUBY_DEBUG_LOG("(nt) wakeup %s", sched->running == th ? "success" : "failed");
896 if (th == sched->running) {
897 rb_ractor_thread_switch(th->ractor, th, false);
898 }
899 }
900 else {
901 // search another ready thread
902 if (can_direct_transfer &&
903 (next_th = sched->running) != NULL &&
904 !next_th->nt // next_th is running or has dedicated nt
905 ) {
906
907 RUBY_DEBUG_LOG("th:%u->%u (direct)", rb_th_serial(th), rb_th_serial(next_th));
908
909 thread_sched_set_unlocked(sched, th);
910 {
911 rb_ractor_set_current_ec(th->ractor, NULL);
912 thread_sched_switch(th, next_th);
913 }
914 thread_sched_set_locked(sched, th);
915 }
916 else {
917 // search another ready ractor
918 struct rb_native_thread *nt = th->nt;
919 native_thread_assign(NULL, th);
920
921 RUBY_DEBUG_LOG("th:%u->%u (ractor scheduling)", rb_th_serial(th), rb_th_serial(next_th));
922
923 thread_sched_set_unlocked(sched, th);
924 {
925 rb_ractor_set_current_ec(th->ractor, NULL);
926 coroutine_transfer0(th->sched.context, nt->nt_context, false);
927 }
928 thread_sched_set_locked(sched, th);
929 }
930
931 VM_ASSERT(rb_current_ec_noinline() == th->ec);
932 }
933 }
934
935 VM_ASSERT(th->nt != NULL);
936 VM_ASSERT(rb_current_ec_noinline() == th->ec);
937 VM_ASSERT(th->sched.waiting_reason.flags == thread_sched_waiting_none);
938
939 // add th to running threads
940 thread_sched_add_running_thread(sched, th);
941 }
942
943 // Control transfer to the current thread is now complete. The original thread
944 // cannot steal control at this point.
945 sched->runnable_hot_th = NULL;
946 sched->runnable_hot_th_waiting = 0;
947
948 // VM_ASSERT(ractor_sched_running_threads_contain_p(th->vm, th)); need locking
949 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_RESUMED, th);
950}
951
952// waiting -> ready -> running (locked)
953static void
954thread_sched_to_running_common(struct rb_thread_sched *sched, rb_thread_t *th)
955{
956 RUBY_DEBUG_LOG("th:%u dedicated:%d", rb_th_serial(th), th_has_dedicated_nt(th));
957
958 VM_ASSERT(sched->running != th);
959 VM_ASSERT(th_has_dedicated_nt(th));
960 VM_ASSERT(GET_THREAD() == th);
961
962 native_thread_dedicated_dec(th->vm, th->ractor, th->nt);
963
964 // waiting -> ready
965 thread_sched_to_ready_common(sched, th, false, false);
966
967 if (sched->running == th) {
968 thread_sched_add_running_thread(sched, th);
969 }
970
971 // TODO: check SNT number
972 thread_sched_wait_running_turn(sched, th, false);
973}
974
975// waiting -> ready -> running
976//
977// `th` had been waiting by `thread_sched_to_waiting()`
978// and run a dedicated task (like waitpid and so on).
979// After the dedicated task, this function is called
980// to join a normal thread-scheduling.
981static void
982thread_sched_to_running(struct rb_thread_sched *sched, rb_thread_t *th)
983{
984 // We are reading and writing these sched fields without lock cover, but
985 // there are no correctness issues resulting from stale cache or delayed writeback.
986 // When it works, this causes the next-scheduled thread to yield the sched lock
987 // briefly so that we can grab it if we're still spinning (not descheduled yet).
988 if (sched->runnable_hot_th == th) {
989 sched->runnable_hot_th_waiting = 1;
990 }
991 thread_sched_lock(sched, th);
992 {
993 thread_sched_to_running_common(sched, th);
994 }
995 thread_sched_unlock(sched, th);
996}
997
998// resume a next thread in the thread ready queue.
999//
1000// deque next running thread from the ready thread queue and
1001// resume this thread if available.
1002//
1003// If the next therad has a dedicated native thraed, simply signal to resume.
1004// Otherwise, make the ractor ready and other nt will run the ractor and the thread.
1005static void
1006thread_sched_wakeup_next_thread(struct rb_thread_sched *sched, rb_thread_t *th, bool will_switch)
1007{
1008 ASSERT_thread_sched_locked(sched, th);
1009
1010 VM_ASSERT(sched->running == th);
1011 VM_ASSERT(sched->running->nt != NULL);
1012
1013 rb_thread_t *next_th = thread_sched_deq(sched);
1014
1015 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
1016 VM_ASSERT(th != next_th);
1017
1018 thread_sched_set_running(sched, next_th);
1019 VM_ASSERT(next_th == sched->running);
1020 thread_sched_wakeup_running_thread(sched, next_th, will_switch);
1021
1022 if (th != next_th) {
1023 thread_sched_del_running_thread(sched, th);
1024 }
1025}
1026
1027// running -> dead (locked)
1028static void
1029thread_sched_to_dead_common(struct rb_thread_sched *sched, rb_thread_t *th)
1030{
1031 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1032
1033 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1034
1035 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1036
1037 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_EXITED, th);
1038}
1039
1040// running -> dead
1041static void
1042thread_sched_to_dead(struct rb_thread_sched *sched, rb_thread_t *th)
1043{
1044 thread_sched_lock(sched, th);
1045 {
1046 thread_sched_to_dead_common(sched, th);
1047 }
1048 thread_sched_unlock(sched, th);
1049}
1050
1051// running -> waiting (locked)
1052//
1053// This thread will run dedicated task (th->nt->dedicated++).
1054static void
1055thread_sched_to_waiting_common(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately)
1056{
1057 RUBY_DEBUG_LOG("th:%u DNT:%d", rb_th_serial(th), th->nt->dedicated);
1058
1059 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1060
1061 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
1062 if (!yield_immediately) {
1063 sched->runnable_hot_th = th;
1064 sched->runnable_hot_th_waiting = 0;
1065 }
1066 thread_sched_wakeup_next_thread(sched, th, false);
1067}
1068
1069// running -> waiting
1070//
1071// This thread will run a dedicated task.
1072static void
1073thread_sched_to_waiting(struct rb_thread_sched *sched, rb_thread_t *th, bool yield_immediately)
1074{
1075 thread_sched_lock(sched, th);
1076 {
1077 thread_sched_to_waiting_common(sched, th, yield_immediately);
1078 }
1079 thread_sched_unlock(sched, th);
1080}
1081
1082// mini utility func
1083// return true if any there are any interrupts
1084static bool
1085ubf_set(rb_thread_t *th, rb_unblock_function_t *func, void *arg)
1086{
1087 VM_ASSERT(func != NULL);
1088
1089 retry:
1090 if (RUBY_VM_INTERRUPTED(th->ec)) {
1091 RUBY_DEBUG_LOG("interrupted:0x%x", th->ec->interrupt_flag);
1092 return true;
1093 }
1094
1095 rb_native_mutex_lock(&th->interrupt_lock);
1096 {
1097 if (!th->ec->raised_flag && RUBY_VM_INTERRUPTED(th->ec)) {
1098 rb_native_mutex_unlock(&th->interrupt_lock);
1099 goto retry;
1100 }
1101
1102 VM_ASSERT(th->unblock.func == NULL);
1103 th->unblock.func = func;
1104 th->unblock.arg = arg;
1105 }
1106 rb_native_mutex_unlock(&th->interrupt_lock);
1107
1108 return false;
1109}
1110
1111static void
1112ubf_clear(rb_thread_t *th)
1113{
1114 if (th->unblock.func) {
1115 rb_native_mutex_lock(&th->interrupt_lock);
1116 {
1117 th->unblock.func = NULL;
1118 th->unblock.arg = NULL;
1119 }
1120 rb_native_mutex_unlock(&th->interrupt_lock);
1121 }
1122}
1123
1124static void
1125ubf_waiting(void *ptr)
1126{
1127 rb_thread_t *th = (rb_thread_t *)ptr;
1128 struct rb_thread_sched *sched = TH_SCHED(th);
1129
1130 // only once. it is safe because th->interrupt_lock is already acquired.
1131 th->unblock.func = NULL;
1132 th->unblock.arg = NULL;
1133
1134 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1135
1136 thread_sched_lock(sched, th);
1137 {
1138 if (sched->running == th) {
1139 // not sleeping yet.
1140 }
1141 else {
1142 thread_sched_to_ready_common(sched, th, true, false);
1143 }
1144 }
1145 thread_sched_unlock(sched, th);
1146}
1147
1148// running -> waiting
1149//
1150// This thread will sleep until other thread wakeup the thread.
1151static void
1152thread_sched_to_waiting_until_wakeup(struct rb_thread_sched *sched, rb_thread_t *th)
1153{
1154 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
1155
1156 RB_VM_SAVE_MACHINE_CONTEXT(th);
1157
1158
1159 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1160
1161 thread_sched_lock(sched, th);
1162 {
1163 // NOTE: there's a lock ordering inversion here with the ubf call, but it's benign.
1164 if (ubf_set(th, ubf_waiting, (void *)th)) {
1165 RUBY_DEBUG_LOG("th:%u interrupted", rb_th_serial(th));
1166 }
1167 else {
1168 bool can_direct_transfer = !th_has_dedicated_nt(th);
1169 // NOTE: th->status is set before and after this sleep outside of this function in `sleep_forever`
1170 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1171 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1172 }
1173 }
1174 thread_sched_unlock(sched, th);
1175
1176 ubf_clear(th);
1177}
1178
1179// run another thread in the ready queue.
1180// continue to run if there are no ready threads.
1181static void
1182thread_sched_yield(struct rb_thread_sched *sched, rb_thread_t *th)
1183{
1184 RUBY_DEBUG_LOG("th:%d sched->readyq_cnt:%d", (int)th->serial, sched->readyq_cnt);
1185
1186 thread_sched_lock(sched, th);
1187 {
1188 if (!ccan_list_empty(&sched->readyq)) {
1189 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1190 thread_sched_wakeup_next_thread(sched, th, !th_has_dedicated_nt(th));
1191 bool can_direct_transfer = !th_has_dedicated_nt(th);
1192 thread_sched_to_ready_common(sched, th, false, can_direct_transfer);
1193 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1194 th->status = THREAD_RUNNABLE;
1195 }
1196 else {
1197 VM_ASSERT(sched->readyq_cnt == 0);
1198 }
1199 }
1200 thread_sched_unlock(sched, th);
1201}
1202
1203void
1204rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork)
1205{
1206 rb_native_mutex_initialize(&sched->lock_);
1207
1208#if VM_CHECK_MODE
1209 sched->lock_owner = NULL;
1210#endif
1211
1212 ccan_list_head_init(&sched->readyq);
1213 sched->readyq_cnt = 0;
1214
1215#if USE_MN_THREADS
1216 if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors
1217#endif
1218}
1219
1220static void
1221coroutine_transfer0(struct coroutine_context *transfer_from, struct coroutine_context *transfer_to, bool to_dead)
1222{
1223#ifdef RUBY_ASAN_ENABLED
1224 void **fake_stack = to_dead ? NULL : &transfer_from->fake_stack;
1225 __sanitizer_start_switch_fiber(fake_stack, transfer_to->stack_base, transfer_to->stack_size);
1226#endif
1227
1228 RBIMPL_ATTR_MAYBE_UNUSED()
1229 struct coroutine_context *returning_from = coroutine_transfer(transfer_from, transfer_to);
1230
1231 /* if to_dead was passed, the caller is promising that this coroutine is finished and it should
1232 * never be resumed! */
1233 VM_ASSERT(!to_dead);
1234#ifdef RUBY_ASAN_ENABLED
1235 __sanitizer_finish_switch_fiber(transfer_from->fake_stack,
1236 (const void**)&returning_from->stack_base, &returning_from->stack_size);
1237#endif
1238
1239}
1240
1241static void
1242thread_sched_switch0(struct coroutine_context *current_cont, rb_thread_t *next_th, struct rb_native_thread *nt, bool to_dead)
1243{
1244 VM_ASSERT(!nt->dedicated);
1245 VM_ASSERT(next_th->nt == NULL);
1246
1247 RUBY_DEBUG_LOG("next_th:%u", rb_th_serial(next_th));
1248
1249 ruby_thread_set_native(next_th);
1250 native_thread_assign(nt, next_th);
1251
1252 coroutine_transfer0(current_cont, next_th->sched.context, to_dead);
1253}
1254
1255static void
1256thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th)
1257{
1258 struct rb_native_thread *nt = cth->nt;
1259 native_thread_assign(NULL, cth);
1260 RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial);
1261 thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED);
1262}
1263
1264#if VM_CHECK_MODE > 0
1265 RBIMPL_ATTR_MAYBE_UNUSED()
1266static unsigned int
1267grq_size(rb_vm_t *vm, rb_ractor_t *cr)
1268{
1269 ASSERT_ractor_sched_locked(vm, cr);
1270
1271 rb_ractor_t *r, *prev_r = NULL;
1272 unsigned int i = 0;
1273
1274 ccan_list_for_each(&vm->ractor.sched.grq, r, threads.sched.grq_node) {
1275 i++;
1276
1277 VM_ASSERT(r != prev_r);
1278 prev_r = r;
1279 }
1280 return i;
1281}
1282#endif
1283
1284static void
1285ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r)
1286{
1287 struct rb_thread_sched *sched = &r->threads.sched;
1288 rb_ractor_t *cr = NULL; // timer thread can call this function
1289
1290 VM_ASSERT(sched->running != NULL);
1291 VM_ASSERT(sched->running->nt == NULL);
1292
1293 ractor_sched_lock(vm, cr);
1294 {
1295#if VM_CHECK_MODE > 0
1296 // check if grq contains r
1297 rb_ractor_t *tr;
1298 ccan_list_for_each(&vm->ractor.sched.grq, tr, threads.sched.grq_node) {
1299 VM_ASSERT(r != tr);
1300 }
1301#endif
1302
1303 ccan_list_add_tail(&vm->ractor.sched.grq, &sched->grq_node);
1304 vm->ractor.sched.grq_cnt++;
1305 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1306
1307 RUBY_DEBUG_LOG("r:%u th:%u grq_cnt:%u", rb_ractor_id(r), rb_th_serial(sched->running), vm->ractor.sched.grq_cnt);
1308
1309 rb_native_cond_signal(&vm->ractor.sched.cond);
1310
1311 // ractor_sched_dump(vm);
1312 }
1313 ractor_sched_unlock(vm, cr);
1314}
1315
1316
1317#ifndef SNT_KEEP_SECONDS
1318#define SNT_KEEP_SECONDS 0
1319#endif
1320
1321#ifndef MINIMUM_SNT
1322// make at least MINIMUM_SNT snts for debug.
1323#define MINIMUM_SNT 0
1324#endif
1325
1326static rb_ractor_t *
1327ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr)
1328{
1329 rb_ractor_t *r;
1330
1331 ractor_sched_lock(vm, cr);
1332 {
1333 RUBY_DEBUG_LOG("empty? %d", ccan_list_empty(&vm->ractor.sched.grq));
1334 // ractor_sched_dump(vm);
1335
1336 VM_ASSERT(rb_current_execution_context(false) == NULL);
1337 VM_ASSERT(grq_size(vm, cr) == vm->ractor.sched.grq_cnt);
1338
1339 while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) {
1340 RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1341
1342#if SNT_KEEP_SECONDS > 0
1343 rb_hrtime_t abs = rb_hrtime_add(rb_hrtime_now(), RB_HRTIME_PER_SEC * SNT_KEEP_SECONDS);
1344 if (native_cond_timedwait(&vm->ractor.sched.cond, &vm->ractor.sched.lock, &abs) == ETIMEDOUT) {
1345 RUBY_DEBUG_LOG("timeout, grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1346 VM_ASSERT(r == NULL);
1347 vm->ractor.sched.snt_cnt--;
1348 vm->ractor.sched.running_cnt--;
1349 break;
1350 }
1351 else {
1352 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1353 }
1354#else
1355 ractor_sched_set_unlocked(vm, cr);
1356 rb_native_cond_wait(&vm->ractor.sched.cond, &vm->ractor.sched.lock);
1357 ractor_sched_set_locked(vm, cr);
1358
1359 RUBY_DEBUG_LOG("wakeup grq_cnt:%d", (int)vm->ractor.sched.grq_cnt);
1360#endif
1361 }
1362
1363 VM_ASSERT(rb_current_execution_context(false) == NULL);
1364
1365 if (r) {
1366 VM_ASSERT(vm->ractor.sched.grq_cnt > 0);
1367 vm->ractor.sched.grq_cnt--;
1368 RUBY_DEBUG_LOG("r:%d grq_cnt:%u", (int)rb_ractor_id(r), vm->ractor.sched.grq_cnt);
1369 }
1370 else {
1371 VM_ASSERT(SNT_KEEP_SECONDS > 0);
1372 // timeout
1373 }
1374 }
1375 ractor_sched_unlock(vm, cr);
1376
1377 return r;
1378}
1379
1380void rb_ractor_lock_self(rb_ractor_t *r);
1381void rb_ractor_unlock_self(rb_ractor_t *r);
1382
1383// The current thread for a ractor is put to "sleep" (descheduled in the STOPPED_FOREVER state) waiting for
1384// a ractor action to wake it up.
1385void
1386rb_ractor_sched_wait(rb_execution_context_t *ec, rb_ractor_t *cr, rb_unblock_function_t *ubf, void *ubf_arg)
1387{
1388 // ractor lock of cr is acquired
1389
1390 RUBY_DEBUG_LOG("start%s", "");
1391
1392 rb_thread_t * volatile th = rb_ec_thread_ptr(ec);
1393 struct rb_thread_sched *sched = TH_SCHED(th);
1394
1395 if (ubf_set(th, ubf, ubf_arg)) {
1396 // interrupted
1397 return;
1398 }
1399
1400 thread_sched_lock(sched, th);
1401 rb_ractor_unlock_self(cr);
1402 {
1403 // setup sleep
1404 bool can_direct_transfer = !th_has_dedicated_nt(th);
1405 RB_VM_SAVE_MACHINE_CONTEXT(th);
1406 th->status = THREAD_STOPPED_FOREVER;
1407 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_SUSPENDED, th);
1408 thread_sched_wakeup_next_thread(sched, th, can_direct_transfer);
1409 // sleep
1410 thread_sched_wait_running_turn(sched, th, can_direct_transfer);
1411 th->status = THREAD_RUNNABLE;
1412 }
1413 thread_sched_unlock(sched, th);
1414 rb_ractor_lock_self(cr);
1415
1416 ubf_clear(th);
1417
1418 RUBY_DEBUG_LOG("end%s", "");
1419}
1420
1421void
1422rb_ractor_sched_wakeup(rb_ractor_t *r, rb_thread_t *r_th)
1423{
1424 // ractor lock of r is NOT acquired
1425 struct rb_thread_sched *sched = TH_SCHED(r_th);
1426
1427 RUBY_DEBUG_LOG("r:%u th:%d", (unsigned int)rb_ractor_id(r), r_th->serial);
1428
1429 thread_sched_lock(sched, r_th);
1430 {
1431 if (r_th->status == THREAD_STOPPED_FOREVER) {
1432 thread_sched_to_ready_common(sched, r_th, true, false);
1433 }
1434 }
1435 thread_sched_unlock(sched, r_th);
1436}
1437
1438static bool
1439ractor_sched_barrier_completed_p(rb_vm_t *vm)
1440{
1441 RUBY_DEBUG_LOG("run:%u wait:%u", vm->ractor.sched.running_cnt, vm->ractor.sched.barrier_waiting_cnt);
1442 VM_ASSERT(vm->ractor.sched.running_cnt - 1 >= vm->ractor.sched.barrier_waiting_cnt);
1443
1444 return (vm->ractor.sched.running_cnt - vm->ractor.sched.barrier_waiting_cnt) == 1;
1445}
1446
1447void
1448rb_ractor_sched_barrier_start(rb_vm_t *vm, rb_ractor_t *cr)
1449{
1450 VM_ASSERT(cr == GET_RACTOR());
1451 VM_ASSERT(vm->ractor.sync.lock_owner == cr); // VM is locked
1452 VM_ASSERT(!vm->ractor.sched.barrier_waiting);
1453 VM_ASSERT(vm->ractor.sched.barrier_waiting_cnt == 0);
1454 VM_ASSERT(vm->ractor.sched.barrier_ractor == NULL);
1455 VM_ASSERT(vm->ractor.sched.barrier_lock_rec == 0);
1456
1457 RUBY_DEBUG_LOG("start serial:%u", vm->ractor.sched.barrier_serial);
1458
1459 unsigned int lock_rec;
1460
1461 ractor_sched_lock(vm, cr);
1462 {
1463 vm->ractor.sched.barrier_waiting = true;
1464 vm->ractor.sched.barrier_ractor = cr;
1465 vm->ractor.sched.barrier_lock_rec = vm->ractor.sync.lock_rec;
1466
1467 // release VM lock
1468 lock_rec = vm->ractor.sync.lock_rec;
1469 vm->ractor.sync.lock_rec = 0;
1470 vm->ractor.sync.lock_owner = NULL;
1471 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1472
1473 // interrupts all running threads
1474 rb_thread_t *ith;
1475 ccan_list_for_each(&vm->ractor.sched.running_threads, ith, sched.node.running_threads) {
1476 if (ith->ractor != cr) {
1477 RUBY_DEBUG_LOG("barrier request to th:%u", rb_th_serial(ith));
1478 RUBY_VM_SET_VM_BARRIER_INTERRUPT(ith->ec);
1479 }
1480 }
1481
1482 // wait for other ractors
1483 while (!ractor_sched_barrier_completed_p(vm)) {
1484 ractor_sched_set_unlocked(vm, cr);
1485 rb_native_cond_wait(&vm->ractor.sched.barrier_complete_cond, &vm->ractor.sched.lock);
1486 ractor_sched_set_locked(vm, cr);
1487 }
1488
1489 RUBY_DEBUG_LOG("completed seirial:%u", vm->ractor.sched.barrier_serial);
1490
1491 // no other ractors are there
1492 vm->ractor.sched.barrier_serial++;
1493 vm->ractor.sched.barrier_waiting_cnt = 0;
1494 rb_native_cond_broadcast(&vm->ractor.sched.barrier_release_cond);
1495
1496 // acquire VM lock
1497 rb_native_mutex_lock(&vm->ractor.sync.lock);
1498 vm->ractor.sync.lock_rec = lock_rec;
1499 vm->ractor.sync.lock_owner = cr;
1500 }
1501
1502 // do not release ractor_sched_lock and there is no newly added (resumed) thread
1503 // thread_sched_setup_running_threads
1504}
1505
1506// called from vm_lock_leave if the vm_lock used for barrierred
1507void
1508rb_ractor_sched_barrier_end(rb_vm_t *vm, rb_ractor_t *cr)
1509{
1510 RUBY_DEBUG_LOG("serial:%u", (unsigned int)vm->ractor.sched.barrier_serial - 1);
1511 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1512 VM_ASSERT(vm->ractor.sched.barrier_ractor);
1513 VM_ASSERT(vm->ractor.sched.barrier_lock_rec > 0);
1514
1515 vm->ractor.sched.barrier_waiting = false;
1516 vm->ractor.sched.barrier_ractor = NULL;
1517 vm->ractor.sched.barrier_lock_rec = 0;
1518 ractor_sched_unlock(vm, cr);
1519}
1520
1521static void
1522ractor_sched_barrier_join_signal_locked(rb_vm_t *vm)
1523{
1524 if (ractor_sched_barrier_completed_p(vm)) {
1525 rb_native_cond_signal(&vm->ractor.sched.barrier_complete_cond);
1526 }
1527}
1528
1529static void
1530ractor_sched_barrier_join_wait_locked(rb_vm_t *vm, rb_thread_t *th)
1531{
1532 VM_ASSERT(vm->ractor.sched.barrier_waiting);
1533
1534 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1535
1536 while (vm->ractor.sched.barrier_serial == barrier_serial) {
1537 RUBY_DEBUG_LOG("sleep serial:%u", barrier_serial);
1538 RB_VM_SAVE_MACHINE_CONTEXT(th);
1539
1540 rb_ractor_t *cr = th->ractor;
1541 ractor_sched_set_unlocked(vm, cr);
1542 rb_native_cond_wait(&vm->ractor.sched.barrier_release_cond, &vm->ractor.sched.lock);
1543 ractor_sched_set_locked(vm, cr);
1544
1545 RUBY_DEBUG_LOG("wakeup serial:%u", barrier_serial);
1546 }
1547}
1548
1549void
1550rb_ractor_sched_barrier_join(rb_vm_t *vm, rb_ractor_t *cr)
1551{
1552 VM_ASSERT(cr->threads.sched.running != NULL); // running ractor
1553 VM_ASSERT(cr == GET_RACTOR());
1554 VM_ASSERT(vm->ractor.sync.lock_owner == NULL); // VM is locked, but owner == NULL
1555 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1556
1557#if USE_RUBY_DEBUG_LOG || VM_CHECK_MODE > 0
1558 unsigned int barrier_serial = vm->ractor.sched.barrier_serial;
1559#endif
1560
1561 RUBY_DEBUG_LOG("join");
1562
1563 rb_native_mutex_unlock(&vm->ractor.sync.lock);
1564 {
1565 VM_ASSERT(vm->ractor.sched.barrier_waiting); // VM needs barrier sync
1566 VM_ASSERT(vm->ractor.sched.barrier_serial == barrier_serial);
1567
1568 ractor_sched_lock(vm, cr);
1569 {
1570 // running_cnt
1571 vm->ractor.sched.barrier_waiting_cnt++;
1572 RUBY_DEBUG_LOG("waiting_cnt:%u serial:%u", vm->ractor.sched.barrier_waiting_cnt, barrier_serial);
1573
1574 ractor_sched_barrier_join_signal_locked(vm);
1575 ractor_sched_barrier_join_wait_locked(vm, cr->threads.sched.running);
1576 }
1577 ractor_sched_unlock(vm, cr);
1578 }
1579
1580 rb_native_mutex_lock(&vm->ractor.sync.lock);
1581 // VM locked here
1582}
1583
1584#if 0
1585// TODO
1586
1587static void clear_thread_cache_altstack(void);
1588
1589static void
1590rb_thread_sched_destroy(struct rb_thread_sched *sched)
1591{
1592 /*
1593 * only called once at VM shutdown (not atfork), another thread
1594 * may still grab vm->gvl.lock when calling gvl_release at
1595 * the end of thread_start_func_2
1596 */
1597 if (0) {
1598 rb_native_mutex_destroy(&sched->lock);
1599 }
1600 clear_thread_cache_altstack();
1601}
1602#endif
1603
1604#ifdef RB_THREAD_T_HAS_NATIVE_ID
1605static int
1606get_native_thread_id(void)
1607{
1608#ifdef __linux__
1609 return (int)syscall(SYS_gettid);
1610#elif defined(__FreeBSD__)
1611 return pthread_getthreadid_np();
1612#endif
1613}
1614#endif
1615
1616#if defined(HAVE_WORKING_FORK)
1617void rb_internal_thread_event_hooks_rw_lock_atfork(void);
1618
1619static void
1620thread_sched_atfork(struct rb_thread_sched *sched)
1621{
1622 current_fork_gen++;
1623 rb_thread_sched_init(sched, true);
1624 rb_thread_t *th = GET_THREAD();
1625 rb_vm_t *vm = GET_VM();
1626
1627 if (th_has_dedicated_nt(th)) {
1628 vm->ractor.sched.snt_cnt = 0;
1629 }
1630 else {
1631 vm->ractor.sched.snt_cnt = 1;
1632 }
1633 vm->ractor.sched.running_cnt = 0;
1634
1635 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1636#if VM_CHECK_MODE > 0
1637 vm->ractor.sched.lock_owner = NULL;
1638 vm->ractor.sched.locked = false;
1639#endif
1640
1641 // rb_native_cond_destroy(&vm->ractor.sched.cond);
1642 rb_native_cond_initialize(&vm->ractor.sched.cond);
1643 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1644 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1645
1646 ccan_list_head_init(&vm->ractor.sched.grq);
1647 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1648 ccan_list_head_init(&vm->ractor.sched.running_threads);
1649
1650 rb_internal_thread_event_hooks_rw_lock_atfork();
1651
1652 VM_ASSERT(sched->is_running);
1653 sched->is_running_timeslice = false;
1654
1655 if (sched->running != th) {
1656 thread_sched_to_running(sched, th);
1657 }
1658 else {
1659 thread_sched_setup_running_threads(sched, th->ractor, vm, th, NULL, NULL);
1660 }
1661
1662#ifdef RB_THREAD_T_HAS_NATIVE_ID
1663 if (th->nt) {
1664 th->nt->tid = get_native_thread_id();
1665 }
1666#endif
1667}
1668
1669#endif
1670
1671#ifdef RB_THREAD_LOCAL_SPECIFIER
1672static RB_THREAD_LOCAL_SPECIFIER rb_thread_t *ruby_native_thread;
1673#else
1674static pthread_key_t ruby_native_thread_key;
1675#endif
1676
1677static void
1678null_func(int i)
1679{
1680 /* null */
1681 // This function can be called from signal handler
1682 // RUBY_DEBUG_LOG("i:%d", i);
1683}
1684
1685rb_thread_t *
1686ruby_thread_from_native(void)
1687{
1688#ifdef RB_THREAD_LOCAL_SPECIFIER
1689 return ruby_native_thread;
1690#else
1691 return pthread_getspecific(ruby_native_thread_key);
1692#endif
1693}
1694
1695int
1696ruby_thread_set_native(rb_thread_t *th)
1697{
1698 if (th) {
1699#ifdef USE_UBF_LIST
1700 ccan_list_node_init(&th->sched.node.ubf);
1701#endif
1702 }
1703
1704 // setup TLS
1705
1706 if (th && th->ec) {
1707 rb_ractor_set_current_ec(th->ractor, th->ec);
1708 }
1709#ifdef RB_THREAD_LOCAL_SPECIFIER
1710 ruby_native_thread = th;
1711 return 1;
1712#else
1713 return pthread_setspecific(ruby_native_thread_key, th) == 0;
1714#endif
1715}
1716
1717static void native_thread_setup(struct rb_native_thread *nt);
1718static void native_thread_setup_on_thread(struct rb_native_thread *nt);
1719
1720void
1721Init_native_thread(rb_thread_t *main_th)
1722{
1723#if defined(HAVE_PTHREAD_CONDATTR_SETCLOCK)
1724 if (condattr_monotonic) {
1725 int r = pthread_condattr_init(condattr_monotonic);
1726 if (r == 0) {
1727 r = pthread_condattr_setclock(condattr_monotonic, CLOCK_MONOTONIC);
1728 }
1729 if (r) condattr_monotonic = NULL;
1730 }
1731#endif
1732
1733#ifndef RB_THREAD_LOCAL_SPECIFIER
1734 if (pthread_key_create(&ruby_native_thread_key, 0) == EAGAIN) {
1735 rb_bug("pthread_key_create failed (ruby_native_thread_key)");
1736 }
1737 if (pthread_key_create(&ruby_current_ec_key, 0) == EAGAIN) {
1738 rb_bug("pthread_key_create failed (ruby_current_ec_key)");
1739 }
1740#endif
1741 ruby_posix_signal(SIGVTALRM, null_func);
1742
1743 // setup vm
1744 rb_vm_t *vm = main_th->vm;
1745 rb_native_mutex_initialize(&vm->ractor.sched.lock);
1746 rb_native_cond_initialize(&vm->ractor.sched.cond);
1747 rb_native_cond_initialize(&vm->ractor.sched.barrier_complete_cond);
1748 rb_native_cond_initialize(&vm->ractor.sched.barrier_release_cond);
1749
1750 ccan_list_head_init(&vm->ractor.sched.grq);
1751 ccan_list_head_init(&vm->ractor.sched.timeslice_threads);
1752 ccan_list_head_init(&vm->ractor.sched.running_threads);
1753
1754 // setup main thread
1755 main_th->nt->thread_id = pthread_self();
1756 main_th->nt->serial = 1;
1757#ifdef RUBY_NT_SERIAL
1758 ruby_nt_serial = 1;
1759#endif
1760 ruby_thread_set_native(main_th);
1761 native_thread_setup(main_th->nt);
1762 native_thread_setup_on_thread(main_th->nt);
1763
1764 TH_SCHED(main_th)->running = main_th;
1765 main_th->has_dedicated_nt = 1;
1766
1767 thread_sched_setup_running_threads(TH_SCHED(main_th), main_th->ractor, vm, main_th, NULL, NULL);
1768
1769 // setup main NT
1770 main_th->nt->dedicated = 1;
1771 main_th->nt->vm = vm;
1772
1773 // setup mn
1774 vm->ractor.sched.dnt_cnt = 1;
1775}
1776
1777extern int ruby_mn_threads_enabled;
1778
1779void
1780ruby_mn_threads_params(void)
1781{
1782 rb_vm_t *vm = GET_VM();
1783 rb_ractor_t *main_ractor = GET_RACTOR();
1784
1785 const char *mn_threads_cstr = getenv("RUBY_MN_THREADS");
1786 bool enable_mn_threads = false;
1787
1788 if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) {
1789 // enabled
1790 ruby_mn_threads_enabled = 1;
1791 }
1792 main_ractor->threads.sched.enable_mn_threads = enable_mn_threads;
1793
1794 const char *max_cpu_cstr = getenv("RUBY_MAX_CPU");
1795 const int default_max_cpu = 8; // TODO: CPU num?
1796 int max_cpu = default_max_cpu;
1797
1798 if (USE_MN_THREADS && max_cpu_cstr) {
1799 int given_max_cpu = atoi(max_cpu_cstr);
1800 if (given_max_cpu > 0) {
1801 max_cpu = given_max_cpu;
1802 }
1803 }
1804
1805 vm->ractor.sched.max_cpu = max_cpu;
1806}
1807
1808static void
1809native_thread_dedicated_inc(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1810{
1811 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated + 1);
1812
1813 if (nt->dedicated == 0) {
1814 ractor_sched_lock(vm, cr);
1815 {
1816 vm->ractor.sched.snt_cnt--;
1817 vm->ractor.sched.dnt_cnt++;
1818 }
1819 ractor_sched_unlock(vm, cr);
1820 }
1821
1822 nt->dedicated++;
1823}
1824
1825static void
1826native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_thread *nt)
1827{
1828 RUBY_DEBUG_LOG("nt:%d %d->%d", nt->serial, nt->dedicated, nt->dedicated - 1);
1829 VM_ASSERT(nt->dedicated > 0);
1830 nt->dedicated--;
1831
1832 if (nt->dedicated == 0) {
1833 ractor_sched_lock(vm, cr);
1834 {
1835 nt->vm->ractor.sched.snt_cnt++;
1836 nt->vm->ractor.sched.dnt_cnt--;
1837 }
1838 ractor_sched_unlock(vm, cr);
1839 }
1840}
1841
1842static void
1843native_thread_assign(struct rb_native_thread *nt, rb_thread_t *th)
1844{
1845#if USE_RUBY_DEBUG_LOG
1846 if (nt) {
1847 if (th->nt) {
1848 RUBY_DEBUG_LOG("th:%d nt:%d->%d", (int)th->serial, (int)th->nt->serial, (int)nt->serial);
1849 }
1850 else {
1851 RUBY_DEBUG_LOG("th:%d nt:NULL->%d", (int)th->serial, (int)nt->serial);
1852 }
1853 }
1854 else {
1855 if (th->nt) {
1856 RUBY_DEBUG_LOG("th:%d nt:%d->NULL", (int)th->serial, (int)th->nt->serial);
1857 }
1858 else {
1859 RUBY_DEBUG_LOG("th:%d nt:NULL->NULL", (int)th->serial);
1860 }
1861 }
1862#endif
1863
1864 th->nt = nt;
1865}
1866
1867static void
1868native_thread_destroy_atfork(struct rb_native_thread *nt)
1869{
1870 if (nt) {
1871 /* We can't call rb_native_cond_destroy here because according to the
1872 * specs of pthread_cond_destroy:
1873 *
1874 * Attempting to destroy a condition variable upon which other threads
1875 * are currently blocked results in undefined behavior.
1876 *
1877 * Specifically, glibc's pthread_cond_destroy waits on all the other
1878 * listeners. Since after forking all the threads are dead, the condition
1879 * variable's listeners will never wake up, so it will hang forever.
1880 */
1881
1882 RB_ALTSTACK_FREE(nt->altstack);
1883 ruby_xfree(nt->nt_context);
1884 ruby_xfree(nt);
1885 }
1886}
1887
1888static void
1889native_thread_destroy(struct rb_native_thread *nt)
1890{
1891 if (nt) {
1892 rb_native_cond_destroy(&nt->cond.readyq);
1893
1894 if (&nt->cond.readyq != &nt->cond.intr) {
1895 rb_native_cond_destroy(&nt->cond.intr);
1896 }
1897
1898 native_thread_destroy_atfork(nt);
1899 }
1900}
1901
1902#if defined HAVE_PTHREAD_GETATTR_NP || defined HAVE_PTHREAD_ATTR_GET_NP
1903#define STACKADDR_AVAILABLE 1
1904#elif defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP
1905#define STACKADDR_AVAILABLE 1
1906#undef MAINSTACKADDR_AVAILABLE
1907#define MAINSTACKADDR_AVAILABLE 1
1908void *pthread_get_stackaddr_np(pthread_t);
1909size_t pthread_get_stacksize_np(pthread_t);
1910#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1911#define STACKADDR_AVAILABLE 1
1912#elif defined HAVE_PTHREAD_GETTHRDS_NP
1913#define STACKADDR_AVAILABLE 1
1914#elif defined __HAIKU__
1915#define STACKADDR_AVAILABLE 1
1916#endif
1917
1918#ifndef MAINSTACKADDR_AVAILABLE
1919# ifdef STACKADDR_AVAILABLE
1920# define MAINSTACKADDR_AVAILABLE 1
1921# else
1922# define MAINSTACKADDR_AVAILABLE 0
1923# endif
1924#endif
1925#if MAINSTACKADDR_AVAILABLE && !defined(get_main_stack)
1926# define get_main_stack(addr, size) get_stack(addr, size)
1927#endif
1928
1929#ifdef STACKADDR_AVAILABLE
1930/*
1931 * Get the initial address and size of current thread's stack
1932 */
1933static int
1934get_stack(void **addr, size_t *size)
1935{
1936#define CHECK_ERR(expr) \
1937 {int err = (expr); if (err) return err;}
1938#ifdef HAVE_PTHREAD_GETATTR_NP /* Linux */
1939 pthread_attr_t attr;
1940 size_t guard = 0;
1941 STACK_GROW_DIR_DETECTION;
1942 CHECK_ERR(pthread_getattr_np(pthread_self(), &attr));
1943# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1944 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1945 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1946# else
1947 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1948 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1949# endif
1950# ifdef HAVE_PTHREAD_ATTR_GETGUARDSIZE
1951 CHECK_ERR(pthread_attr_getguardsize(&attr, &guard));
1952# else
1953 guard = getpagesize();
1954# endif
1955 *size -= guard;
1956 pthread_attr_destroy(&attr);
1957#elif defined HAVE_PTHREAD_ATTR_GET_NP /* FreeBSD, DragonFly BSD, NetBSD */
1958 pthread_attr_t attr;
1959 CHECK_ERR(pthread_attr_init(&attr));
1960 CHECK_ERR(pthread_attr_get_np(pthread_self(), &attr));
1961# ifdef HAVE_PTHREAD_ATTR_GETSTACK
1962 CHECK_ERR(pthread_attr_getstack(&attr, addr, size));
1963# else
1964 CHECK_ERR(pthread_attr_getstackaddr(&attr, addr));
1965 CHECK_ERR(pthread_attr_getstacksize(&attr, size));
1966# endif
1967 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1968 pthread_attr_destroy(&attr);
1969#elif (defined HAVE_PTHREAD_GET_STACKADDR_NP && defined HAVE_PTHREAD_GET_STACKSIZE_NP) /* MacOS X */
1970 pthread_t th = pthread_self();
1971 *addr = pthread_get_stackaddr_np(th);
1972 *size = pthread_get_stacksize_np(th);
1973#elif defined HAVE_THR_STKSEGMENT || defined HAVE_PTHREAD_STACKSEG_NP
1974 stack_t stk;
1975# if defined HAVE_THR_STKSEGMENT /* Solaris */
1976 CHECK_ERR(thr_stksegment(&stk));
1977# else /* OpenBSD */
1978 CHECK_ERR(pthread_stackseg_np(pthread_self(), &stk));
1979# endif
1980 *addr = stk.ss_sp;
1981 *size = stk.ss_size;
1982#elif defined HAVE_PTHREAD_GETTHRDS_NP /* AIX */
1983 pthread_t th = pthread_self();
1984 struct __pthrdsinfo thinfo;
1985 char reg[256];
1986 int regsiz=sizeof(reg);
1987 CHECK_ERR(pthread_getthrds_np(&th, PTHRDSINFO_QUERY_ALL,
1988 &thinfo, sizeof(thinfo),
1989 &reg, &regsiz));
1990 *addr = thinfo.__pi_stackaddr;
1991 /* Must not use thinfo.__pi_stacksize for size.
1992 It is around 3KB smaller than the correct size
1993 calculated by thinfo.__pi_stackend - thinfo.__pi_stackaddr. */
1994 *size = thinfo.__pi_stackend - thinfo.__pi_stackaddr;
1995 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
1996#elif defined __HAIKU__
1997 thread_info info;
1998 STACK_GROW_DIR_DETECTION;
1999 CHECK_ERR(get_thread_info(find_thread(NULL), &info));
2000 *addr = info.stack_base;
2001 *size = (uintptr_t)info.stack_end - (uintptr_t)info.stack_base;
2002 STACK_DIR_UPPER((void)0, (void)(*addr = (char *)*addr + *size));
2003#else
2004#error STACKADDR_AVAILABLE is defined but not implemented.
2005#endif
2006 return 0;
2007#undef CHECK_ERR
2008}
2009#endif
2010
2011static struct {
2012 rb_nativethread_id_t id;
2013 size_t stack_maxsize;
2014 VALUE *stack_start;
2015} native_main_thread;
2016
2017#ifdef STACK_END_ADDRESS
2018extern void *STACK_END_ADDRESS;
2019#endif
2020
2021enum {
2022 RUBY_STACK_SPACE_LIMIT = 1024 * 1024, /* 1024KB */
2023 RUBY_STACK_SPACE_RATIO = 5
2024};
2025
2026static size_t
2027space_size(size_t stack_size)
2028{
2029 size_t space_size = stack_size / RUBY_STACK_SPACE_RATIO;
2030 if (space_size > RUBY_STACK_SPACE_LIMIT) {
2031 return RUBY_STACK_SPACE_LIMIT;
2032 }
2033 else {
2034 return space_size;
2035 }
2036}
2037
2038static void
2039native_thread_init_main_thread_stack(void *addr)
2040{
2041 native_main_thread.id = pthread_self();
2042#ifdef RUBY_ASAN_ENABLED
2043 addr = asan_get_real_stack_addr((void *)addr);
2044#endif
2045
2046#if MAINSTACKADDR_AVAILABLE
2047 if (native_main_thread.stack_maxsize) return;
2048 {
2049 void* stackaddr;
2050 size_t size;
2051 if (get_main_stack(&stackaddr, &size) == 0) {
2052 native_main_thread.stack_maxsize = size;
2053 native_main_thread.stack_start = stackaddr;
2054 goto bound_check;
2055 }
2056 }
2057#endif
2058#ifdef STACK_END_ADDRESS
2059 native_main_thread.stack_start = STACK_END_ADDRESS;
2060#else
2061 if (!native_main_thread.stack_start ||
2062 STACK_UPPER((VALUE *)(void *)&addr,
2063 native_main_thread.stack_start > (VALUE *)addr,
2064 native_main_thread.stack_start < (VALUE *)addr)) {
2065 native_main_thread.stack_start = (VALUE *)addr;
2066 }
2067#endif
2068 {
2069#if defined(HAVE_GETRLIMIT)
2070#if defined(PTHREAD_STACK_DEFAULT)
2071# if PTHREAD_STACK_DEFAULT < RUBY_STACK_SPACE*5
2072# error "PTHREAD_STACK_DEFAULT is too small"
2073# endif
2074 size_t size = PTHREAD_STACK_DEFAULT;
2075#else
2076 size_t size = RUBY_VM_THREAD_VM_STACK_SIZE;
2077#endif
2078 size_t space;
2079 int pagesize = getpagesize();
2080 struct rlimit rlim;
2081 STACK_GROW_DIR_DETECTION;
2082 if (getrlimit(RLIMIT_STACK, &rlim) == 0) {
2083 size = (size_t)rlim.rlim_cur;
2084 }
2085 addr = native_main_thread.stack_start;
2086 if (IS_STACK_DIR_UPPER()) {
2087 space = ((size_t)((char *)addr + size) / pagesize) * pagesize - (size_t)addr;
2088 }
2089 else {
2090 space = (size_t)addr - ((size_t)((char *)addr - size) / pagesize + 1) * pagesize;
2091 }
2092 native_main_thread.stack_maxsize = space;
2093#endif
2094 }
2095
2096#if MAINSTACKADDR_AVAILABLE
2097 bound_check:
2098#endif
2099 /* If addr is out of range of main-thread stack range estimation, */
2100 /* it should be on co-routine (alternative stack). [Feature #2294] */
2101 {
2102 void *start, *end;
2103 STACK_GROW_DIR_DETECTION;
2104
2105 if (IS_STACK_DIR_UPPER()) {
2106 start = native_main_thread.stack_start;
2107 end = (char *)native_main_thread.stack_start + native_main_thread.stack_maxsize;
2108 }
2109 else {
2110 start = (char *)native_main_thread.stack_start - native_main_thread.stack_maxsize;
2111 end = native_main_thread.stack_start;
2112 }
2113
2114 if ((void *)addr < start || (void *)addr > end) {
2115 /* out of range */
2116 native_main_thread.stack_start = (VALUE *)addr;
2117 native_main_thread.stack_maxsize = 0; /* unknown */
2118 }
2119 }
2120}
2121
2122#define CHECK_ERR(expr) \
2123 {int err = (expr); if (err) {rb_bug_errno(#expr, err);}}
2124
2125static int
2126native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame)
2127{
2128 rb_nativethread_id_t curr = pthread_self();
2129#ifdef RUBY_ASAN_ENABLED
2130 local_in_parent_frame = asan_get_real_stack_addr(local_in_parent_frame);
2131 th->ec->machine.asan_fake_stack_handle = asan_get_thread_fake_stack_handle();
2132#endif
2133
2134 if (!native_main_thread.id) {
2135 /* This thread is the first thread, must be the main thread -
2136 * configure the native_main_thread object */
2137 native_thread_init_main_thread_stack(local_in_parent_frame);
2138 }
2139
2140 if (pthread_equal(curr, native_main_thread.id)) {
2141 th->ec->machine.stack_start = native_main_thread.stack_start;
2142 th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize;
2143 }
2144 else {
2145#ifdef STACKADDR_AVAILABLE
2146 if (th_has_dedicated_nt(th)) {
2147 void *start;
2148 size_t size;
2149
2150 if (get_stack(&start, &size) == 0) {
2151 uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame;
2152 th->ec->machine.stack_start = local_in_parent_frame;
2153 th->ec->machine.stack_maxsize = size - diff;
2154 }
2155 }
2156#else
2157 rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread");
2158#endif
2159 }
2160
2161 return 0;
2162}
2163
2164struct nt_param {
2165 rb_vm_t *vm;
2166 struct rb_native_thread *nt;
2167};
2168
2169static void *
2170nt_start(void *ptr);
2171
2172static int
2173native_thread_create0(struct rb_native_thread *nt)
2174{
2175 int err = 0;
2176 pthread_attr_t attr;
2177
2178 const size_t stack_size = nt->vm->default_params.thread_machine_stack_size;
2179 const size_t space = space_size(stack_size);
2180
2181 nt->machine_stack_maxsize = stack_size - space;
2182
2183#ifdef USE_SIGALTSTACK
2184 nt->altstack = rb_allocate_sigaltstack();
2185#endif
2186
2187 CHECK_ERR(pthread_attr_init(&attr));
2188
2189# ifdef PTHREAD_STACK_MIN
2190 RUBY_DEBUG_LOG("stack size: %lu", (unsigned long)stack_size);
2191 CHECK_ERR(pthread_attr_setstacksize(&attr, stack_size));
2192# endif
2193
2194# ifdef HAVE_PTHREAD_ATTR_SETINHERITSCHED
2195 CHECK_ERR(pthread_attr_setinheritsched(&attr, PTHREAD_INHERIT_SCHED));
2196# endif
2197 CHECK_ERR(pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED));
2198
2199 err = pthread_create(&nt->thread_id, &attr, nt_start, nt);
2200
2201 RUBY_DEBUG_LOG("nt:%d err:%d", (int)nt->serial, err);
2202
2203 CHECK_ERR(pthread_attr_destroy(&attr));
2204
2205 return err;
2206}
2207
2208static void
2209native_thread_setup(struct rb_native_thread *nt)
2210{
2211 // init cond
2212 rb_native_cond_initialize(&nt->cond.readyq);
2213
2214 if (&nt->cond.readyq != &nt->cond.intr) {
2215 rb_native_cond_initialize(&nt->cond.intr);
2216 }
2217}
2218
2219static void
2220native_thread_setup_on_thread(struct rb_native_thread *nt)
2221{
2222 // init tid
2223#ifdef RB_THREAD_T_HAS_NATIVE_ID
2224 nt->tid = get_native_thread_id();
2225#endif
2226
2227 // init signal handler
2228 RB_ALTSTACK_INIT(nt->altstack, nt->altstack);
2229}
2230
2231static struct rb_native_thread *
2232native_thread_alloc(void)
2233{
2234 struct rb_native_thread *nt = ZALLOC(struct rb_native_thread);
2235 native_thread_setup(nt);
2236
2237#if USE_MN_THREADS
2238 nt->nt_context = ruby_xmalloc(sizeof(struct coroutine_context));
2239#endif
2240
2241#if USE_RUBY_DEBUG_LOG
2242 static rb_atomic_t nt_serial = 2;
2243 nt->serial = RUBY_ATOMIC_FETCH_ADD(nt_serial, 1);
2244#endif
2245 return nt;
2246}
2247
2248static int
2249native_thread_create_dedicated(rb_thread_t *th)
2250{
2251 th->nt = native_thread_alloc();
2252 th->nt->vm = th->vm;
2253 th->nt->running_thread = th;
2254 th->nt->dedicated = 1;
2255
2256 // vm stack
2257 size_t vm_stack_word_size = th->vm->default_params.thread_vm_stack_size / sizeof(VALUE);
2258 void *vm_stack = ruby_xmalloc(vm_stack_word_size * sizeof(VALUE));
2259 th->sched.malloc_stack = true;
2260 rb_ec_initialize_vm_stack(th->ec, vm_stack, vm_stack_word_size);
2261 th->sched.context_stack = vm_stack;
2262
2263
2264 int err = native_thread_create0(th->nt);
2265 if (!err) {
2266 // setup
2267 thread_sched_to_ready(TH_SCHED(th), th);
2268 }
2269 return err;
2270}
2271
2272static void
2273call_thread_start_func_2(rb_thread_t *th)
2274{
2275 /* Capture the address of a local in this stack frame to mark the beginning of the
2276 machine stack for this thread. This is required even if we can tell the real
2277 stack beginning from the pthread API in native_thread_init_stack, because
2278 glibc stores some of its own data on the stack before calling into user code
2279 on a new thread, and replacing that data on fiber-switch would break it (see
2280 bug #13887) */
2281 VALUE stack_start = 0;
2282 VALUE *stack_start_addr = asan_get_real_stack_addr(&stack_start);
2283
2284 native_thread_init_stack(th, stack_start_addr);
2285 thread_start_func_2(th, th->ec->machine.stack_start);
2286}
2287
2288static void *
2289nt_start(void *ptr)
2290{
2291 struct rb_native_thread *nt = (struct rb_native_thread *)ptr;
2292 rb_vm_t *vm = nt->vm;
2293
2294 native_thread_setup_on_thread(nt);
2295
2296 // init tid
2297#ifdef RB_THREAD_T_HAS_NATIVE_ID
2298 nt->tid = get_native_thread_id();
2299#endif
2300
2301#if USE_RUBY_DEBUG_LOG && defined(RUBY_NT_SERIAL)
2302 ruby_nt_serial = nt->serial;
2303#endif
2304
2305 RUBY_DEBUG_LOG("nt:%u", nt->serial);
2306
2307 if (!nt->dedicated) {
2308 coroutine_initialize_main(nt->nt_context);
2309 }
2310
2311 while (1) {
2312 if (nt->dedicated) {
2313 // wait running turn
2314 rb_thread_t *th = nt->running_thread;
2315 struct rb_thread_sched *sched = TH_SCHED(th);
2316
2317 RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th));
2318 ruby_thread_set_native(th);
2319
2320 thread_sched_lock(sched, th);
2321 {
2322 if (sched->running == th) {
2323 thread_sched_add_running_thread(sched, th);
2324 }
2325 thread_sched_wait_running_turn(sched, th, false);
2326 }
2327 thread_sched_unlock(sched, th);
2328
2329 // start threads
2330 call_thread_start_func_2(th);
2331 break; // TODO: allow to change to the SNT
2332 }
2333 else {
2334 RUBY_DEBUG_LOG("check next");
2335 rb_ractor_t *r = ractor_sched_deq(vm, NULL);
2336
2337 if (r) {
2338 struct rb_thread_sched *sched = &r->threads.sched;
2339
2340 thread_sched_lock(sched, NULL);
2341 {
2342 rb_thread_t *next_th = sched->running;
2343
2344 if (next_th && next_th->nt == NULL) {
2345 RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial);
2346 thread_sched_switch0(nt->nt_context, next_th, nt, false);
2347 }
2348 else {
2349 RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th);
2350 }
2351 }
2352 thread_sched_unlock(sched, NULL);
2353 }
2354 else {
2355 // timeout -> deleted.
2356 break;
2357 }
2358
2359 if (nt->dedicated) {
2360 // SNT becomes DNT while running
2361 break;
2362 }
2363 }
2364 }
2365
2366 return NULL;
2367}
2368
2369static int native_thread_create_shared(rb_thread_t *th);
2370
2371#if USE_MN_THREADS
2372static void nt_free_stack(void *mstack);
2373#endif
2374
2375void
2376rb_threadptr_remove(rb_thread_t *th)
2377{
2378#if USE_MN_THREADS
2379 if (th->sched.malloc_stack) {
2380 // dedicated
2381 return;
2382 }
2383 else {
2384 rb_vm_t *vm = th->vm;
2385 th->sched.finished = false;
2386
2387 RB_VM_LOCKING() {
2388 ccan_list_add(&vm->ractor.sched.zombie_threads, &th->sched.node.zombie_threads);
2389 }
2390 }
2391#endif
2392}
2393
2394void
2395rb_threadptr_sched_free(rb_thread_t *th)
2396{
2397#if USE_MN_THREADS
2398 if (th->sched.malloc_stack) {
2399 // has dedicated
2400 ruby_xfree(th->sched.context_stack);
2401 native_thread_destroy(th->nt);
2402 }
2403 else {
2404 nt_free_stack(th->sched.context_stack);
2405 // TODO: how to free nt and nt->altstack?
2406 }
2407
2408 ruby_xfree(th->sched.context);
2409 th->sched.context = NULL;
2410 // VM_ASSERT(th->sched.context == NULL);
2411#else
2412 ruby_xfree(th->sched.context_stack);
2413 native_thread_destroy(th->nt);
2414#endif
2415
2416 th->nt = NULL;
2417}
2418
2419void
2420rb_thread_sched_mark_zombies(rb_vm_t *vm)
2421{
2422 if (!ccan_list_empty(&vm->ractor.sched.zombie_threads)) {
2423 rb_thread_t *zombie_th, *next_zombie_th;
2424 ccan_list_for_each_safe(&vm->ractor.sched.zombie_threads, zombie_th, next_zombie_th, sched.node.zombie_threads) {
2425 if (zombie_th->sched.finished) {
2426 ccan_list_del_init(&zombie_th->sched.node.zombie_threads);
2427 }
2428 else {
2429 rb_gc_mark(zombie_th->self);
2430 }
2431 }
2432 }
2433}
2434
2435static int
2436native_thread_create(rb_thread_t *th)
2437{
2438 VM_ASSERT(th->nt == 0);
2439 RUBY_DEBUG_LOG("th:%d has_dnt:%d", th->serial, th->has_dedicated_nt);
2440 RB_INTERNAL_THREAD_HOOK(RUBY_INTERNAL_THREAD_EVENT_STARTED, th);
2441
2442 if (!th->ractor->threads.sched.enable_mn_threads) {
2443 th->has_dedicated_nt = 1;
2444 }
2445
2446 if (th->has_dedicated_nt) {
2447 return native_thread_create_dedicated(th);
2448 }
2449 else {
2450 return native_thread_create_shared(th);
2451 }
2452}
2453
2454#if USE_NATIVE_THREAD_PRIORITY
2455
2456static void
2457native_thread_apply_priority(rb_thread_t *th)
2458{
2459#if defined(_POSIX_PRIORITY_SCHEDULING) && (_POSIX_PRIORITY_SCHEDULING > 0)
2460 struct sched_param sp;
2461 int policy;
2462 int priority = 0 - th->priority;
2463 int max, min;
2464 pthread_getschedparam(th->nt->thread_id, &policy, &sp);
2465 max = sched_get_priority_max(policy);
2466 min = sched_get_priority_min(policy);
2467
2468 if (min > priority) {
2469 priority = min;
2470 }
2471 else if (max < priority) {
2472 priority = max;
2473 }
2474
2475 sp.sched_priority = priority;
2476 pthread_setschedparam(th->nt->thread_id, policy, &sp);
2477#else
2478 /* not touched */
2479#endif
2480}
2481
2482#endif /* USE_NATIVE_THREAD_PRIORITY */
2483
2484static int
2485native_fd_select(int n, rb_fdset_t *readfds, rb_fdset_t *writefds, rb_fdset_t *exceptfds, struct timeval *timeout, rb_thread_t *th)
2486{
2487 return rb_fd_select(n, readfds, writefds, exceptfds, timeout);
2488}
2489
2490static void
2491ubf_pthread_cond_signal(void *ptr)
2492{
2493 rb_thread_t *th = (rb_thread_t *)ptr;
2494 RUBY_DEBUG_LOG("th:%u on nt:%d", rb_th_serial(th), (int)th->nt->serial);
2495 rb_native_cond_signal(&th->nt->cond.intr);
2496}
2497
2498static void
2499native_cond_sleep(rb_thread_t *th, rb_hrtime_t *rel)
2500{
2501 rb_nativethread_lock_t *lock = &th->interrupt_lock;
2502 rb_nativethread_cond_t *cond = &th->nt->cond.intr;
2503
2504 /* Solaris cond_timedwait() return EINVAL if an argument is greater than
2505 * current_time + 100,000,000. So cut up to 100,000,000. This is
2506 * considered as a kind of spurious wakeup. The caller to native_sleep
2507 * should care about spurious wakeup.
2508 *
2509 * See also [Bug #1341] [ruby-core:29702]
2510 * http://download.oracle.com/docs/cd/E19683-01/816-0216/6m6ngupgv/index.html
2511 */
2512 const rb_hrtime_t max = (rb_hrtime_t)100000000 * RB_HRTIME_PER_SEC;
2513
2514 THREAD_BLOCKING_BEGIN(th);
2515 {
2516 rb_native_mutex_lock(lock);
2517 th->unblock.func = ubf_pthread_cond_signal;
2518 th->unblock.arg = th;
2519
2520 if (RUBY_VM_INTERRUPTED(th->ec)) {
2521 /* interrupted. return immediate */
2522 RUBY_DEBUG_LOG("interrupted before sleep th:%u", rb_th_serial(th));
2523 }
2524 else {
2525 if (!rel) {
2526 rb_native_cond_wait(cond, lock);
2527 }
2528 else {
2529 rb_hrtime_t end;
2530
2531 if (*rel > max) {
2532 *rel = max;
2533 }
2534
2535 end = native_cond_timeout(cond, *rel);
2536 native_cond_timedwait(cond, lock, &end);
2537 }
2538 }
2539 th->unblock.func = 0;
2540
2541 rb_native_mutex_unlock(lock);
2542 }
2543 THREAD_BLOCKING_END(th);
2544
2545 RUBY_DEBUG_LOG("done th:%u", rb_th_serial(th));
2546}
2547
2548#ifdef USE_UBF_LIST
2549static CCAN_LIST_HEAD(ubf_list_head);
2550static rb_nativethread_lock_t ubf_list_lock = RB_NATIVETHREAD_LOCK_INIT;
2551
2552static void
2553ubf_list_atfork(void)
2554{
2555 ccan_list_head_init(&ubf_list_head);
2556 rb_native_mutex_initialize(&ubf_list_lock);
2557}
2558
2559 RBIMPL_ATTR_MAYBE_UNUSED()
2560static bool
2561ubf_list_contain_p(rb_thread_t *th)
2562{
2563 rb_thread_t *list_th;
2564 ccan_list_for_each(&ubf_list_head, list_th, sched.node.ubf) {
2565 if (list_th == th) return true;
2566 }
2567 return false;
2568}
2569
2570/* The thread 'th' is registered to be trying unblock. */
2571static void
2572register_ubf_list(rb_thread_t *th)
2573{
2574 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2575 struct ccan_list_node *node = &th->sched.node.ubf;
2576
2577 VM_ASSERT(th->unblock.func != NULL);
2578
2579 rb_native_mutex_lock(&ubf_list_lock);
2580 {
2581 // check not connected yet
2582 if (ccan_list_empty((struct ccan_list_head*)node)) {
2583 VM_ASSERT(!ubf_list_contain_p(th));
2584 ccan_list_add(&ubf_list_head, node);
2585 }
2586 }
2587 rb_native_mutex_unlock(&ubf_list_lock);
2588
2589 timer_thread_wakeup();
2590}
2591
2592/* The thread 'th' is unblocked. It no longer need to be registered. */
2593static void
2594unregister_ubf_list(rb_thread_t *th)
2595{
2596 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
2597 struct ccan_list_node *node = &th->sched.node.ubf;
2598
2599 /* we can't allow re-entry into ubf_list_head */
2600 VM_ASSERT(th->unblock.func == NULL);
2601
2602 if (!ccan_list_empty((struct ccan_list_head*)node)) {
2603 rb_native_mutex_lock(&ubf_list_lock);
2604 {
2605 VM_ASSERT(ubf_list_contain_p(th));
2606 ccan_list_del_init(node);
2607 }
2608 rb_native_mutex_unlock(&ubf_list_lock);
2609 }
2610}
2611
2612/*
2613 * send a signal to intent that a target thread return from blocking syscall.
2614 * Maybe any signal is ok, but we chose SIGVTALRM.
2615 */
2616static void
2617ubf_wakeup_thread(rb_thread_t *th)
2618{
2619 RUBY_DEBUG_LOG("th:%u thread_id:%p", rb_th_serial(th), (void *)th->nt->thread_id);
2620
2621 pthread_kill(th->nt->thread_id, SIGVTALRM);
2622}
2623
2624static void
2625ubf_select(void *ptr)
2626{
2627 rb_thread_t *th = (rb_thread_t *)ptr;
2628 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(th));
2629 ubf_wakeup_thread(th);
2630 register_ubf_list(th);
2631}
2632
2633static bool
2634ubf_threads_empty(void)
2635{
2636 return ccan_list_empty(&ubf_list_head) != 0;
2637}
2638
2639static void
2640ubf_wakeup_all_threads(void)
2641{
2642 rb_thread_t *th;
2643 rb_native_mutex_lock(&ubf_list_lock);
2644 {
2645 ccan_list_for_each(&ubf_list_head, th, sched.node.ubf) {
2646 ubf_wakeup_thread(th);
2647 }
2648 }
2649 rb_native_mutex_unlock(&ubf_list_lock);
2650}
2651
2652#else /* USE_UBF_LIST */
2653#define register_ubf_list(th) (void)(th)
2654#define unregister_ubf_list(th) (void)(th)
2655#define ubf_select 0
2656static void ubf_wakeup_all_threads(void) { return; }
2657static bool ubf_threads_empty(void) { return true; }
2658#define ubf_list_atfork() do {} while (0)
2659#endif /* USE_UBF_LIST */
2660
2661#define TT_DEBUG 0
2662#define WRITE_CONST(fd, str) (void)(write((fd),(str),sizeof(str)-1)<0)
2663
2664void
2665rb_thread_wakeup_timer_thread(int sig)
2666{
2667 // This function can be called from signal handlers so that
2668 // pthread_mutex_lock() should not be used.
2669
2670 // wakeup timer thread
2671 timer_thread_wakeup_force();
2672
2673 // interrupt main thread if main thread is available
2674 if (RUBY_ATOMIC_LOAD(system_working)) {
2675 rb_vm_t *vm = GET_VM();
2676 rb_thread_t *main_th = vm->ractor.main_thread;
2677
2678 if (main_th) {
2679 volatile rb_execution_context_t *main_th_ec = ACCESS_ONCE(rb_execution_context_t *, main_th->ec);
2680
2681 if (main_th_ec) {
2682 RUBY_VM_SET_TRAP_INTERRUPT(main_th_ec);
2683
2684 if (vm->ubf_async_safe && main_th->unblock.func) {
2685 (main_th->unblock.func)(main_th->unblock.arg);
2686 }
2687 }
2688 }
2689 }
2690}
2691
2692#define CLOSE_INVALIDATE_PAIR(expr) \
2693 close_invalidate_pair(expr,"close_invalidate: "#expr)
2694static void
2695close_invalidate(int *fdp, const char *msg)
2696{
2697 int fd = *fdp;
2698
2699 *fdp = -1;
2700 if (close(fd) < 0) {
2701 async_bug_fd(msg, errno, fd);
2702 }
2703}
2704
2705static void
2706close_invalidate_pair(int fds[2], const char *msg)
2707{
2708 if (USE_EVENTFD && fds[0] == fds[1]) {
2709 fds[1] = -1; // disable write port first
2710 close_invalidate(&fds[0], msg);
2711 }
2712 else {
2713 close_invalidate(&fds[1], msg);
2714 close_invalidate(&fds[0], msg);
2715 }
2716}
2717
2718static void
2719set_nonblock(int fd)
2720{
2721 int oflags;
2722 int err;
2723
2724 oflags = fcntl(fd, F_GETFL);
2725 if (oflags == -1)
2726 rb_sys_fail(0);
2727 oflags |= O_NONBLOCK;
2728 err = fcntl(fd, F_SETFL, oflags);
2729 if (err == -1)
2730 rb_sys_fail(0);
2731}
2732
2733/* communication pipe with timer thread and signal handler */
2734static void
2735setup_communication_pipe_internal(int pipes[2])
2736{
2737 int err;
2738
2739 if (pipes[0] > 0 || pipes[1] > 0) {
2740 VM_ASSERT(pipes[0] > 0);
2741 VM_ASSERT(pipes[1] > 0);
2742 return;
2743 }
2744
2745 /*
2746 * Don't bother with eventfd on ancient Linux 2.6.22..2.6.26 which were
2747 * missing EFD_* flags, they can fall back to pipe
2748 */
2749#if USE_EVENTFD && defined(EFD_NONBLOCK) && defined(EFD_CLOEXEC)
2750 pipes[0] = pipes[1] = eventfd(0, EFD_NONBLOCK|EFD_CLOEXEC);
2751
2752 if (pipes[0] >= 0) {
2753 rb_update_max_fd(pipes[0]);
2754 return;
2755 }
2756#endif
2757
2758 err = rb_cloexec_pipe(pipes);
2759 if (err != 0) {
2760 rb_bug("can not create communication pipe");
2761 }
2762 rb_update_max_fd(pipes[0]);
2763 rb_update_max_fd(pipes[1]);
2764 set_nonblock(pipes[0]);
2765 set_nonblock(pipes[1]);
2766}
2767
2768#if !defined(SET_CURRENT_THREAD_NAME) && defined(__linux__) && defined(PR_SET_NAME)
2769# define SET_CURRENT_THREAD_NAME(name) prctl(PR_SET_NAME, name)
2770#endif
2771
2772enum {
2773 THREAD_NAME_MAX =
2774#if defined(__linux__)
2775 16
2776#elif defined(__APPLE__)
2777/* Undocumented, and main thread seems unlimited */
2778 64
2779#else
2780 16
2781#endif
2782};
2783
2784static VALUE threadptr_invoke_proc_location(rb_thread_t *th);
2785
2786static void
2787native_set_thread_name(rb_thread_t *th)
2788{
2789#ifdef SET_CURRENT_THREAD_NAME
2790 VALUE loc;
2791 if (!NIL_P(loc = th->name)) {
2792 SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc));
2793 }
2794 else if ((loc = threadptr_invoke_proc_location(th)) != Qnil) {
2795 char *name, *p;
2796 char buf[THREAD_NAME_MAX];
2797 size_t len;
2798 int n;
2799
2800 name = RSTRING_PTR(RARRAY_AREF(loc, 0));
2801 p = strrchr(name, '/'); /* show only the basename of the path. */
2802 if (p && p[1])
2803 name = p + 1;
2804
2805 n = snprintf(buf, sizeof(buf), "%s:%d", name, NUM2INT(RARRAY_AREF(loc, 1)));
2806 RB_GC_GUARD(loc);
2807
2808 len = (size_t)n;
2809 if (len >= sizeof(buf)) {
2810 buf[sizeof(buf)-2] = '*';
2811 buf[sizeof(buf)-1] = '\0';
2812 }
2813 SET_CURRENT_THREAD_NAME(buf);
2814 }
2815#endif
2816}
2817
2818static void
2819native_set_another_thread_name(rb_nativethread_id_t thread_id, VALUE name)
2820{
2821#if defined SET_ANOTHER_THREAD_NAME || defined SET_CURRENT_THREAD_NAME
2822 char buf[THREAD_NAME_MAX];
2823 const char *s = "";
2824# if !defined SET_ANOTHER_THREAD_NAME
2825 if (!pthread_equal(pthread_self(), thread_id)) return;
2826# endif
2827 if (!NIL_P(name)) {
2828 long n;
2829 RSTRING_GETMEM(name, s, n);
2830 if (n >= (int)sizeof(buf)) {
2831 memcpy(buf, s, sizeof(buf)-1);
2832 buf[sizeof(buf)-1] = '\0';
2833 s = buf;
2834 }
2835 }
2836# if defined SET_ANOTHER_THREAD_NAME
2837 SET_ANOTHER_THREAD_NAME(thread_id, s);
2838# elif defined SET_CURRENT_THREAD_NAME
2839 SET_CURRENT_THREAD_NAME(s);
2840# endif
2841#endif
2842}
2843
2844#if defined(RB_THREAD_T_HAS_NATIVE_ID) || defined(__APPLE__)
2845static VALUE
2846native_thread_native_thread_id(rb_thread_t *target_th)
2847{
2848 if (!target_th->nt) return Qnil;
2849
2850#ifdef RB_THREAD_T_HAS_NATIVE_ID
2851 int tid = target_th->nt->tid;
2852 if (tid == 0) return Qnil;
2853 return INT2FIX(tid);
2854#elif defined(__APPLE__)
2855 uint64_t tid;
2856/* The first condition is needed because MAC_OS_X_VERSION_10_6
2857 is not defined on 10.5, and while __POWERPC__ takes care of ppc/ppc64,
2858 i386 will be broken without this. Note, 10.5 is supported with GCC upstream,
2859 so it has C++17 and everything needed to build modern Ruby. */
2860# if (!defined(MAC_OS_X_VERSION_10_6) || \
2861 (MAC_OS_X_VERSION_MAX_ALLOWED < MAC_OS_X_VERSION_10_6) || \
2862 defined(__POWERPC__) /* never defined for PowerPC platforms */)
2863 const bool no_pthread_threadid_np = true;
2864# define NO_PTHREAD_MACH_THREAD_NP 1
2865# elif MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_6
2866 const bool no_pthread_threadid_np = false;
2867# else
2868# if !(defined(__has_attribute) && __has_attribute(availability))
2869 /* __API_AVAILABLE macro does nothing on gcc */
2870 __attribute__((weak)) int pthread_threadid_np(pthread_t, uint64_t*);
2871# endif
2872 /* Check weakly linked symbol */
2873 const bool no_pthread_threadid_np = !&pthread_threadid_np;
2874# endif
2875 if (no_pthread_threadid_np) {
2876 return ULL2NUM(pthread_mach_thread_np(pthread_self()));
2877 }
2878# ifndef NO_PTHREAD_MACH_THREAD_NP
2879 int e = pthread_threadid_np(target_th->nt->thread_id, &tid);
2880 if (e != 0) rb_syserr_fail(e, "pthread_threadid_np");
2881 return ULL2NUM((unsigned long long)tid);
2882# endif
2883#endif
2884}
2885# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 1
2886#else
2887# define USE_NATIVE_THREAD_NATIVE_THREAD_ID 0
2888#endif
2889
2890static struct {
2891 rb_serial_t created_fork_gen;
2892 pthread_t pthread_id;
2893
2894 int comm_fds[2]; // r, w
2895
2896#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
2897 int event_fd; // kernel event queue fd (epoll/kqueue)
2898#endif
2899#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
2900#define EPOLL_EVENTS_MAX 0x10
2901 struct epoll_event finished_events[EPOLL_EVENTS_MAX];
2902#elif HAVE_SYS_EVENT_H && USE_MN_THREADS
2903#define KQUEUE_EVENTS_MAX 0x10
2904 struct kevent finished_events[KQUEUE_EVENTS_MAX];
2905#endif
2906
2907 // waiting threads list
2908 struct ccan_list_head waiting; // waiting threads in ractors
2909 pthread_mutex_t waiting_lock;
2910} timer_th = {
2911 .created_fork_gen = 0,
2912};
2913
2914#define TIMER_THREAD_CREATED_P() (timer_th.created_fork_gen == current_fork_gen)
2915
2916static void timer_thread_check_timeslice(rb_vm_t *vm);
2917static int timer_thread_set_timeout(rb_vm_t *vm);
2918static void timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial);
2919
2920#include "thread_pthread_mn.c"
2921
2922static rb_thread_t *
2923thread_sched_waiting_thread(struct rb_thread_sched_waiting *w)
2924{
2925 if (w) {
2926 return (rb_thread_t *)((size_t)w - offsetof(rb_thread_t, sched.waiting_reason));
2927 }
2928 else {
2929 return NULL;
2930 }
2931}
2932
2933static int
2934timer_thread_set_timeout(rb_vm_t *vm)
2935{
2936#if 0
2937 return 10; // ms
2938#else
2939 int timeout = -1;
2940
2941 ractor_sched_lock(vm, NULL);
2942 {
2943 if ( !ccan_list_empty(&vm->ractor.sched.timeslice_threads) // (1-1) Provide time slice for active NTs
2944 || !ubf_threads_empty() // (1-3) Periodic UBF
2945 || vm->ractor.sched.grq_cnt > 0 // (1-4) Lazy GRQ deq start
2946 ) {
2947
2948 RUBY_DEBUG_LOG("timeslice:%d ubf:%d grq:%d",
2949 !ccan_list_empty(&vm->ractor.sched.timeslice_threads),
2950 !ubf_threads_empty(),
2951 (vm->ractor.sched.grq_cnt > 0));
2952
2953 timeout = 10; // ms
2954 vm->ractor.sched.timeslice_wait_inf = false;
2955 }
2956 else {
2957 vm->ractor.sched.timeslice_wait_inf = true;
2958 }
2959 }
2960 ractor_sched_unlock(vm, NULL);
2961
2962 // Always check waiting threads to find minimum timeout
2963 // even when scheduler has work (grq_cnt > 0)
2964 rb_native_mutex_lock(&timer_th.waiting_lock);
2965 {
2966 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
2967 rb_thread_t *th = thread_sched_waiting_thread(w);
2968
2969 if (th && (th->sched.waiting_reason.flags & thread_sched_waiting_timeout)) {
2970 rb_hrtime_t now = rb_hrtime_now();
2971 rb_hrtime_t hrrel = rb_hrtime_sub(th->sched.waiting_reason.data.timeout, now);
2972
2973 RUBY_DEBUG_LOG("th:%u now:%lu rel:%lu", rb_th_serial(th), (unsigned long)now, (unsigned long)hrrel);
2974
2975 // TODO: overflow?
2976 int thread_timeout = (int)((hrrel + RB_HRTIME_PER_MSEC - 1) / RB_HRTIME_PER_MSEC); // ms
2977
2978 // Use minimum of scheduler timeout and thread sleep timeout
2979 if (timeout < 0 || thread_timeout < timeout) {
2980 timeout = thread_timeout;
2981 }
2982 }
2983 }
2984 rb_native_mutex_unlock(&timer_th.waiting_lock);
2985
2986 RUBY_DEBUG_LOG("timeout:%d inf:%d", timeout, (int)vm->ractor.sched.timeslice_wait_inf);
2987
2988 // fprintf(stderr, "timeout:%d\n", timeout);
2989 return timeout;
2990#endif
2991}
2992
2993static void
2994timer_thread_check_signal(rb_vm_t *vm)
2995{
2996 // ruby_sigchld_handler(vm); TODO
2997
2998 int signum = rb_signal_buff_size();
2999 if (UNLIKELY(signum > 0) && vm->ractor.main_thread) {
3000 RUBY_DEBUG_LOG("signum:%d", signum);
3001 threadptr_trap_interrupt(vm->ractor.main_thread);
3002 }
3003}
3004
3005static bool
3006timer_thread_check_exceed(rb_hrtime_t abs, rb_hrtime_t now)
3007{
3008 if (abs < now) {
3009 return true;
3010 }
3011 else if (abs - now < RB_HRTIME_PER_MSEC) {
3012 return true; // too short time
3013 }
3014 else {
3015 return false;
3016 }
3017}
3018
3019static rb_thread_t *
3020timer_thread_deq_wakeup(rb_vm_t *vm, rb_hrtime_t now, uint32_t *event_serial)
3021{
3022 struct rb_thread_sched_waiting *w = ccan_list_top(&timer_th.waiting, struct rb_thread_sched_waiting, node);
3023
3024 if (w != NULL &&
3025 (w->flags & thread_sched_waiting_timeout) &&
3026 timer_thread_check_exceed(w->data.timeout, now)) {
3027
3028 RUBY_DEBUG_LOG("wakeup th:%u", rb_th_serial(thread_sched_waiting_thread(w)));
3029
3030 // delete from waiting list
3031 ccan_list_del_init(&w->node);
3032
3033 // setup result
3034 w->flags = thread_sched_waiting_none;
3035 w->data.result = 0;
3036
3037 rb_thread_t *th = thread_sched_waiting_thread(w);
3038 *event_serial = w->data.event_serial;
3039 return th;
3040 }
3041
3042 return NULL;
3043}
3044
3045static void
3046timer_thread_wakeup_thread_locked(struct rb_thread_sched *sched, rb_thread_t *th, uint32_t event_serial)
3047{
3048 if (sched->running != th && th->sched.event_serial == event_serial) {
3049 thread_sched_to_ready_common(sched, th, true, false);
3050 }
3051}
3052
3053static void
3054timer_thread_wakeup_thread(rb_thread_t *th, uint32_t event_serial)
3055{
3056 RUBY_DEBUG_LOG("th:%u", rb_th_serial(th));
3057 struct rb_thread_sched *sched = TH_SCHED(th);
3058
3059 thread_sched_lock(sched, th);
3060 {
3061 timer_thread_wakeup_thread_locked(sched, th, event_serial);
3062 }
3063 thread_sched_unlock(sched, th);
3064}
3065
3066static void
3067timer_thread_check_timeout(rb_vm_t *vm)
3068{
3069 rb_hrtime_t now = rb_hrtime_now();
3070 rb_thread_t *th;
3071 uint32_t event_serial;
3072
3073 rb_native_mutex_lock(&timer_th.waiting_lock);
3074 {
3075 while ((th = timer_thread_deq_wakeup(vm, now, &event_serial)) != NULL) {
3076 rb_native_mutex_unlock(&timer_th.waiting_lock);
3077 timer_thread_wakeup_thread(th, event_serial);
3078 rb_native_mutex_lock(&timer_th.waiting_lock);
3079 }
3080 }
3081 rb_native_mutex_unlock(&timer_th.waiting_lock);
3082}
3083
3084static void
3085timer_thread_check_timeslice(rb_vm_t *vm)
3086{
3087 // TODO: check time
3088 rb_thread_t *th;
3089 ccan_list_for_each(&vm->ractor.sched.timeslice_threads, th, sched.node.timeslice_threads) {
3090 RUBY_DEBUG_LOG("timeslice th:%u", rb_th_serial(th));
3091 RUBY_VM_SET_TIMER_INTERRUPT(th->ec);
3092 }
3093}
3094
3095void
3096rb_assert_sig(void)
3097{
3098 sigset_t oldmask;
3099 pthread_sigmask(0, NULL, &oldmask);
3100 if (sigismember(&oldmask, SIGVTALRM)) {
3101 rb_bug("!!!");
3102 }
3103 else {
3104 RUBY_DEBUG_LOG("ok");
3105 }
3106}
3107
3108static void *
3109timer_thread_func(void *ptr)
3110{
3111 rb_vm_t *vm = (rb_vm_t *)ptr;
3112#if defined(RUBY_NT_SERIAL)
3113 ruby_nt_serial = (rb_atomic_t)-1;
3114#endif
3115
3116 RUBY_DEBUG_LOG("started%s", "");
3117
3118 while (RUBY_ATOMIC_LOAD(system_working)) {
3119 timer_thread_check_signal(vm);
3120 timer_thread_check_timeout(vm);
3121 ubf_wakeup_all_threads();
3122
3123 RUBY_DEBUG_LOG("system_working:%d", RUBY_ATOMIC_LOAD(system_working));
3124 timer_thread_polling(vm);
3125 }
3126
3127 RUBY_DEBUG_LOG("terminated");
3128 return NULL;
3129}
3130
3131/* only use signal-safe system calls here */
3132static void
3133signal_communication_pipe(int fd)
3134{
3135#if USE_EVENTFD
3136 const uint64_t buff = 1;
3137#else
3138 const char buff = '!';
3139#endif
3140 ssize_t result;
3141
3142 /* already opened */
3143 if (fd >= 0) {
3144 retry:
3145 if ((result = write(fd, &buff, sizeof(buff))) <= 0) {
3146 int e = errno;
3147 switch (e) {
3148 case EINTR: goto retry;
3149 case EAGAIN:
3150#if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
3151 case EWOULDBLOCK:
3152#endif
3153 break;
3154 default:
3155 async_bug_fd("rb_thread_wakeup_timer_thread: write", e, fd);
3156 }
3157 }
3158 if (TT_DEBUG) WRITE_CONST(2, "rb_thread_wakeup_timer_thread: write\n");
3159 }
3160 else {
3161 // ignore wakeup
3162 }
3163}
3164
3165static void
3166timer_thread_wakeup_force(void)
3167{
3168 // should not use RUBY_DEBUG_LOG() because it can be called within signal handlers.
3169 signal_communication_pipe(timer_th.comm_fds[1]);
3170}
3171
3172static void
3173timer_thread_wakeup_locked(rb_vm_t *vm)
3174{
3175 // should be locked before.
3176 ASSERT_ractor_sched_locked(vm, NULL);
3177
3178 if (timer_th.created_fork_gen == current_fork_gen) {
3179 if (vm->ractor.sched.timeslice_wait_inf) {
3180 RUBY_DEBUG_LOG("wakeup with fd:%d", timer_th.comm_fds[1]);
3181 timer_thread_wakeup_force();
3182 }
3183 else {
3184 RUBY_DEBUG_LOG("will be wakeup...");
3185 }
3186 }
3187}
3188
3189static void
3190timer_thread_wakeup(void)
3191{
3192 rb_vm_t *vm = GET_VM();
3193
3194 ractor_sched_lock(vm, NULL);
3195 {
3196 timer_thread_wakeup_locked(vm);
3197 }
3198 ractor_sched_unlock(vm, NULL);
3199}
3200
3201static void
3202rb_thread_create_timer_thread(void)
3203{
3204 rb_serial_t created_fork_gen = timer_th.created_fork_gen;
3205
3206 RUBY_DEBUG_LOG("fork_gen create:%d current:%d", (int)created_fork_gen, (int)current_fork_gen);
3207
3208 timer_th.created_fork_gen = current_fork_gen;
3209
3210 if (created_fork_gen != current_fork_gen) {
3211 if (created_fork_gen != 0) {
3212 RUBY_DEBUG_LOG("forked child process");
3213
3214 CLOSE_INVALIDATE_PAIR(timer_th.comm_fds);
3215#if HAVE_SYS_EPOLL_H && USE_MN_THREADS
3216 close_invalidate(&timer_th.event_fd, "close event_fd");
3217#endif
3218 rb_native_mutex_destroy(&timer_th.waiting_lock);
3219 }
3220
3221 ccan_list_head_init(&timer_th.waiting);
3222 rb_native_mutex_initialize(&timer_th.waiting_lock);
3223
3224 // open communication channel
3225 setup_communication_pipe_internal(timer_th.comm_fds);
3226
3227 // open event fd
3228 timer_thread_setup_mn();
3229 }
3230
3231 pthread_create(&timer_th.pthread_id, NULL, timer_thread_func, GET_VM());
3232}
3233
3234static int
3235native_stop_timer_thread(void)
3236{
3237 RUBY_ATOMIC_SET(system_working, 0);
3238
3239 RUBY_DEBUG_LOG("wakeup send %d", timer_th.comm_fds[1]);
3240 timer_thread_wakeup_force();
3241 RUBY_DEBUG_LOG("wakeup sent");
3242 pthread_join(timer_th.pthread_id, NULL);
3243
3244 if (TT_DEBUG) fprintf(stderr, "stop timer thread\n");
3245
3246 return 1;
3247}
3248
3249static void
3250native_reset_timer_thread(void)
3251{
3252 //
3253}
3254
3255#ifdef HAVE_SIGALTSTACK
3256int
3257ruby_stack_overflowed_p(const rb_thread_t *th, const void *addr)
3258{
3259 void *base;
3260 size_t size;
3261 const size_t water_mark = 1024 * 1024;
3262 STACK_GROW_DIR_DETECTION;
3263
3264 if (th) {
3265 size = th->ec->machine.stack_maxsize;
3266 base = (char *)th->ec->machine.stack_start - STACK_DIR_UPPER(0, size);
3267 }
3268#ifdef STACKADDR_AVAILABLE
3269 else if (get_stack(&base, &size) == 0) {
3270# ifdef __APPLE__
3271 if (pthread_equal(th->nt->thread_id, native_main_thread.id)) {
3272 struct rlimit rlim;
3273 if (getrlimit(RLIMIT_STACK, &rlim) == 0 && rlim.rlim_cur > size) {
3274 size = (size_t)rlim.rlim_cur;
3275 }
3276 }
3277# endif
3278 base = (char *)base + STACK_DIR_UPPER(+size, -size);
3279 }
3280#endif
3281 else {
3282 return 0;
3283 }
3284
3285 size /= RUBY_STACK_SPACE_RATIO;
3286 if (size > water_mark) size = water_mark;
3287 if (IS_STACK_DIR_UPPER()) {
3288 if (size > ~(size_t)base+1) size = ~(size_t)base+1;
3289 if (addr > base && addr <= (void *)((char *)base + size)) return 1;
3290 }
3291 else {
3292 if (size > (size_t)base) size = (size_t)base;
3293 if (addr > (void *)((char *)base - size) && addr <= base) return 1;
3294 }
3295 return 0;
3296}
3297#endif
3298
3299int
3300 rb_reserved_fd_p(int fd)
3301{
3302 /* no false-positive if out-of-FD at startup */
3303 if (fd < 0) return 0;
3304
3305 if (fd == timer_th.comm_fds[0] ||
3306 fd == timer_th.comm_fds[1]
3307#if (HAVE_SYS_EPOLL_H || HAVE_SYS_EVENT_H) && USE_MN_THREADS
3308 || fd == timer_th.event_fd
3309#endif
3310 ) {
3311 goto check_fork_gen;
3312 }
3313 return 0;
3314
3315 check_fork_gen:
3316 if (timer_th.created_fork_gen == current_fork_gen) {
3317 /* async-signal-safe */
3318 return 1;
3319 }
3320 else {
3321 return 0;
3322 }
3323}
3324
3325rb_nativethread_id_t
3326 rb_nativethread_self(void)
3327{
3328 return pthread_self();
3329}
3330
3331#if defined(USE_POLL) && !defined(HAVE_PPOLL)
3332/* TODO: don't ignore sigmask */
3333static int
3334ruby_ppoll(struct pollfd *fds, nfds_t nfds,
3335 const struct timespec *ts, const sigset_t *sigmask)
3336{
3337 int timeout_ms;
3338
3339 if (ts) {
3340 int tmp, tmp2;
3341
3342 if (ts->tv_sec > INT_MAX/1000)
3343 timeout_ms = INT_MAX;
3344 else {
3345 tmp = (int)(ts->tv_sec * 1000);
3346 /* round up 1ns to 1ms to avoid excessive wakeups for <1ms sleep */
3347 tmp2 = (int)((ts->tv_nsec + 999999L) / (1000L * 1000L));
3348 if (INT_MAX - tmp < tmp2)
3349 timeout_ms = INT_MAX;
3350 else
3351 timeout_ms = (int)(tmp + tmp2);
3352 }
3353 }
3354 else
3355 timeout_ms = -1;
3356
3357 return poll(fds, nfds, timeout_ms);
3358}
3359# define ppoll(fds,nfds,ts,sigmask) ruby_ppoll((fds),(nfds),(ts),(sigmask))
3360#endif
3361
3362/*
3363 * Single CPU setups benefit from explicit sched_yield() before ppoll(),
3364 * since threads may be too starved to enter the GVL waitqueue for
3365 * us to detect contention. Instead, we want to kick other threads
3366 * so they can run and possibly prevent us from entering slow paths
3367 * in ppoll() or similar syscalls.
3368 *
3369 * Confirmed on FreeBSD 11.2 and Linux 4.19.
3370 * [ruby-core:90417] [Bug #15398]
3371 */
3372#define THREAD_BLOCKING_YIELD(th) do { \
3373 const rb_thread_t *next_th; \
3374 struct rb_thread_sched *sched = TH_SCHED(th); \
3375 RB_VM_SAVE_MACHINE_CONTEXT(th); \
3376 thread_sched_to_waiting(sched, (th)); \
3377 next_th = sched->running; \
3378 rb_native_mutex_unlock(&sched->lock_); \
3379 native_thread_yield(); /* TODO: needed? */ \
3380 if (!next_th && rb_ractor_living_thread_num(th->ractor) > 1) { \
3381 native_thread_yield(); \
3382 }
3383
3384static void
3385native_sleep(rb_thread_t *th, rb_hrtime_t *rel)
3386{
3387 struct rb_thread_sched *sched = TH_SCHED(th);
3388
3389 RUBY_DEBUG_LOG("rel:%d", rel ? (int)*rel : 0);
3390 if (rel) {
3391 if (th_has_dedicated_nt(th)) {
3392 native_cond_sleep(th, rel);
3393 }
3394 else {
3395 thread_sched_wait_events(sched, th, -1, thread_sched_waiting_timeout, rel);
3396 }
3397 }
3398 else {
3399 thread_sched_to_waiting_until_wakeup(sched, th);
3400 }
3401
3402 RUBY_DEBUG_LOG("wakeup");
3403}
3404
3405// fork read-write lock (only for pthread)
3406static pthread_rwlock_t rb_thread_fork_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3407
3408void
3409rb_thread_release_fork_lock(void)
3410{
3411 int r;
3412 if ((r = pthread_rwlock_unlock(&rb_thread_fork_rw_lock))) {
3413 rb_bug_errno("pthread_rwlock_unlock", r);
3414 }
3415}
3416
3417void
3418rb_thread_reset_fork_lock(void)
3419{
3420 int r;
3421 if ((r = pthread_rwlock_destroy(&rb_thread_fork_rw_lock))) {
3422 rb_bug_errno("pthread_rwlock_destroy", r);
3423 }
3424
3425 if ((r = pthread_rwlock_init(&rb_thread_fork_rw_lock, NULL))) {
3426 rb_bug_errno("pthread_rwlock_init", r);
3427 }
3428}
3429
3430void *
3431rb_thread_prevent_fork(void *(*func)(void *), void *data)
3432{
3433 int r;
3434 if ((r = pthread_rwlock_rdlock(&rb_thread_fork_rw_lock))) {
3435 rb_bug_errno("pthread_rwlock_rdlock", r);
3436 }
3437 void *result = func(data);
3438 rb_thread_release_fork_lock();
3439 return result;
3440}
3441
3442void
3443rb_thread_acquire_fork_lock(void)
3444{
3445 int r;
3446 if ((r = pthread_rwlock_wrlock(&rb_thread_fork_rw_lock))) {
3447 rb_bug_errno("pthread_rwlock_wrlock", r);
3448 }
3449}
3450
3451// thread internal event hooks (only for pthread)
3452
3453struct rb_internal_thread_event_hook {
3454 rb_internal_thread_event_callback callback;
3455 rb_event_flag_t event;
3456 void *user_data;
3457
3458 struct rb_internal_thread_event_hook *next;
3459};
3460
3461static pthread_rwlock_t rb_internal_thread_event_hooks_rw_lock = PTHREAD_RWLOCK_INITIALIZER;
3462
3463#if defined(HAVE_WORKING_FORK)
3464void
3465rb_internal_thread_event_hooks_rw_lock_atfork(void)
3466{
3467 // After fork(), this rwlock may have been held by a now-dead thread.
3468 //
3469 // pthread_rwlock_destroy() on a held lock is undefined behavior, and
3470 // pthread_rwlock_init() on an already-initialized lock is also undefined
3471 // behavior
3472 //
3473 // Direct assignment of PTHREAD_RWLOCK_INITIALIZER is safe and portable.
3474 rb_internal_thread_event_hooks_rw_lock =
3475 (pthread_rwlock_t)PTHREAD_RWLOCK_INITIALIZER;
3476}
3477#endif
3478
3479rb_internal_thread_event_hook_t *
3480 rb_internal_thread_add_event_hook(rb_internal_thread_event_callback callback, rb_event_flag_t internal_event, void *user_data)
3481{
3482 rb_internal_thread_event_hook_t *hook = ALLOC_N(rb_internal_thread_event_hook_t, 1);
3483 hook->callback = callback;
3484 hook->user_data = user_data;
3485 hook->event = internal_event;
3486
3487 int r;
3488 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3489 rb_bug_errno("pthread_rwlock_wrlock", r);
3490 }
3491
3492 hook->next = rb_internal_thread_event_hooks;
3493 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook);
3494
3495 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3496 rb_bug_errno("pthread_rwlock_unlock", r);
3497 }
3498 return hook;
3499}
3500
3501bool
3502 rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t * hook)
3503{
3504 int r;
3505 if ((r = pthread_rwlock_wrlock(&rb_internal_thread_event_hooks_rw_lock))) {
3506 rb_bug_errno("pthread_rwlock_wrlock", r);
3507 }
3508
3509 bool success = FALSE;
3510
3511 if (rb_internal_thread_event_hooks == hook) {
3512 ATOMIC_PTR_EXCHANGE(rb_internal_thread_event_hooks, hook->next);
3513 success = TRUE;
3514 }
3515 else {
3516 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3517
3518 do {
3519 if (h->next == hook) {
3520 h->next = hook->next;
3521 success = TRUE;
3522 break;
3523 }
3524 } while ((h = h->next));
3525 }
3526
3527 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3528 rb_bug_errno("pthread_rwlock_unlock", r);
3529 }
3530
3531 if (success) {
3532 ruby_xfree(hook);
3533 }
3534 return success;
3535}
3536
3537static void
3538rb_thread_execute_hooks(rb_event_flag_t event, rb_thread_t *th)
3539{
3540 int r;
3541 if ((r = pthread_rwlock_rdlock(&rb_internal_thread_event_hooks_rw_lock))) {
3542 rb_bug_errno("pthread_rwlock_rdlock", r);
3543 }
3544
3545 if (rb_internal_thread_event_hooks) {
3546 rb_internal_thread_event_hook_t *h = rb_internal_thread_event_hooks;
3547 do {
3548 if (h->event & event) {
3549 rb_internal_thread_event_data_t event_data = {
3550 .thread = th->self,
3551 };
3552 (*h->callback)(event, &event_data, h->user_data);
3553 }
3554 } while((h = h->next));
3555 }
3556 if ((r = pthread_rwlock_unlock(&rb_internal_thread_event_hooks_rw_lock))) {
3557 rb_bug_errno("pthread_rwlock_unlock", r);
3558 }
3559}
3560
3561// return true if the current thread acquires DNT.
3562// return false if the current thread already acquires DNT.
3563bool
3564 rb_thread_lock_native_thread(void)
3565{
3566 rb_thread_t *th = GET_THREAD();
3567 bool is_snt = th->nt->dedicated == 0;
3568 native_thread_dedicated_inc(th->vm, th->ractor, th->nt);
3569
3570 return is_snt;
3571}
3572
3573void
3574rb_thread_malloc_stack_set(rb_thread_t *th, void *stack)
3575{
3576 th->sched.malloc_stack = true;
3577 th->sched.context_stack = stack;
3578}
3579
3580#endif /* THREAD_SYSTEM_DEPENDENT_IMPLEMENTATION */
rb_atomic_t
std::atomic< unsigned > rb_atomic_t
Type that is eligible for atomic operations.
Definition atomic.h:69
RUBY_ATOMIC_FETCH_ADD
#define RUBY_ATOMIC_FETCH_ADD(var, val)
Atomically replaces the value pointed by var with the result of addition of val to the old value of v...
Definition atomic.h:118
RUBY_ATOMIC_LOAD
#define RUBY_ATOMIC_LOAD(var)
Atomic load.
Definition atomic.h:175
RUBY_ATOMIC_SET
#define RUBY_ATOMIC_SET(var, val)
Identical to RUBY_ATOMIC_EXCHANGE, except for the return type.
Definition atomic.h:185
rb_event_flag_t
uint32_t rb_event_flag_t
Represents event(s).
Definition event.h:108
INT2FIX
#define INT2FIX
Old name of RB_INT2FIX.
Definition long.h:48
ZALLOC
#define ZALLOC
Old name of RB_ZALLOC.
Definition memory.h:402
ALLOC_N
#define ALLOC_N
Old name of RB_ALLOC_N.
Definition memory.h:399
ULL2NUM
#define ULL2NUM
Old name of RB_ULL2NUM.
Definition long_long.h:31
NUM2INT
#define NUM2INT
Old name of RB_NUM2INT.
Definition int.h:44
Qnil
#define Qnil
Old name of RUBY_Qnil.
Definition special_consts.h:60
NIL_P
#define NIL_P
Old name of RB_NIL_P.
Definition special_consts.h:55
rb_eNotImpError
VALUE rb_eNotImpError
NotImplementedError exception.
Definition error.c:1441
rb_syserr_fail
void rb_syserr_fail(int e, const char *mesg)
Raises appropriate exception that represents a C errno.
Definition error.c:3909
rb_bug_errno
void rb_bug_errno(const char *mesg, int errno_arg)
This is a wrapper of rb_bug() which automatically constructs appropriate message from the passed errn...
Definition error.c:1141
rb_cloexec_pipe
int rb_cloexec_pipe(int fildes[2])
Opens a pipe with closing on exec.
Definition io.c:427
rb_update_max_fd
void rb_update_max_fd(int fd)
Informs the interpreter that the passed fd can be the max.
Definition io.c:248
rb_reserved_fd_p
int rb_reserved_fd_p(int fd)
Queries if the given FD is reserved or not.
rb_unblock_function_t
void rb_unblock_function_t(void *)
This is the type of UBFs.
Definition thread.h:336
rb_timespec_now
void rb_timespec_now(struct timespec *ts)
Fills the current time into the given struct.
Definition time.c:2003
len
int len
Length of the buffer.
Definition io.h:8
RUBY_INTERNAL_THREAD_EVENT_RESUMED
#define RUBY_INTERNAL_THREAD_EVENT_RESUMED
Triggered when a thread successfully acquired the GVL.
Definition thread.h:238
rb_internal_thread_add_event_hook
rb_internal_thread_event_hook_t * rb_internal_thread_add_event_hook(rb_internal_thread_event_callback func, rb_event_flag_t events, void *data)
Registers a thread event hook function.
RUBY_INTERNAL_THREAD_EVENT_EXITED
#define RUBY_INTERNAL_THREAD_EVENT_EXITED
Triggered when a thread exits.
Definition thread.h:252
RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
#define RUBY_INTERNAL_THREAD_EVENT_SUSPENDED
Triggered when a thread released the GVL.
Definition thread.h:245
rb_thread_lock_native_thread
bool rb_thread_lock_native_thread(void)
Declare the current Ruby thread should acquire a dedicated native thread on M:N thread scheduler.
RUBY_INTERNAL_THREAD_EVENT_STARTED
#define RUBY_INTERNAL_THREAD_EVENT_STARTED
Triggered when a new thread is started.
Definition thread.h:224
rb_internal_thread_remove_event_hook
bool rb_internal_thread_remove_event_hook(rb_internal_thread_event_hook_t *hook)
Unregister the passed hook.
RUBY_INTERNAL_THREAD_EVENT_READY
#define RUBY_INTERNAL_THREAD_EVENT_READY
Triggered when a thread attempt to acquire the GVL.
Definition thread.h:231
RBIMPL_ATTR_MAYBE_UNUSED
#define RBIMPL_ATTR_MAYBE_UNUSED()
Wraps (or simulates) [[maybe_unused]].
Definition maybe_unused.h:35
RB_GC_GUARD
#define RB_GC_GUARD(v)
Prevents premature destruction of local objects.
Definition memory.h:167
rb_fd_select
#define rb_fd_select
Waits for multiple file descriptors at once.
Definition posix.h:66
RARRAY_AREF
#define RARRAY_AREF(a, i)
Definition rarray.h:403
RSTRING_GETMEM
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
Definition rstring.h:450
errno
#define errno
Ractor-aware version of errno.
Definition ruby.h:388
coroutine_context
Definition Context.h:36
rb_fdset_t
The data structure which wraps the fd_set bitmap used by select(2).
Definition largesize.h:71
rb_native_thread
Definition thread_none.h:11
rb_thread_sched_waiting
Definition thread_pthread.h:26
rb_thread_sched
Definition thread_none.h:16
timespec
Definition missing.h:74
timeval
Definition missing.h:63
tr
Definition string.c:8221
rb_nativethread_self
rb_nativethread_id_t rb_nativethread_self(void)
Queries the ID of the native thread that is calling this function.
rb_native_mutex_lock
void rb_native_mutex_lock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_lock.
rb_native_cond_initialize
void rb_native_cond_initialize(rb_nativethread_cond_t *cond)
Fills the passed condition variable with an initial value.
rb_native_mutex_trylock
int rb_native_mutex_trylock(rb_nativethread_lock_t *lock)
Identical to rb_native_mutex_lock(), except it doesn't block in case rb_native_mutex_lock() would.
rb_native_cond_broadcast
void rb_native_cond_broadcast(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_mutex_initialize
void rb_native_mutex_initialize(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_initialize.
rb_native_mutex_unlock
void rb_native_mutex_unlock(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_unlock.
rb_native_mutex_destroy
void rb_native_mutex_destroy(rb_nativethread_lock_t *lock)
Just another name of rb_nativethread_lock_destroy.
rb_native_cond_destroy
void rb_native_cond_destroy(rb_nativethread_cond_t *cond)
Destroys the passed condition variable.
rb_native_cond_signal
void rb_native_cond_signal(rb_nativethread_cond_t *cond)
Signals a condition variable.
rb_native_cond_wait
void rb_native_cond_wait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex)
Waits for the passed condition variable to be signalled.
rb_native_cond_timedwait
void rb_native_cond_timedwait(rb_nativethread_cond_t *cond, rb_nativethread_lock_t *mutex, unsigned long msec)
Identical to rb_native_cond_wait(), except it additionally takes timeout in msec resolution.
__attribute__
Definition vm_insnhelper.c:2410
VALUE
uintptr_t VALUE
Type that represents a Ruby object.
Definition value.h:40
ruby_xfree
void ruby_xfree(void *ptr)
Deallocates a storage instance.
Definition gc.c:5270
ruby_xmalloc
void * ruby_xmalloc(size_t size)
Allocates a storage instance.
Definition gc.c:5132