libpthread/src/pthread.c
1/*
2 * Copyright (c) 2000-2013 Apple Inc. All rights reserved.
3 *
4 * @APPLE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. Please obtain a copy of the License at
10 * http://www.opensource.apple.com/apsl/ and read it before using this
11 * file.
12 *
13 * The Original Code and all software distributed under the License are
14 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
15 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
16 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
18 * Please see the License for the specific language governing rights and
19 * limitations under the License.
20 *
21 * @APPLE_LICENSE_HEADER_END@
22 */
23/*
24 * Copyright 1996 1995 by Open Software Foundation, Inc. 1997 1996 1995 1994 1993 1992 1991
25 * All Rights Reserved
26 *
27 * Permission to use, copy, modify, and distribute this software and
28 * its documentation for any purpose and without fee is hereby granted,
29 * provided that the above copyright notice appears in all copies and
30 * that both the copyright notice and this permission notice appear in
31 * supporting documentation.
32 *
33 * OSF DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE
34 * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
35 * FOR A PARTICULAR PURPOSE.
36 *
37 * IN NO EVENT SHALL OSF BE LIABLE FOR ANY SPECIAL, INDIRECT, OR
38 * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
39 * LOSS OF USE, DATA OR PROFITS, WHETHER IN ACTION OF CONTRACT,
40 * NEGLIGENCE, OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
41 * WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
42 *
43 */
44/*
45 * MkLinux
46 */
47
48/*
49 * POSIX Pthread Library
50 */
51
52#include <mach/port.h>
53#include "internal.h"
54
55#include <stdlib.h>
56#include <errno.h>
57#include <signal.h>
58#include <unistd.h>
59#include <mach/mach_init.h>
60#include <mach/mach_vm.h>
61#include <mach/mach_sync_ipc.h>
62#include <mach/thread_switch.h>
63#include <sys/time.h>
64#include <sys/resource.h>
65#include <sys/sysctl.h>
66#include <sys/queue.h>
67#include <sys/ulock.h>
68#include <sys/mman.h>
69#include <machine/vmparam.h>
70#define __APPLE_API_PRIVATE
71#include <machine/cpu_capabilities.h>
72#if __has_include(<ptrauth.h>)
73#include <ptrauth.h>
74#endif // __has_include(<ptrauth.h>)
75#include <os/thread_self_restrict.h>
76#include <os/tsd.h>
77
78#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
79#include <mach-o/dyld_priv.h>
80#include <mach-o/loader.h>
81#include <mach-o/getsect.h>
82#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
83
84// Default stack size is 512KB; independent of the main thread's stack size.
85#define DEFAULT_STACK_SIZE (size_t)(512 * 1024)
86
87//
88// Global constants
89//
90
91/*
92 * The pthread may be offset into a page. In that event, by contract
93 * with the kernel, the allocation will extend PTHREAD_SIZE from the
94 * start of the next page. There's also one page worth of allocation
95 * below stacksize for the guard page. <rdar://problem/19941744>
96 */
97#define PTHREAD_SIZE ((size_t)mach_vm_round_page(sizeof(struct pthread_s)))
98#define PTHREAD_ALLOCADDR(stackaddr, stacksize) ((stackaddr - stacksize) - vm_page_size)
99#define PTHREAD_ALLOCSIZE(stackaddr, stacksize) ((round_page((uintptr_t)stackaddr) + PTHREAD_SIZE) - (uintptr_t)PTHREAD_ALLOCADDR(stackaddr, stacksize))
100
101static const pthread_attr_t _pthread_attr_default = {
102 .sig = _PTHREAD_ATTR_SIG,
103 .stacksize = 0,
104 .detached = PTHREAD_CREATE_JOINABLE,
105 .inherit = _PTHREAD_DEFAULT_INHERITSCHED,
106 .policy = _PTHREAD_DEFAULT_POLICY,
107 .defaultguardpage = true,
108 // compile time constant for _pthread_default_priority(0)
109 .qosclass = (1U << (THREAD_QOS_LEGACY - 1 + _PTHREAD_PRIORITY_QOS_CLASS_SHIFT)) |
110 ((uint8_t)-1 & _PTHREAD_PRIORITY_PRIORITY_MASK),
111 .work_interval_port = MACH_PORT_NULL,
112};
113
114#if PTHREAD_LAYOUT_SPI
115
116const struct pthread_layout_offsets_s pthread_layout_offsets = {
117 .plo_version = 1,
118 .plo_pthread_tsd_base_offset = offsetof(struct pthread_s, tsd),
119 .plo_pthread_tsd_base_address_offset = 0,
120 .plo_pthread_tsd_entry_size = sizeof(((struct pthread_s *)NULL)->tsd[0]),
121};
122
123#endif // PTHREAD_LAYOUT_SPI
124
125//
126// Global exported variables
127//
128
129// This global should be used (carefully) by anyone needing to know if a
130// pthread (other than the main thread) has been created.
131int __is_threaded = 0;
132const int __unix_conforming = 1; // we're always conformant, but it's exported
133
134//
135// Global internal variables
136//
137
138// _pthread_list_lock protects _pthread_count, access to the __pthread_head
139// list. Externally imported by pthread_cancelable.c.
140struct __pthread_list __pthread_head = TAILQ_HEAD_INITIALIZER(__pthread_head);
141_pthread_lock _pthread_list_lock = _PTHREAD_LOCK_INITIALIZER;
142
143uint32_t _main_qos;
144
145#if VARIANT_DYLD
146// The main thread's pthread_t
147struct pthread_s _main_thread OS_ALIGNED(64);
148#else // VARIANT_DYLD
149pthread_t _main_thread_ptr;
150void *(*_pthread_malloc)(size_t);
151void (*_pthread_free)(void *);
152#endif // VARIANT_DYLD
153
154#if PTHREAD_DEBUG_LOG
155#include <fcntl.h>
156int _pthread_debuglog;
157uint64_t _pthread_debugstart;
158#endif
159
160//
161// Global static variables
162//
163static bool __workq_newapi;
164static uint8_t default_priority;
165#if !VARIANT_DYLD
166static uint8_t max_priority;
167static uint8_t min_priority;
168#endif // !VARIANT_DYLD
169static int _pthread_count = 1;
170static int pthread_concurrency;
171uintptr_t _pthread_ptr_munge_token;
172
173static bool pthread_yield_to_zero = true;
174
175
176static void (*exitf)(int) = __exit;
177
178// workgroup support
179static struct pthread_workgroup_functions_s *_pthread_workgroup_functions;
180
181// work queue support data
182OS_NORETURN OS_COLD
183static void
184__pthread_invalid_keventfunction(void **events, int *nevents)
185{
186 PTHREAD_CLIENT_CRASH(0, "Invalid kqworkq setup");
187}
188
189OS_NORETURN OS_COLD
190static void
191__pthread_invalid_workloopfunction(uint64_t *workloop_id, void **events, int *nevents)
192{
193 PTHREAD_CLIENT_CRASH(0, "Invalid kqwl setup");
194}
195static pthread_workqueue_function2_t __libdispatch_workerfunction;
196static pthread_workqueue_function_kevent_t __libdispatch_keventfunction = &__pthread_invalid_keventfunction;
197static pthread_workqueue_function_workloop_t __libdispatch_workloopfunction = &__pthread_invalid_workloopfunction;
198static int __pthread_supported_features; // supported feature set
199
200#if defined(__i386__) || defined(__x86_64__)
201static mach_vm_address_t __pthread_stack_hint = 0xB0000000;
202#elif defined(__arm__) || defined(__arm64__)
203static mach_vm_address_t __pthread_stack_hint = 0x30000000;
204#else
205#error no __pthread_stack_hint for this architecture
206#endif
207
208void
209_pthread_globals_init(void *ctx)
210{
211 pthread_globals_t globals = ctx;
212 for (thread_qos_t qos = 0; qos < THREAD_QOS_LAST; qos++) {
213 globals->qmp_logical[qos] = QOS_PARALLELISM_NOT_QUERIED;
214 globals->qmp_physical[qos] = QOS_PARALLELISM_NOT_QUERIED;
215 globals->cluster_physical[qos] = QOS_PARALLELISM_NOT_QUERIED;
216 }
217}
218
219//
220// Function prototypes
221//
222
223// pthread primitives
224static inline void _pthread_struct_init(pthread_t t, const pthread_attr_t *attrs,
225 void *stack, size_t stacksize, void *freeaddr, size_t freesize);
226
227#if VARIANT_DYLD
228static void _pthread_set_self_dyld(void);
229#endif // VARIANT_DYLD
230static inline void _pthread_set_self_internal(pthread_t);
231
232static inline void __pthread_started_thread(pthread_t t);
233
234static void _pthread_exit(pthread_t self, void *value_ptr) __dead2;
235
236static inline void _pthread_introspection_thread_create(pthread_t t);
237static inline void _pthread_introspection_thread_start(pthread_t t);
238static inline void _pthread_introspection_thread_terminate(pthread_t t);
239static inline void _pthread_introspection_thread_destroy(pthread_t t);
240
241/*
242 * Flags filed passed to bsdthread_create and back in pthread_start
243 * 31 <---------------------------------> 0
244 * _________________________________________
245 * | flags(8) | policy(8) | importance(16) |
246 * -----------------------------------------
247 */
248#define PTHREAD_START_CUSTOM 0x01000000 // <rdar://problem/34501401>
249#define PTHREAD_START_SETSCHED 0x02000000
250// was PTHREAD_START_DETACHED 0x04000000
251#define PTHREAD_START_QOSCLASS 0x08000000
252#define PTHREAD_START_TSD_BASE_SET 0x10000000
253#define PTHREAD_START_SUSPENDED 0x20000000
254#define PTHREAD_START_QOSCLASS_MASK 0x00ffffff
255#define PTHREAD_START_POLICY_BITSHIFT 16
256#define PTHREAD_START_POLICY_MASK 0xff
257#define PTHREAD_START_IMPORTANCE_MASK 0xffff
258
259#pragma mark pthread attrs
260
261int
262pthread_attr_destroy(pthread_attr_t *attr)
263{
264 int ret = EINVAL;
265 if (attr->sig == _PTHREAD_ATTR_SIG) {
266 if (MACH_PORT_VALID(attr->work_interval_port)) {
267 ret = mach_port_mod_refs(mach_task_self(), attr->work_interval_port,
268 MACH_PORT_RIGHT_SEND, -1);
269 if (ret == KERN_SUCCESS) {
270 attr->sig = 0;
271 }
272 } else {
273 attr->sig = 0;
274 ret = 0;
275 }
276 }
277 return ret;
278}
279
280int
281pthread_attr_getdetachstate(const pthread_attr_t *attr, int *detachstate)
282{
283 int ret = EINVAL;
284 if (attr->sig == _PTHREAD_ATTR_SIG) {
285 *detachstate = attr->detached;
286 ret = 0;
287 }
288 return ret;
289}
290
291int
292pthread_attr_getinheritsched(const pthread_attr_t *attr, int *inheritsched)
293{
294 int ret = EINVAL;
295 if (attr->sig == _PTHREAD_ATTR_SIG) {
296 *inheritsched = attr->inherit;
297 ret = 0;
298 }
299 return ret;
300}
301
302static OS_ALWAYS_INLINE void
303_pthread_attr_get_schedparam(const pthread_attr_t *attr,
304 struct sched_param *param)
305{
306 if (attr->schedset) {
307 *param = attr->param;
308 } else {
309 param->sched_priority = default_priority;
310 param->quantum = 10; /* quantum isn't public yet */
311 }
312}
313
314int
315pthread_attr_getschedparam(const pthread_attr_t *attr, struct sched_param *param)
316{
317 int ret = EINVAL;
318 if (attr->sig == _PTHREAD_ATTR_SIG) {
319 _pthread_attr_get_schedparam(attr, param);
320 ret = 0;
321 }
322 return ret;
323}
324
325int
326pthread_attr_getschedpolicy(const pthread_attr_t *attr, int *policy)
327{
328 int ret = EINVAL;
329 if (attr->sig == _PTHREAD_ATTR_SIG) {
330 *policy = attr->policy;
331 ret = 0;
332 }
333 return ret;
334}
335
336int
337pthread_attr_init(pthread_attr_t *attr)
338{
339 *attr = _pthread_attr_default;
340 return 0;
341}
342
343int
344pthread_attr_setdetachstate(pthread_attr_t *attr, int detachstate)
345{
346 int ret = EINVAL;
347 if (attr->sig == _PTHREAD_ATTR_SIG &&
348 (detachstate == PTHREAD_CREATE_JOINABLE ||
349 detachstate == PTHREAD_CREATE_DETACHED)) {
350 attr->detached = detachstate;
351 ret = 0;
352 }
353 return ret;
354}
355
356int
357pthread_attr_setinheritsched(pthread_attr_t *attr, int inheritsched)
358{
359 int ret = EINVAL;
360 if (attr->sig == _PTHREAD_ATTR_SIG &&
361 (inheritsched == PTHREAD_INHERIT_SCHED ||
362 inheritsched == PTHREAD_EXPLICIT_SCHED)) {
363 attr->inherit = inheritsched;
364 ret = 0;
365 }
366 return ret;
367}
368
369int
370pthread_attr_setschedparam(pthread_attr_t *attr, const struct sched_param *param)
371{
372 int ret = EINVAL;
373 if (attr->sig == _PTHREAD_ATTR_SIG) {
374 /* TODO: Validate sched_param fields */
375 attr->param = *param;
376 attr->schedset = 1;
377 ret = 0;
378 }
379 return ret;
380}
381
382#define _PTHREAD_POLICY_IS_FIXEDPRI(x) ((x) == SCHED_RR || (x) == SCHED_FIFO)
383
384int
385pthread_attr_setschedpolicy(pthread_attr_t *attr, int policy)
386{
387 int ret = EINVAL;
388 if (attr->sig == _PTHREAD_ATTR_SIG && (policy == SCHED_OTHER ||
389 policy == SCHED_RR || policy == SCHED_FIFO)) {
390 if (!_PTHREAD_POLICY_IS_FIXEDPRI(policy)) {
391 /* non-fixedpri policy should remove cpupercent */
392 attr->cpupercentset = 0;
393 }
394 attr->policy = policy;
395 attr->policyset = 1;
396 ret = 0;
397 }
398 return ret;
399}
400
401int
402pthread_attr_setscope(pthread_attr_t *attr, int scope)
403{
404 int ret = EINVAL;
405 if (attr->sig == _PTHREAD_ATTR_SIG) {
406 if (scope == PTHREAD_SCOPE_SYSTEM) {
407 // No attribute yet for the scope.
408 ret = 0;
409 } else if (scope == PTHREAD_SCOPE_PROCESS) {
410 ret = ENOTSUP;
411 }
412 }
413 return ret;
414}
415
416int
417pthread_attr_getscope(const pthread_attr_t *attr, int *scope)
418{
419 int ret = EINVAL;
420 if (attr->sig == _PTHREAD_ATTR_SIG) {
421 *scope = PTHREAD_SCOPE_SYSTEM;
422 ret = 0;
423 }
424 return ret;
425}
426
427int
428pthread_attr_getstackaddr(const pthread_attr_t *attr, void **stackaddr)
429{
430 int ret = EINVAL;
431 if (attr->sig == _PTHREAD_ATTR_SIG) {
432 *stackaddr = attr->stackaddr;
433 ret = 0;
434 }
435 return ret;
436}
437
438int
439pthread_attr_setstackaddr(pthread_attr_t *attr, void *stackaddr)
440{
441 int ret = EINVAL;
442 if (attr->sig == _PTHREAD_ATTR_SIG &&
443 ((mach_vm_address_t)stackaddr & vm_page_mask) == 0) {
444 attr->stackaddr = stackaddr;
445 attr->defaultguardpage = false;
446 attr->guardsize = 0;
447 ret = 0;
448 }
449 return ret;
450}
451
452static inline size_t
453_pthread_attr_stacksize(const pthread_attr_t *attr)
454{
455 return attr->stacksize ? attr->stacksize : DEFAULT_STACK_SIZE;
456}
457
458int
459pthread_attr_getstacksize(const pthread_attr_t *attr, size_t *stacksize)
460{
461 int ret = EINVAL;
462 if (attr->sig == _PTHREAD_ATTR_SIG) {
463 *stacksize = _pthread_attr_stacksize(attr);
464 ret = 0;
465 }
466 return ret;
467}
468
469int
470pthread_attr_setstacksize(pthread_attr_t *attr, size_t stacksize)
471{
472#if TARGET_OS_OSX
473 // If the caller is doing something reasonable, help them out.
474 if (stacksize % 0x1000 == 0) {
475 stacksize = round_page(stacksize);
476 }
477#endif // TARGET_OS_OSX
478
479 int ret = EINVAL;
480 if (attr->sig == _PTHREAD_ATTR_SIG &&
481 ((stacksize & vm_page_mask) == 0) &&
482 stacksize >= PTHREAD_STACK_MIN) {
483 attr->stacksize = stacksize;
484 ret = 0;
485 }
486 return ret;
487}
488
489int
490pthread_attr_getstack(const pthread_attr_t *attr, void **stackaddr, size_t * stacksize)
491{
492 int ret = EINVAL;
493 if (attr->sig == _PTHREAD_ATTR_SIG) {
494 *stackaddr = (void *)((uintptr_t)attr->stackaddr - attr->stacksize);
495 *stacksize = _pthread_attr_stacksize(attr);
496 ret = 0;
497 }
498 return ret;
499}
500
501// Per SUSv3, the stackaddr is the base address, the lowest addressable byte
502// address. This is not the same as in pthread_attr_setstackaddr.
503int
504pthread_attr_setstack(pthread_attr_t *attr, void *stackaddr, size_t stacksize)
505{
506 int ret = EINVAL;
507 if (attr->sig == _PTHREAD_ATTR_SIG &&
508 (((mach_vm_address_t)stackaddr & vm_page_mask) == 0) &&
509 ((stacksize & vm_page_mask) == 0) &&
510 stacksize >= PTHREAD_STACK_MIN) {
511 attr->stackaddr = (void *)((uintptr_t)stackaddr + stacksize);
512 attr->stacksize = stacksize;
513 ret = 0;
514 }
515 return ret;
516}
517
518int
519pthread_attr_setguardsize(pthread_attr_t *attr, size_t guardsize)
520{
521#if TARGET_OS_OSX
522 // If the caller is doing something reasonable, help them out.
523 if (guardsize % 0x1000 == 0) {
524 guardsize = round_page(guardsize);
525 }
526#endif // TARGET_OS_OSX
527
528 int ret = EINVAL;
529 if (attr->sig == _PTHREAD_ATTR_SIG &&
530 (guardsize & vm_page_mask) == 0) {
531 /* Guardsize of 0 is valid, means no guard */
532 attr->defaultguardpage = false;
533 attr->guardsize = guardsize;
534 ret = 0;
535 }
536 return ret;
537}
538
539static inline size_t
540_pthread_attr_guardsize(const pthread_attr_t *attr)
541{
542 return attr->defaultguardpage ? vm_page_size : attr->guardsize;
543}
544
545int
546pthread_attr_getguardsize(const pthread_attr_t *attr, size_t *guardsize)
547{
548 int ret = EINVAL;
549 if (attr->sig == _PTHREAD_ATTR_SIG) {
550 *guardsize = _pthread_attr_guardsize(attr);
551 ret = 0;
552 }
553 return ret;
554}
555
556int
557pthread_attr_setcpupercent_np(pthread_attr_t *attr, int percent,
558 unsigned long refillms)
559{
560 int ret = EINVAL;
561 if (attr->sig == _PTHREAD_ATTR_SIG && percent < UINT8_MAX &&
562 refillms < _PTHREAD_ATTR_REFILLMS_MAX && attr->policyset &&
563 _PTHREAD_POLICY_IS_FIXEDPRI(attr->policy)) {
564 attr->cpupercent = percent;
565 attr->refillms = (uint32_t)(refillms & 0x00ffffff);
566 attr->cpupercentset = 1;
567 ret = 0;
568 }
569 return ret;
570}
571
572int
573pthread_attr_setworkinterval_np(pthread_attr_t *attr, mach_port_t work_interval_port)
574{
575 int ret = EINVAL;
576 if (attr->sig == _PTHREAD_ATTR_SIG && MACH_PORT_VALID(work_interval_port)) {
577 ret = mach_port_mod_refs(mach_task_self(), work_interval_port, MACH_PORT_RIGHT_SEND, 1);
578 if (ret == KERN_SUCCESS) {
579 attr->work_interval_port = work_interval_port;
580 }
581 }
582 return ret;
583}
584
585#pragma mark pthread lifetime
586
587// Allocate a thread structure, stack and guard page.
588//
589// The thread structure may optionally be placed in the same allocation as the
590// stack, residing above the top of the stack. This cannot be done if a
591// custom stack address is provided.
592//
593// Similarly the guard page cannot be allocated if a custom stack address is
594// provided.
595//
596// The allocated thread structure is initialized with values that indicate how
597// it should be freed.
598
599static pthread_t
600_pthread_allocate(const pthread_attr_t *attrs, void **stack,
601 bool from_mach_thread)
602{
603 mach_vm_address_t allocaddr = __pthread_stack_hint;
604 size_t allocsize, guardsize, stacksize, pthreadoff;
605 kern_return_t kr;
606 pthread_t t;
607
608 if (os_unlikely(attrs->stacksize != 0 &&
609 attrs->stacksize < PTHREAD_STACK_MIN)) {
610 PTHREAD_CLIENT_CRASH(attrs->stacksize, "Stack size in attrs is too small");
611 }
612
613 if (os_unlikely((mach_vm_address_t)attrs->stackaddr & vm_page_mask)) {
614 PTHREAD_CLIENT_CRASH(attrs->stackaddr, "Unaligned stack addr in attrs");
615 }
616
617 // Allocate a pthread structure if necessary
618
619 if (attrs->stackaddr != NULL) {
620 allocsize = PTHREAD_SIZE;
621 guardsize = 0;
622 pthreadoff = 0;
623 // <rdar://problem/42588315> if the attrs struct specifies a custom
624 // stack address but not a custom size, using ->stacksize here instead
625 // of _pthread_attr_stacksize stores stacksize as zero, indicating
626 // that the stack size is unknown.
627 stacksize = attrs->stacksize;
628 } else {
629 guardsize = _pthread_attr_guardsize(attrs);
630 stacksize = _pthread_attr_stacksize(attrs) + PTHREAD_T_OFFSET;
631 pthreadoff = stacksize + guardsize;
632 allocsize = pthreadoff + PTHREAD_SIZE;
633 allocsize = mach_vm_round_page(allocsize);
634 }
635
636 kr = mach_vm_map(mach_task_self(), &allocaddr, allocsize, vm_page_size - 1,
637 VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE, MEMORY_OBJECT_NULL,
638 0, FALSE, VM_PROT_DEFAULT, VM_PROT_ALL, VM_INHERIT_DEFAULT);
639
640 if (kr != KERN_SUCCESS) {
641 kr = mach_vm_allocate(mach_task_self(), &allocaddr, allocsize,
642 VM_MAKE_TAG(VM_MEMORY_STACK)| VM_FLAGS_ANYWHERE);
643 } else if (__syscall_logger && !from_mach_thread) {
644 // libsyscall will not output malloc stack logging events when
645 // VM_MEMORY_STACK is passed in to facilitate mach thread promotion.
646 // To avoid losing the stack traces for normal p-thread create
647 // operations, libpthread must pretend to be the vm syscall and log
648 // the allocations. <rdar://36418708>
649 int eventTypeFlags = stack_logging_type_vm_allocate |
650 stack_logging_type_mapped_file_or_shared_mem;
651 __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK),
652 (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0,
653 (uintptr_t)allocaddr, 0);
654 }
655
656 if (kr != KERN_SUCCESS) {
657 *stack = NULL;
658 return NULL;
659 } else if (__syscall_logger && !from_mach_thread) {
660 // libsyscall will not output malloc stack logging events when
661 // VM_MEMORY_STACK is passed in to facilitate mach thread promotion.
662 // To avoid losing the stack traces for normal p-thread create
663 // operations, libpthread must pretend to be the vm syscall and log
664 // the allocations. <rdar://36418708>
665 int eventTypeFlags = stack_logging_type_vm_allocate;
666 __syscall_logger(eventTypeFlags | VM_MAKE_TAG(VM_MEMORY_STACK),
667 (uintptr_t)mach_task_self(), (uintptr_t)allocsize, 0,
668 (uintptr_t)allocaddr, 0);
669 }
670
671 // The stack grows down.
672 // Set the guard page at the lowest address of the
673 // newly allocated stack. Return the highest address
674 // of the stack.
675 if (guardsize) {
676 (void)mach_vm_protect(mach_task_self(), allocaddr, guardsize,
677 FALSE, VM_PROT_NONE);
678 }
679
680 // Thread structure resides at the top of the stack (when using a
681 // custom stack, allocsize == PTHREAD_SIZE, so places the pthread_t
682 // at allocaddr).
683 t = (pthread_t)(allocaddr + pthreadoff);
684 if (attrs->stackaddr) {
685 *stack = attrs->stackaddr;
686 } else {
687 *stack = t;
688 }
689
690 _pthread_struct_init(t, attrs, *stack, stacksize, allocaddr, allocsize);
691 return t;
692}
693
694OS_NOINLINE
695void
696_pthread_deallocate(pthread_t t, bool from_mach_thread)
697{
698 kern_return_t ret;
699
700 // Don't free the main thread.
701 if (t != main_thread()) {
702 if (!from_mach_thread) { // see __pthread_add_thread
703 _pthread_introspection_thread_destroy(t);
704 }
705 ret = mach_vm_deallocate(mach_task_self(), t->freeaddr, t->freesize);
706 if (ret != KERN_SUCCESS) {
707 PTHREAD_INTERNAL_CRASH(ret, "Unable to deallocate stack");
708 }
709 }
710}
711
712#pragma clang diagnostic push
713#pragma clang diagnostic ignored "-Wreturn-stack-address"
714
715OS_NOINLINE
716static void*
717_pthread_current_stack_address(void)
718{
719 int a;
720 return &a;
721}
722
723#pragma clang diagnostic pop
724
725static void
726_pthread_joiner_wake(pthread_t thread)
727{
728 uint32_t *exit_gate = &thread->tl_exit_gate;
729
730 for (;;) {
731 int ret = __ulock_wake(UL_UNFAIR_LOCK | ULF_NO_ERRNO, exit_gate, 0);
732 if (ret == 0 || ret == -ENOENT) {
733 return;
734 }
735 if (ret != -EINTR) {
736 PTHREAD_INTERNAL_CRASH(-ret, "pthread_join() wake failure");
737 }
738 }
739}
740
741static void
742_pthread_dealloc_reply_port(pthread_t self)
743{
744 mach_port_t port = _pthread_tsd_slot(self, MIG_REPLY);
745 if (port != MACH_PORT_NULL) {
746 // this will also set the TSD to MACH_PORT_NULL
747 mig_dealloc_reply_port(port);
748 }
749}
750
751static void
752_pthread_dealloc_special_reply_port(pthread_t self)
753{
754 mach_port_t port = _pthread_tsd_slot(self, MACH_SPECIAL_REPLY);
755 if (port != MACH_PORT_NULL) {
756 _pthread_tsd_slot(self, MACH_SPECIAL_REPLY) = MACH_PORT_NULL;
757 thread_destruct_special_reply_port(port, THREAD_SPECIAL_REPLY_PORT_ALL);
758 }
759}
760
761// Terminates the thread if called from the currently running thread.
762OS_NORETURN OS_NOINLINE OS_NOT_TAIL_CALLED
763static void
764_pthread_terminate(pthread_t t, void *exit_value)
765{
766 _pthread_introspection_thread_terminate(t);
767
768 uintptr_t freeaddr = (uintptr_t)t->freeaddr;
769 size_t freesize = t->freesize;
770 bool should_exit;
771
772 // the size of just the stack
773 size_t freesize_stack = t->freesize;
774
775 // We usually pass our structure+stack to bsdthread_terminate to free, but
776 // if we get told to keep the pthread_t structure around then we need to
777 // adjust the free size and addr in the pthread_t to just refer to the
778 // structure and not the stack. If we do end up deallocating the
779 // structure, this is useless work since no one can read the result, but we
780 // can't do it after the call to pthread_remove_thread because it isn't
781 // safe to dereference t after that.
782 if ((void*)t > t->freeaddr && (void*)t < t->freeaddr + t->freesize){
783 // Check to ensure the pthread structure itself is part of the
784 // allocation described by freeaddr/freesize, in which case we split and
785 // only deallocate the area below the pthread structure. In the event of a
786 // custom stack, the freeaddr/size will be the pthread structure itself, in
787 // which case we shouldn't free anything (the final else case).
788 freesize_stack = trunc_page((uintptr_t)t - (uintptr_t)freeaddr);
789
790 // describe just the remainder for deallocation when the pthread_t goes away
791 t->freeaddr += freesize_stack;
792 t->freesize -= freesize_stack;
793 } else if (t == main_thread()) {
794 freeaddr = t->stackaddr - pthread_get_stacksize_np(t);
795 uintptr_t stackborder = trunc_page((uintptr_t)_pthread_current_stack_address());
796 freesize_stack = stackborder - freeaddr;
797 } else {
798 freesize_stack = 0;
799 }
800
801 mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF);
802 bool keep_thread_struct = false, needs_wake = false;
803 semaphore_t custom_stack_sema = MACH_PORT_NULL;
804
805 _pthread_dealloc_special_reply_port(t);
806 _pthread_dealloc_reply_port(t);
807
808 _pthread_lock_lock(&_pthread_list_lock);
809
810 // This piece of code interacts with pthread_join. It will always:
811 // - set tl_exit_gate to MACH_PORT_DEAD (thread exited)
812 // - set tl_exit_value to the value passed to pthread_exit()
813 // - decrement _pthread_count, so that we can exit the process when all
814 // threads exited even if not all of them were joined.
815 t->tl_exit_gate = MACH_PORT_DEAD;
816 t->tl_exit_value = exit_value;
817 should_exit = (--_pthread_count <= 0);
818
819 // If we see a joiner, we prepost that the join has to succeed,
820 // and the joiner is committed to finish (even if it was canceled)
821 if (t->tl_join_ctx) {
822 custom_stack_sema = _pthread_joiner_prepost_wake(t); // unsets tl_joinable
823 needs_wake = true;
824 }
825
826 // Joinable threads that have no joiner yet are kept on the thread list
827 // so that pthread_join() can later discover the thread when it is joined,
828 // and will have to do the pthread_t cleanup.
829 if (t->tl_joinable) {
830 t->tl_joiner_cleans_up = keep_thread_struct = true;
831 } else {
832 TAILQ_REMOVE(&__pthread_head, t, tl_plist);
833 }
834
835 _pthread_lock_unlock(&_pthread_list_lock);
836
837 if (needs_wake) {
838 // When we found a waiter, we want to drop the very contended list lock
839 // before we do the syscall in _pthread_joiner_wake(). Then, we decide
840 // who gets to cleanup the pthread_t between the joiner and the exiting
841 // thread:
842 // - the joiner tries to set tl_join_ctx to NULL
843 // - the exiting thread tries to set tl_joiner_cleans_up to true
844 // Whoever does it first commits the other guy to cleanup the pthread_t
845 _pthread_joiner_wake(t);
846 _pthread_lock_lock(&_pthread_list_lock);
847 if (t->tl_join_ctx) {
848 t->tl_joiner_cleans_up = true;
849 keep_thread_struct = true;
850 }
851 _pthread_lock_unlock(&_pthread_list_lock);
852 }
853
854 //
855 // /!\ dereferencing `t` past this point is not safe /!\
856 //
857
858 if (keep_thread_struct || t == main_thread()) {
859 // Use the adjusted freesize of just the stack that we computed above.
860 freesize = freesize_stack;
861 } else {
862 _pthread_introspection_thread_destroy(t);
863 }
864
865 // Check if there is nothing to free because the thread has a custom
866 // stack allocation and is joinable.
867 if (freesize == 0) {
868 freeaddr = 0;
869 }
870 if (should_exit) {
871 exitf(0);
872 }
873 __bsdthread_terminate((void *)freeaddr, freesize, kport, custom_stack_sema);
874 PTHREAD_INTERNAL_CRASH(t, "thread didn't terminate");
875}
876
877OS_NORETURN
878static void
879_pthread_terminate_invoke(pthread_t t, void *exit_value)
880{
881#if PTHREAD_T_OFFSET
882 void *p = NULL;
883 // <rdar://problem/25688492> During pthread termination there is a race
884 // between pthread_join and pthread_terminate; if the joiner is responsible
885 // for cleaning up the pthread_t struct, then it may destroy some part of the
886 // stack with it on 16k OSes. So that this doesn't cause _pthread_terminate()
887 // to crash because its stack has been removed from under its feet, just make
888 // sure termination happens in a part of the stack that is not on the same
889 // page as the pthread_t.
890 if (trunc_page((uintptr_t)__builtin_frame_address(0)) ==
891 trunc_page((uintptr_t)t)) {
892 p = alloca(PTHREAD_T_OFFSET);
893 }
894 // And this __asm__ volatile is needed to stop the compiler from optimising
895 // away the alloca() completely.
896 __asm__ volatile ("" : : "r"(p) );
897#endif
898 _pthread_terminate(t, exit_value);
899}
900
901#pragma mark pthread start / body
902
903void
904_pthread_start(pthread_t self, mach_port_t kport,
905 __unused void *(*fun)(void *), __unused void *arg,
906 __unused size_t stacksize, unsigned int pflags)
907{
908 if (os_unlikely(pflags & PTHREAD_START_SUSPENDED)) {
909 PTHREAD_INTERNAL_CRASH(pflags,
910 "kernel without PTHREAD_START_SUSPENDED support");
911 }
912 if (os_unlikely((pflags & PTHREAD_START_TSD_BASE_SET) == 0)) {
913 PTHREAD_INTERNAL_CRASH(pflags,
914 "thread_set_tsd_base() wasn't called by the kernel");
915 }
916 PTHREAD_DEBUG_ASSERT(MACH_PORT_VALID(kport));
917 PTHREAD_DEBUG_ASSERT(_pthread_tsd_slot(self, MACH_THREAD_SELF) == kport);
918 _pthread_validate_signature(self);
919 _pthread_markcancel_if_canceled(self, kport);
920
921 _pthread_set_self_internal(self);
922 __pthread_started_thread(self);
923 _pthread_exit(self, (self->fun)(self->arg));
924}
925
926OS_ALWAYS_INLINE
927static inline void
928_pthread_struct_init(pthread_t t, const pthread_attr_t *attrs,
929 void *stackaddr, size_t stacksize, void *freeaddr, size_t freesize)
930{
931 _pthread_init_signature(t);
932 _pthread_tsd_slot(t, PTHREAD_SELF) = t;
933 _pthread_tsd_slot(t, ERRNO) = &t->err_no;
934 if (attrs->schedset == 0) {
935 _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) = attrs->qosclass;
936 } else {
937 _pthread_tsd_slot(t, PTHREAD_QOS_CLASS) =
938 _pthread_unspecified_priority();
939 }
940 _pthread_tsd_slot(t, PTR_MUNGE) = _pthread_ptr_munge_token;
941 t->tl_has_custom_stack = (attrs->stackaddr != NULL);
942
943 _pthread_lock_init(&t->lock);
944
945 t->stackaddr = stackaddr;
946 t->stackbottom = stackaddr - stacksize;
947 t->freeaddr = freeaddr;
948 t->freesize = freesize;
949
950 t->guardsize = _pthread_attr_guardsize(attrs);
951 t->tl_joinable = (attrs->detached == PTHREAD_CREATE_JOINABLE);
952 t->inherit = attrs->inherit;
953 t->tl_policy = attrs->policy;
954 t->schedset = attrs->schedset;
955 _pthread_attr_get_schedparam(attrs, &t->tl_param);
956 t->cancel_state = PTHREAD_CANCEL_ENABLE | PTHREAD_CANCEL_DEFERRED;
957}
958
959#pragma mark pthread public interface
960
961/* Non portable public api to know whether this process has(had) atleast one thread
962 * apart from main thread. There could be race if there is a thread in the process of
963 * creation at the time of call . It does not tell whether there are more than one thread
964 * at this point of time.
965 */
966int
967pthread_is_threaded_np(void)
968{
969 return __is_threaded;
970}
971
972mach_port_t
973pthread_mach_thread_np(pthread_t t)
974{
975 mach_port_t kport = MACH_PORT_NULL;
976 (void)_pthread_is_valid(t, &kport);
977 return kport;
978}
979
980pthread_t
981pthread_from_mach_thread_np(mach_port_t kernel_thread)
982{
983 pthread_t p = NULL;
984
985 /* No need to wait as mach port is already known */
986 _pthread_lock_lock(&_pthread_list_lock);
987
988 TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
989 if (_pthread_tsd_slot(p, MACH_THREAD_SELF) == kernel_thread) {
990 break;
991 }
992 }
993
994 _pthread_lock_unlock(&_pthread_list_lock);
995
996 return p;
997}
998
999size_t
1000pthread_get_stacksize_np(pthread_t t)
1001{
1002 size_t size = 0;
1003
1004 if (t == NULL) {
1005 return ESRCH; // XXX bug?
1006 }
1007
1008#if TARGET_OS_OSX
1009 // The default rlimit based allocations will be provided with a stacksize
1010 // of the current limit and a freesize of the max. However, custom
1011 // allocations will just have the guard page to free. If we aren't in the
1012 // latter case, call into rlimit to determine the current stack size. In
1013 // the event that the current limit == max limit then we'll fall down the
1014 // fast path, but since it's unlikely that the limit is going to be lowered
1015 // after it's been change to the max, we should be fine.
1016 //
1017 // Of course, on arm rlim_cur == rlim_max and there's only the one guard
1018 // page. So, we can skip all this there.
1019 if (t == main_thread()) {
1020 size_t stacksize = t->stackaddr - t->stackbottom;
1021
1022 if (stacksize + vm_page_size != t->freesize) {
1023 // We want to call getrlimit() just once, as it's relatively
1024 // expensive
1025 static size_t rlimit_stack;
1026
1027 if (rlimit_stack == 0) {
1028 struct rlimit limit;
1029 int ret = getrlimit(RLIMIT_STACK, &limit);
1030
1031 if (ret == 0) {
1032 rlimit_stack = (size_t) limit.rlim_cur;
1033 }
1034 }
1035
1036 if (rlimit_stack == 0 || rlimit_stack > t->freesize) {
1037 return stacksize;
1038 } else {
1039 return round_page(rlimit_stack);
1040 }
1041 }
1042 }
1043#endif /* TARGET_OS_OSX */
1044
1045 if (t == pthread_self() || t == main_thread()) {
1046 size = t->stackaddr - t->stackbottom;;
1047 goto out;
1048 }
1049
1050 if (_pthread_validate_thread_and_list_lock(t)) {
1051 size = t->stackaddr - t->stackbottom;;
1052 _pthread_lock_unlock(&_pthread_list_lock);
1053 }
1054
1055out:
1056 // <rdar://problem/42588315> binary compatibility issues force us to return
1057 // DEFAULT_STACK_SIZE here when we do not know the size of the stack
1058 return size ? size : DEFAULT_STACK_SIZE;
1059}
1060
1061void *
1062pthread_get_stackaddr_np(pthread_t t)
1063{
1064 // since the main thread will not get de-allocated from underneath us
1065 if (t == pthread_self() || t == main_thread()) {
1066 return t->stackaddr;
1067 }
1068
1069 if (!_pthread_validate_thread_and_list_lock(t)) {
1070 return (void *)(uintptr_t)ESRCH; // XXX bug?
1071 }
1072
1073 void *addr = t->stackaddr;
1074 _pthread_lock_unlock(&_pthread_list_lock);
1075 return addr;
1076}
1077
1078pthread_t
1079pthread_main_thread_np(void)
1080{
1081 return main_thread();
1082}
1083
1084/* returns non-zero if the current thread is the main thread */
1085int
1086pthread_main_np(void)
1087{
1088 return pthread_self() == main_thread();
1089}
1090
1091static int
1092_pthread_threadid_slow(pthread_t thread, uint64_t *thread_id)
1093{
1094 unsigned int info_count = THREAD_IDENTIFIER_INFO_COUNT;
1095 mach_port_t thport = _pthread_tsd_slot(thread, MACH_THREAD_SELF);
1096 struct thread_identifier_info info;
1097 kern_return_t kr;
1098
1099 kr = thread_info(thport, THREAD_IDENTIFIER_INFO,
1100 (thread_info_t)&info, &info_count);
1101 if (kr == KERN_SUCCESS && info.thread_id) {
1102 *thread_id = info.thread_id;
1103#if __LP64__
1104 os_atomic_store(&thread->thread_id, info.thread_id, relaxed);
1105#else
1106 os_atomic_store_wide(&thread->thread_id, info.thread_id, relaxed);
1107#endif
1108 return 0;
1109 }
1110 return EINVAL;
1111}
1112
1113/*
1114 * if we are passed in a pthread_t that is NULL, then we return the current
1115 * thread's thread_id. So folks don't have to call pthread_self, in addition to
1116 * us doing it, if they just want their thread_id.
1117 */
1118int
1119pthread_threadid_np(pthread_t thread, uint64_t *thread_id)
1120{
1121 int res = 0;
1122 pthread_t self = pthread_self();
1123
1124 if (thread_id == NULL) {
1125 return EINVAL;
1126 }
1127
1128 if (thread == NULL || thread == self) {
1129 *thread_id = self->thread_id;
1130 } else if (!_pthread_validate_thread_and_list_lock(thread)) {
1131 res = ESRCH;
1132 } else {
1133#if __LP64__
1134 *thread_id = os_atomic_load(&thread->thread_id, relaxed);
1135#else
1136 *thread_id = os_atomic_load_wide(&thread->thread_id, relaxed);
1137#endif
1138 if (os_unlikely(*thread_id == 0)) {
1139 // there is a race at init because the thread sets its own TID.
1140 // correct this by asking mach
1141 res = _pthread_threadid_slow(thread, thread_id);
1142 }
1143 _pthread_lock_unlock(&_pthread_list_lock);
1144 }
1145 return res;
1146}
1147
1148int
1149pthread_cpu_number_np(size_t *cpu_id)
1150{
1151 if (cpu_id == NULL) {
1152 errno = EINVAL;
1153 return errno;
1154 }
1155
1156 *cpu_id = _os_cpu_number();
1157 return 0;
1158}
1159
1160int
1161pthread_getname_np(pthread_t thread, char *threadname, size_t len)
1162{
1163 if (thread == pthread_self()) {
1164 strlcpy(threadname, thread->pthread_name, len);
1165 return 0;
1166 }
1167
1168 if (!_pthread_validate_thread_and_list_lock(thread)) {
1169 return ESRCH;
1170 }
1171
1172 strlcpy(threadname, thread->pthread_name, len);
1173 _pthread_lock_unlock(&_pthread_list_lock);
1174 return 0;
1175}
1176
1177int
1178pthread_setname_np(const char *name)
1179{
1180 int res;
1181 pthread_t self = pthread_self();
1182
1183 size_t len = 0;
1184 if (name != NULL) {
1185 len = strlen(name);
1186 }
1187
1188 _pthread_validate_signature(self);
1189
1190 res = __proc_info(5, getpid(), 2, (uint64_t)0, (void*)name, (int)len);
1191 if (res == 0) {
1192 if (len > 0) {
1193 strlcpy(self->pthread_name, name, MAXTHREADNAMESIZE);
1194 } else {
1195 bzero(self->pthread_name, MAXTHREADNAMESIZE);
1196 }
1197 }
1198 return res;
1199
1200}
1201
1202#if TARGET_OS_OSX || PTHREAD_TARGET_OS_IOS_ONLY
1203
1204#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1205
1206#if !__LP64__
1207#error _PTHREAD_CONFIG_JIT_WRITE_PROTECT should imply __LP64__
1208#endif // !__LP64__
1209
1210static union _pthread_jit_config_u {
1211 struct {
1212 bool allowlist_enabled;
1213 bool allowlist_freeze_late;
1214 bool allowlist_frozen;
1215 uint32_t allowlist_count;
1216 pthread_jit_write_callback_t allowlist[];
1217 };
1218 uint8_t page[PAGE_MAX_SIZE];
1219} _pthread_jit_config OS_ALIGNED(PAGE_MAX_SIZE);
1220
1221_Static_assert(sizeof(_pthread_jit_config) == PAGE_MAX_SIZE,
1222 "_pthread_jit_config size must be PAGE_MAX_SIZE");
1223
1224#define _PTHREAD_JIT_ALLOWLIST_CAPACITY \
1225 ((PAGE_MAX_SIZE - offsetof(union _pthread_jit_config_u, allowlist)) / \
1226 sizeof(pthread_jit_write_callback_t))
1227
1228static _pthread_lock _pthread_jit_config_lock = _PTHREAD_LOCK_INITIALIZER;
1229
1230static void
1231_pthread_jit_write_protect_global_init(void)
1232{
1233 // On lockdown-default platforms, we lock down the allowlist if the process
1234 // is JIT-entitled at all. Otherwise, we lock down only if the process
1235 // specifically has the JIT-allowlist entitlement.
1236#if _PTHREAD_CONFIG_JIT_LOCKDOWN_DEFAULT
1237 bool allowlist_feature_enabled =
1238 (__pthread_supported_features & PTHREAD_FEATURE_JIT_ENTITLED);
1239#else
1240 bool allowlist_feature_enabled =
1241 (__pthread_supported_features & PTHREAD_FEATURE_JIT_ALLOWLIST);
1242#endif
1243 if (os_thread_self_restrict_rwx_is_supported() &&
1244 allowlist_feature_enabled) {
1245 union _pthread_jit_config_u *config = &_pthread_jit_config;
1246
1247 uintptr_t alignment = ((uintptr_t)config) % PAGE_MAX_SIZE;
1248 if (alignment != 0) {
1249 PTHREAD_INTERNAL_CRASH(alignment, "jit config invalid alignment");
1250 }
1251
1252 // Remap the config page as permanent. Note that this discards any
1253 // previous content, so all changes to the config must come _after_ this
1254 // point.
1255 //
1256 // TODO: find a suitable tag for this memory (rdar://75202626)
1257 mach_vm_address_t allocaddr = (mach_vm_address_t)config;
1258 kern_return_t kr = mach_vm_map(mach_task_self(), &allocaddr,
1259 sizeof(*config), PAGE_MAX_SIZE - 1,
1260 VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE | VM_FLAGS_PERMANENT,
1261 MEMORY_OBJECT_NULL, 0, FALSE, VM_PROT_DEFAULT, VM_PROT_DEFAULT,
1262 VM_INHERIT_DEFAULT);
1263 if (kr != KERN_SUCCESS || allocaddr != (mach_vm_address_t)config) {
1264 PTHREAD_INTERNAL_CRASH(kr, "jit config vm_map PERMANENT failed");
1265 }
1266
1267 config->allowlist_enabled = true;
1268 if (__pthread_supported_features & PTHREAD_FEATURE_JIT_FREEZE_LATE) {
1269 config->allowlist_freeze_late = true;
1270 }
1271 }
1272}
1273
1274static void
1275_pthread_jit_write_protect_bulk_image_load_callback(unsigned int image_count,
1276 const struct mach_header *mhs[], __unused const char *paths[])
1277{
1278 union _pthread_jit_config_u *config = &_pthread_jit_config;
1279
1280 if (config->allowlist_freeze_late) {
1281 _pthread_lock_lock(&_pthread_jit_config_lock);
1282 }
1283
1284 // Ignore dlopen()s after the allowlist is frozen
1285 if (config->allowlist_frozen) {
1286 goto done;
1287 }
1288
1289 for (unsigned int i = 0; i < image_count; i++) {
1290 const struct mach_header_64 *mh = (const struct mach_header_64 *)mhs[i];
1291
1292 if (mh->flags & MH_DYLIB_IN_CACHE) {
1293 // Internal code should use os_thread_self_restrict_rwx*()
1294 continue;
1295 }
1296
1297 unsigned long size = 0;
1298 uint8_t *data = getsectiondata(mh, "__DATA_CONST", "__pth_jit_func",
1299 &size);
1300 if (!data || !size ||
1301 (size % sizeof(pthread_jit_write_callback_t) != 0)) {
1302 continue;
1303 }
1304
1305 const pthread_jit_write_callback_t *image_allowlist =
1306 (const pthread_jit_write_callback_t *)data;
1307 size_t image_allowlist_count = size / sizeof(image_allowlist[0]);
1308
1309 // Copy allowed function pointers from the section up to the
1310 // NULL-terminator
1311 for (size_t j = 0; j < image_allowlist_count && image_allowlist[j]; j++) {
1312 if (config->allowlist_count >= _PTHREAD_JIT_ALLOWLIST_CAPACITY) {
1313 PTHREAD_CLIENT_CRASH(config->allowlist_count,
1314 "Too many pthread jit write callbacks");
1315 }
1316
1317 config->allowlist[config->allowlist_count] = image_allowlist[j];
1318 config->allowlist_count++;
1319 }
1320 }
1321
1322done:
1323 if (config->allowlist_freeze_late) {
1324 _pthread_lock_unlock(&_pthread_jit_config_lock);
1325 }
1326}
1327
1328static void
1329_pthread_jit_write_protect_freeze_config(void)
1330{
1331 union _pthread_jit_config_u *config = &_pthread_jit_config;
1332
1333 // By protecting config read-only with set_maximum, prevent an attack
1334 // from later altering its protection back to writeable.
1335 kern_return_t kr = mach_vm_protect(mach_task_self(),
1336 (mach_vm_address_t)config, sizeof(*config),
1337 /* set_maximum */ true, VM_PROT_READ);
1338 if (kr != KERN_SUCCESS) {
1339 PTHREAD_INTERNAL_CRASH(kr, "jit config vm_protect() failed");
1340 }
1341
1342}
1343
1344static void
1345_pthread_jit_write_protect_late_init(void)
1346{
1347 union _pthread_jit_config_u *config = &_pthread_jit_config;
1348 if (config->allowlist_enabled) {
1349 if (__pthread_supported_features & PTHREAD_FEATURE_JIT_ALLOWLIST) {
1350 // Iterate all initial images for allowed functions
1351 _dyld_register_for_bulk_image_loads(
1352 _pthread_jit_write_protect_bulk_image_load_callback);
1353 }
1354
1355 if (!config->allowlist_freeze_late) {
1356 // Since there's no way to un-register the image load callback, just
1357 // remember that we're no longer interested in new images
1358 config->allowlist_frozen = true;
1359 _pthread_jit_write_protect_freeze_config();
1360 }
1361 }
1362}
1363
1364#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1365
1366#if TARGET_OS_OSX
1367
1368void
1369pthread_jit_write_protect_np(int enable)
1370{
1371 if (!os_thread_self_restrict_rwx_is_supported()) {
1372 return;
1373 }
1374
1375 if (enable) {
1376 os_thread_self_restrict_rwx_to_rx();
1377 } else {
1378 os_thread_self_restrict_rwx_to_rw();
1379 }
1380
1381#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1382 // Validate _after_ toggling.
1383 union _pthread_jit_config_u *config = &_pthread_jit_config;
1384 if (config->allowlist_enabled) {
1385 if (!enable) {
1386 os_thread_self_restrict_rwx_to_rx();
1387 }
1388
1389 PTHREAD_CLIENT_CRASH(0,
1390 "pthread_jit_write_protect_np() disallowed by allowlist");
1391 }
1392#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1393}
1394
1395#endif // TARGET_OS_OSX
1396
1397int
1398pthread_jit_write_protect_supported_np()
1399{
1400 return os_thread_self_restrict_rwx_is_supported();
1401}
1402
1403int
1404pthread_jit_write_with_callback_np(pthread_jit_write_callback_t callback,
1405 void *ctx)
1406{
1407#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1408 if (!os_thread_self_restrict_rwx_is_supported()) {
1409 return callback(ctx);
1410 }
1411
1412 // The toggle to RW must come before validation. If we validate before
1413 // toggling, an attack might try to jump into the middle of the body after
1414 // the validation.
1415 os_thread_self_restrict_rwx_to_rw();
1416
1417 union _pthread_jit_config_u *config = &_pthread_jit_config;
1418 if (config->allowlist_enabled) {
1419 if (!config->allowlist_frozen) {
1420 PTHREAD_CLIENT_CRASH(config->allowlist_freeze_late,
1421 "pthread_jit_write_with_callback_np() called before "
1422 "pthread_jit_write_freeze_callbacks_np()");
1423 }
1424
1425 uint32_t i;
1426 for (i = 0; i < config->allowlist_count; i++) {
1427 if (config->allowlist[i] == callback) {
1428 break;
1429 }
1430 }
1431
1432 if (i == config->allowlist_count) {
1433 // Just to be safe, toggle back to RX before bailing out
1434 os_thread_self_restrict_rwx_to_rx();
1435
1436 PTHREAD_CLIENT_CRASH((uintptr_t)callback,
1437 "pthread_jit_write_with_callback_np() callback not in allowlist");
1438 }
1439 }
1440
1441 int rv = callback(ctx);
1442
1443 os_thread_self_restrict_rwx_to_rx();
1444
1445 return rv;
1446#else // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1447 return callback(ctx);
1448#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1449}
1450
1451void
1452pthread_jit_write_freeze_callbacks_np(void)
1453{
1454#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1455 if (!os_thread_self_restrict_rwx_is_supported()) {
1456 return;
1457 }
1458
1459 union _pthread_jit_config_u *config = &_pthread_jit_config;
1460
1461 if (!config->allowlist_freeze_late) {
1462 PTHREAD_CLIENT_CRASH(0, "pthread_jit_write_freeze_callbacks_np() "
1463 "called but freeze-late entitlement is missing");
1464 }
1465
1466 _pthread_lock_lock(&_pthread_jit_config_lock);
1467 if (config->allowlist_frozen) {
1468 PTHREAD_CLIENT_CRASH(0,
1469 "pthread_jit_write_freeze_callbacks_np() already called");
1470 }
1471 config->allowlist_frozen = true;
1472 _pthread_lock_unlock(&_pthread_jit_config_lock);
1473
1474 _pthread_jit_write_protect_freeze_config();
1475#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
1476}
1477
1478#endif // TARGET_OS_OSX || PTHREAD_TARGET_OS_IOS_ONLY
1479
1480OS_ALWAYS_INLINE
1481static inline void
1482__pthread_add_thread(pthread_t t, mach_port_t self, bool from_mach_thread)
1483{
1484 _pthread_lock_lock(&_pthread_list_lock, self);
1485 TAILQ_INSERT_TAIL(&__pthread_head, t, tl_plist);
1486 _pthread_count++;
1487 _pthread_lock_unlock(&_pthread_list_lock, self);
1488
1489 if (!from_mach_thread) {
1490 // PR-26275485: Mach threads will likely crash trying to run
1491 // introspection code. Since the fall out from the introspection
1492 // code not seeing the injected thread is likely less than crashing
1493 // in the introspection code, just don't make the call.
1494 _pthread_introspection_thread_create(t);
1495 }
1496}
1497
1498OS_ALWAYS_INLINE
1499static inline void
1500__pthread_undo_add_thread(pthread_t t, mach_port_t self)
1501{
1502 _pthread_lock_lock(&_pthread_list_lock, self);
1503 TAILQ_REMOVE(&__pthread_head, t, tl_plist);
1504 _pthread_count--;
1505 _pthread_lock_unlock(&_pthread_list_lock, self);
1506}
1507
1508OS_ALWAYS_INLINE
1509static inline void
1510__pthread_started_thread(pthread_t t)
1511{
1512 mach_port_t kport = _pthread_tsd_slot(t, MACH_THREAD_SELF);
1513 if (os_unlikely(!MACH_PORT_VALID(kport))) {
1514 PTHREAD_CLIENT_CRASH(kport,
1515 "Unable to allocate thread port, possible port leak");
1516 }
1517 _pthread_introspection_thread_start(t);
1518}
1519
1520#define _PTHREAD_CREATE_NONE 0x0
1521#define _PTHREAD_CREATE_FROM_MACH_THREAD 0x1
1522#define _PTHREAD_CREATE_SUSPENDED 0x2
1523
1524static int
1525_pthread_create(pthread_t *thread, const pthread_attr_t *attrs,
1526 void *(*start_routine)(void *), void *arg, unsigned int create_flags)
1527{
1528 pthread_t t = NULL;
1529 void *stack = NULL;
1530 bool from_mach_thread = (create_flags & _PTHREAD_CREATE_FROM_MACH_THREAD);
1531 mach_port_t self_kport;
1532 int rc = 0;
1533
1534 if (attrs == NULL) {
1535 attrs = &_pthread_attr_default;
1536 } else if (attrs->sig != _PTHREAD_ATTR_SIG) {
1537 return EINVAL;
1538 }
1539
1540 unsigned int flags = PTHREAD_START_CUSTOM;
1541 if (attrs->schedset != 0) {
1542 struct sched_param p;
1543 _pthread_attr_get_schedparam(attrs, &p);
1544 flags |= PTHREAD_START_SETSCHED;
1545 flags |= ((attrs->policy & PTHREAD_START_POLICY_MASK) << PTHREAD_START_POLICY_BITSHIFT);
1546 flags |= (p.sched_priority & PTHREAD_START_IMPORTANCE_MASK);
1547 } else if (attrs->qosclass != 0) {
1548 flags |= PTHREAD_START_QOSCLASS;
1549 flags |= (attrs->qosclass & PTHREAD_START_QOSCLASS_MASK);
1550 }
1551 if (create_flags & _PTHREAD_CREATE_SUSPENDED) {
1552 flags |= PTHREAD_START_SUSPENDED;
1553 }
1554
1555 __is_threaded = 1;
1556
1557 t = _pthread_allocate(attrs, &stack, from_mach_thread);
1558 if (t == NULL) {
1559 return EAGAIN;
1560 }
1561
1562 if (os_unlikely(from_mach_thread)) {
1563 self_kport = mach_thread_self();
1564 } else {
1565 self_kport = _pthread_mach_thread_self_direct();
1566 }
1567
1568 t->arg = arg;
1569 t->fun = start_routine;
1570 __pthread_add_thread(t, self_kport, from_mach_thread);
1571
1572 if (__bsdthread_create(start_routine, arg, stack, t, flags) ==
1573 (pthread_t)-1) {
1574 if (errno == EMFILE) {
1575 PTHREAD_CLIENT_CRASH(0,
1576 "Unable to allocate thread port, possible port leak");
1577 }
1578 __pthread_undo_add_thread(t, self_kport);
1579 _pthread_deallocate(t, from_mach_thread);
1580 t = NULL;
1581 rc = EAGAIN;
1582 }
1583 if (from_mach_thread) {
1584 mach_port_deallocate(mach_task_self(), self_kport);
1585 }
1586
1587 // n.b. if a thread is created detached and exits, t will be invalid
1588 *thread = t;
1589 return rc;
1590}
1591
1592int
1593pthread_create(pthread_t *thread, const pthread_attr_t *attr,
1594 void *(*start_routine)(void *), void *arg)
1595{
1596 unsigned int flags = _PTHREAD_CREATE_NONE;
1597 return _pthread_create(thread, attr, start_routine, arg, flags);
1598}
1599
1600int
1601pthread_create_from_mach_thread(pthread_t *thread, const pthread_attr_t *attr,
1602 void *(*start_routine)(void *), void *arg)
1603{
1604 unsigned int flags = _PTHREAD_CREATE_FROM_MACH_THREAD;
1605 return _pthread_create(thread, attr, start_routine, arg, flags);
1606}
1607
1608int
1609pthread_create_suspended_np(pthread_t *thread, const pthread_attr_t *attr,
1610 void *(*start_routine)(void *), void *arg)
1611{
1612 unsigned int flags = _PTHREAD_CREATE_SUSPENDED;
1613 return _pthread_create(thread, attr, start_routine, arg, flags);
1614}
1615
1616void
1617pthread_install_workgroup_functions_np(
1618 const struct pthread_workgroup_functions_s *pwgf)
1619{
1620 if (_pthread_workgroup_functions) {
1621 PTHREAD_CLIENT_CRASH((uintptr_t)_pthread_workgroup_functions,
1622 "workgroup functions already installed");
1623 }
1624
1625 _pthread_workgroup_functions = pwgf;
1626}
1627
1628int
1629pthread_create_with_workgroup_np(pthread_t *thread, os_workgroup_t wg,
1630 const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
1631{
1632 if (!_pthread_workgroup_functions) {
1633 PTHREAD_CLIENT_CRASH(0, "workgroup functions not yet installed");
1634 }
1635
1636 return _pthread_workgroup_functions->pwgf_create_with_workgroup(thread, wg,
1637 attr, start_routine, arg);
1638}
1639
1640int
1641pthread_detach(pthread_t thread)
1642{
1643 int res = 0;
1644 bool join = false, wake = false;
1645
1646 if (!_pthread_validate_thread_and_list_lock(thread)) {
1647 return ESRCH;
1648 }
1649
1650 if (!thread->tl_joinable) {
1651 res = EINVAL;
1652 } else if (thread->tl_exit_gate == MACH_PORT_DEAD) {
1653 // Join the thread if it's already exited.
1654 join = true;
1655 } else {
1656 thread->tl_joinable = false; // _pthread_joiner_prepost_wake uses this
1657 if (thread->tl_join_ctx) {
1658 (void)_pthread_joiner_prepost_wake(thread);
1659 wake = true;
1660 }
1661 }
1662 _pthread_lock_unlock(&_pthread_list_lock);
1663
1664 if (join) {
1665 pthread_join(thread, NULL);
1666 } else if (wake) {
1667 _pthread_joiner_wake(thread);
1668 }
1669 return res;
1670}
1671
1672int
1673pthread_kill(pthread_t th, int sig)
1674{
1675 if (sig < 0 || sig > NSIG) {
1676 return EINVAL;
1677 }
1678
1679 mach_port_t kport = MACH_PORT_NULL;
1680 {
1681 if (!_pthread_is_valid(th, &kport)) {
1682 return ESRCH;
1683 }
1684 }
1685
1686 int ret = __pthread_kill(kport, sig);
1687
1688 if (ret == -1) {
1689 ret = errno;
1690 }
1691 return ret;
1692}
1693
1694int
1695__pthread_workqueue_setkill(int enable)
1696{
1697 {
1698 return __bsdthread_ctl(BSDTHREAD_CTL_WORKQ_ALLOW_KILL, enable, 0, 0);
1699 }
1700}
1701
1702/*
1703 * Terminate a thread.
1704 */
1705
1706OS_NORETURN
1707static void
1708_pthread_exit(pthread_t self, void *exit_value)
1709{
1710 struct __darwin_pthread_handler_rec *handler;
1711
1712 // Disable signal delivery while we clean up
1713 __disable_threadsignal(1);
1714
1715 // Set cancel state to disable and type to deferred
1716 _pthread_setcancelstate_exit(self, exit_value);
1717
1718 while ((handler = self->__cleanup_stack) != 0) {
1719 (handler->__routine)(handler->__arg);
1720 self->__cleanup_stack = handler->__next;
1721 }
1722 _pthread_tsd_cleanup(self);
1723
1724 // Clear per-thread semaphore cache
1725 os_put_cached_semaphore(SEMAPHORE_NULL);
1726
1727 _pthread_terminate_invoke(self, exit_value);
1728}
1729
1730void
1731pthread_exit(void *exit_value)
1732{
1733 pthread_t self = pthread_self();
1734 if (os_unlikely(self->wqthread)) {
1735 PTHREAD_CLIENT_CRASH(0, "pthread_exit() called from a thread "
1736 "not created by pthread_create()");
1737 }
1738 _pthread_validate_signature(self);
1739 _pthread_exit(self, exit_value);
1740}
1741
1742int
1743pthread_self_is_exiting_np(void)
1744{
1745 return (os_atomic_load(&pthread_self()->cancel_state, relaxed) &
1746 _PTHREAD_CANCEL_EXITING) != 0;
1747}
1748
1749int
1750pthread_getschedparam(pthread_t thread, int *policy, struct sched_param *param)
1751{
1752 if (!_pthread_validate_thread_and_list_lock(thread)) {
1753 return ESRCH;
1754 }
1755
1756 if (policy) *policy = thread->tl_policy;
1757 if (param) *param = thread->tl_param;
1758 _pthread_lock_unlock(&_pthread_list_lock);
1759 return 0;
1760}
1761
1762OS_ALWAYS_INLINE
1763static inline int
1764pthread_setschedparam_internal(pthread_t thread, mach_port_t kport, int policy,
1765 const struct sched_param *param)
1766{
1767 policy_base_data_t bases;
1768 policy_base_t base;
1769 mach_msg_type_number_t count;
1770 kern_return_t ret;
1771
1772 if (os_unlikely(thread->wqthread)) {
1773 return ENOTSUP;
1774 }
1775
1776 switch (policy) {
1777 case SCHED_OTHER:
1778 bases.ts.base_priority = param->sched_priority;
1779 base = (policy_base_t)&bases.ts;
1780 count = POLICY_TIMESHARE_BASE_COUNT;
1781 break;
1782 case SCHED_FIFO:
1783 bases.fifo.base_priority = param->sched_priority;
1784 base = (policy_base_t)&bases.fifo;
1785 count = POLICY_FIFO_BASE_COUNT;
1786 break;
1787 case SCHED_RR:
1788 bases.rr.base_priority = param->sched_priority;
1789 /* quantum isn't public yet */
1790 bases.rr.quantum = param->quantum;
1791 base = (policy_base_t)&bases.rr;
1792 count = POLICY_RR_BASE_COUNT;
1793 break;
1794 default:
1795 return EINVAL;
1796 }
1797 ret = thread_policy(kport, policy, base, count, TRUE);
1798 return (ret != KERN_SUCCESS) ? EINVAL : 0;
1799}
1800
1801int
1802pthread_setschedparam(pthread_t t, int policy, const struct sched_param *param)
1803{
1804 mach_port_t kport = MACH_PORT_NULL;
1805 int bypass = 1;
1806
1807 // since the main thread will not get de-allocated from underneath us
1808 if (t == pthread_self() || t == main_thread()) {
1809 _pthread_validate_signature(t);
1810 kport = _pthread_tsd_slot(t, MACH_THREAD_SELF);
1811 } else {
1812 bypass = 0;
1813 if (!_pthread_is_valid(t, &kport)) {
1814 return ESRCH;
1815 }
1816 }
1817
1818 int res = pthread_setschedparam_internal(t, kport, policy, param);
1819 if (res) return res;
1820
1821 if (bypass) {
1822 _pthread_lock_lock(&_pthread_list_lock);
1823 } else if (!_pthread_validate_thread_and_list_lock(t)) {
1824 // Ensure the thread is still valid.
1825 return ESRCH;
1826 }
1827
1828 t->tl_policy = policy;
1829 t->tl_param = *param;
1830 _pthread_lock_unlock(&_pthread_list_lock);
1831 return 0;
1832}
1833
1834int
1835sched_get_priority_min(int policy)
1836{
1837 return default_priority - 16;
1838}
1839
1840int
1841sched_get_priority_max(int policy)
1842{
1843 return default_priority + 16;
1844}
1845
1846int
1847pthread_equal(pthread_t t1, pthread_t t2)
1848{
1849 return (t1 == t2);
1850}
1851
1852OS_NOINLINE
1853void
1854_pthread_set_self(pthread_t p)
1855{
1856#if VARIANT_DYLD
1857 if (os_likely(!p)) {
1858 return _pthread_set_self_dyld();
1859 }
1860#endif // VARIANT_DYLD
1861 _pthread_set_self_internal(p);
1862 _thread_set_tsd_base(&p->tsd[0]);
1863}
1864
1865#if VARIANT_DYLD
1866// _pthread_set_self_dyld is noinline+noexport to allow the option for
1867// static libsyscall to adopt this as the entry point from mach_init if
1868// desired
1869OS_NOINLINE
1870static void
1871_pthread_set_self_dyld(void)
1872{
1873 pthread_t p = main_thread();
1874 p->thread_id = __thread_selfid();
1875
1876 if (os_unlikely(p->thread_id == -1ull)) {
1877 PTHREAD_INTERNAL_CRASH(0, "failed to set thread_id");
1878 }
1879
1880 // <rdar://problem/40930651> pthread self and the errno address are the
1881 // bare minimium TSD setup that dyld needs to actually function. Without
1882 // this, TSD access will fail and crash if it uses bits of Libc prior to
1883 // library initialization. __pthread_init will finish the initialization
1884 // during library init.
1885 _pthread_tsd_slot(p, PTHREAD_SELF) = p;
1886 _pthread_tsd_slot(p, ERRNO) = &p->err_no;
1887 _thread_set_tsd_base(&p->tsd[0]);
1888}
1889#endif // VARIANT_DYLD
1890
1891OS_ALWAYS_INLINE
1892static inline void
1893_pthread_set_self_internal(pthread_t p)
1894{
1895#if __LP64__
1896 os_atomic_store(&p->thread_id, __thread_selfid(), relaxed);
1897#else
1898 os_atomic_store_wide(&p->thread_id, __thread_selfid(), relaxed);
1899#endif
1900
1901 if (os_unlikely(p->thread_id == -1ull)) {
1902 PTHREAD_INTERNAL_CRASH(0, "failed to set thread_id");
1903 }
1904}
1905
1906// <rdar://problem/28984807> pthread_once should have an acquire barrier
1907OS_ALWAYS_INLINE
1908static inline void
1909_os_once_acquire(os_once_t *predicate, void *context, os_function_t function)
1910{
1911 if (OS_EXPECT(os_atomic_load(predicate, acquire), ~0l) != ~0l) {
1912 _os_once(predicate, context, function);
1913 OS_COMPILER_CAN_ASSUME(*predicate == ~0l);
1914 }
1915}
1916
1917struct _pthread_once_context {
1918 pthread_once_t *pthread_once;
1919 void (*routine)(void);
1920};
1921
1922static void
1923__pthread_once_handler(void *context)
1924{
1925 struct _pthread_once_context *ctx = context;
1926 pthread_cleanup_push((void*)__os_once_reset, &ctx->pthread_once->once);
1927 ctx->routine();
1928 pthread_cleanup_pop(0);
1929 ctx->pthread_once->sig = _PTHREAD_ONCE_SIG;
1930}
1931
1932int
1933pthread_once(pthread_once_t *once_control, void (*init_routine)(void))
1934{
1935 struct _pthread_once_context ctx = { once_control, init_routine };
1936 do {
1937 _os_once_acquire(&once_control->once, &ctx, __pthread_once_handler);
1938 } while (once_control->sig == _PTHREAD_ONCE_SIG_init);
1939 return 0;
1940}
1941
1942int
1943pthread_getconcurrency(void)
1944{
1945 return pthread_concurrency;
1946}
1947
1948int
1949pthread_setconcurrency(int new_level)
1950{
1951 if (new_level < 0) {
1952 return EINVAL;
1953 }
1954 pthread_concurrency = new_level;
1955 return 0;
1956}
1957
1958/*
1959 * Perform package initialization - called automatically when application starts
1960 */
1961
1962#if !VARIANT_DYLD
1963static unsigned long
1964_pthread_strtoul(const char *p, const char **endptr, int base)
1965{
1966 uintptr_t val = 0;
1967
1968 // Expect hex string starting with "0x"
1969 if ((base == 16 || base == 0) && p && p[0] == '0' && p[1] == 'x') {
1970 p += 2;
1971 while (1) {
1972 char c = *p;
1973 if ('0' <= c && c <= '9') {
1974 val = (val << 4) + (c - '0');
1975 } else if ('a' <= c && c <= 'f') {
1976 val = (val << 4) + (c - 'a' + 10);
1977 } else if ('A' <= c && c <= 'F') {
1978 val = (val << 4) + (c - 'A' + 10);
1979 } else {
1980 break;
1981 }
1982 ++p;
1983 }
1984 }
1985
1986 *endptr = (char *)p;
1987 return val;
1988}
1989
1990static int
1991parse_main_stack_params(const char *apple[],
1992 void **stackaddr,
1993 size_t *stacksize,
1994 void **allocaddr,
1995 size_t *allocsize)
1996{
1997 const char *p = _simple_getenv(apple, "main_stack");
1998 if (!p) return 0;
1999
2000 int ret = 0;
2001 const char *s = p;
2002
2003 *stackaddr = _pthread_strtoul(s, &s, 16);
2004 if (*s != ',') goto out;
2005
2006 *stacksize = _pthread_strtoul(s + 1, &s, 16);
2007 if (*s != ',') goto out;
2008
2009 *allocaddr = _pthread_strtoul(s + 1, &s, 16);
2010 if (*s != ',') goto out;
2011
2012 *allocsize = _pthread_strtoul(s + 1, &s, 16);
2013 if (*s != ',' && *s != 0) goto out;
2014
2015 ret = 1;
2016out:
2017 bzero((char *)p, strlen(p));
2018 return ret;
2019}
2020
2021static void
2022parse_ptr_munge_params(const char *envp[], const char *apple[])
2023{
2024 const char *p, *s;
2025 uintptr_t token = 0;
2026 p = _simple_getenv(apple, "ptr_munge");
2027 if (p) {
2028 token = _pthread_strtoul(p, &s, 16);
2029 bzero((char *)p, strlen(p));
2030 }
2031 /*
2032 * In DEBUG we allow the environment variable to override the kernel
2033 * security setting, including setting it to 0 which is helpful during
2034 * debugging sessions.
2035 *
2036 * For other cases, the token must be set by the kernel or the environment
2037 * variable to a non 0 value.
2038 */
2039#if !DEBUG
2040 if (!token) {
2041#endif
2042 p = _simple_getenv(envp, "PTHREAD_PTR_MUNGE_TOKEN");
2043 if (p) {
2044 uintptr_t t = _pthread_strtoul(p, &s, 16);
2045 if (t) token = t;
2046 }
2047#if !DEBUG
2048 }
2049
2050 if (!token) {
2051 PTHREAD_INTERNAL_CRASH(token, "Token from the kernel is 0");
2052 }
2053#endif // !DEBUG
2054
2055 _pthread_ptr_munge_token = token;
2056 // we need to refresh the main thread signature now that we changed
2057 // the munge token. We need to do it while TSAN will not look at it
2058 _pthread_init_signature(_main_thread_ptr);
2059}
2060
2061static void
2062parse_main_thread_port(const char *apple[], mach_port_name_t *main_th)
2063{
2064 const char *p, *s;
2065 p = _simple_getenv(apple, "th_port");
2066 if (p) {
2067 *main_th = (mach_port_name_t)_pthread_strtoul(p, &s, 16);
2068 bzero((char *)p, strlen(p));
2069 }
2070}
2071
2072int
2073__pthread_init(const struct _libpthread_functions *pthread_funcs,
2074 const char *envp[], const char *apple[],
2075 const struct ProgramVars *vars __unused)
2076{
2077 // Save our provided pushed-down functions
2078 if (pthread_funcs) {
2079 exitf = pthread_funcs->exit;
2080
2081 if (pthread_funcs->version >= 2) {
2082 _pthread_malloc = pthread_funcs->malloc;
2083 _pthread_free = pthread_funcs->free;
2084 }
2085 }
2086
2087 // libpthread.a in dyld "owns" the main thread structure itself and sets
2088 // up the tsd to point to it. So take the pthread_self() from there
2089 // and make it our main thread point.
2090 pthread_t thread = _pthread_self_direct();
2091 if (os_unlikely(thread == NULL)) {
2092 PTHREAD_INTERNAL_CRASH(0, "PTHREAD_SELF TSD not initialized");
2093 }
2094 _main_thread_ptr = thread;
2095 // this needs to be done early so that pthread_self() works in TSAN
2096 _pthread_init_signature(thread);
2097
2098 //
2099 // Get host information
2100 //
2101
2102 kern_return_t kr;
2103 host_flavor_t flavor = HOST_PRIORITY_INFO;
2104 mach_msg_type_number_t count = HOST_PRIORITY_INFO_COUNT;
2105 host_priority_info_data_t priority_info;
2106 host_t host = mach_host_self();
2107 kr = host_info(host, flavor, (host_info_t)&priority_info, &count);
2108 if (kr != KERN_SUCCESS) {
2109 PTHREAD_INTERNAL_CRASH(kr, "host_info() failed");
2110 } else {
2111 default_priority = (uint8_t)priority_info.user_priority;
2112 min_priority = (uint8_t)priority_info.minimum_priority;
2113 max_priority = (uint8_t)priority_info.maximum_priority;
2114 }
2115 mach_port_deallocate(mach_task_self(), host);
2116
2117 //
2118 // Set up the main thread structure
2119 //
2120
2121 // Get the address and size of the main thread's stack from the kernel.
2122 void *stackaddr = 0;
2123 size_t stacksize = 0;
2124 void *allocaddr = 0;
2125 size_t allocsize = 0;
2126 if (!parse_main_stack_params(apple, &stackaddr, &stacksize, &allocaddr, &allocsize) ||
2127 stackaddr == NULL || stacksize == 0) {
2128 // Fall back to previous bevhaior.
2129 size_t len = sizeof(stackaddr);
2130 int mib[] = { CTL_KERN, KERN_USRSTACK };
2131 if (__sysctl(mib, 2, &stackaddr, &len, NULL, 0) != 0) {
2132#if defined(__LP64__)
2133 stackaddr = (void *)USRSTACK64;
2134#else
2135 stackaddr = (void *)USRSTACK;
2136#endif
2137 }
2138 stacksize = DFLSSIZ;
2139 allocaddr = 0;
2140 allocsize = 0;
2141 }
2142
2143 // Initialize random ptr_munge token from the kernel.
2144 parse_ptr_munge_params(envp, apple);
2145
2146 PTHREAD_DEBUG_ASSERT(_pthread_attr_default.qosclass ==
2147 _pthread_default_priority(0));
2148 _pthread_struct_init(thread, &_pthread_attr_default,
2149 stackaddr, stacksize, allocaddr, allocsize);
2150 thread->tl_joinable = true;
2151
2152 // Get main thread port name from the kernel.
2153 mach_port_name_t main_th_port = MACH_PORT_NULL;
2154 parse_main_thread_port(apple, &main_th_port);
2155
2156 // Finish initialization with common code that is reinvoked on the
2157 // child side of a fork.
2158
2159 // Finishes initialization of main thread attributes.
2160 // Initializes the thread list and add the main thread.
2161 // Calls _pthread_set_self() to prepare the main thread for execution.
2162 _pthread_main_thread_init(thread, main_th_port);
2163
2164 struct _pthread_registration_data registration_data;
2165 // Set up kernel entry points with __bsdthread_register.
2166 _pthread_bsdthread_init(®istration_data);
2167
2168 // Have pthread_key and pthread_mutex do their init envvar checks.
2169 _pthread_key_global_init(envp);
2170 _pthread_mutex_global_init(envp, ®istration_data);
2171#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
2172 _pthread_jit_write_protect_global_init();
2173#endif
2174
2175#if PTHREAD_DEBUG_LOG
2176 _SIMPLE_STRING path = _simple_salloc();
2177 _simple_sprintf(path, "/var/tmp/libpthread.%d.log", getpid());
2178 _pthread_debuglog = open(_simple_string(path),
2179 O_WRONLY | O_APPEND | O_CREAT | O_NOFOLLOW | O_CLOEXEC, 0666);
2180 _simple_sfree(path);
2181 _pthread_debugstart = mach_absolute_time();
2182#endif
2183
2184 const char *envvar = _simple_getenv(envp, "PTHREAD_YIELD_TO_ZERO");
2185 if (envvar) {
2186 pthread_yield_to_zero = (envvar[0] == '1');
2187 }
2188
2189
2190 return 0;
2191}
2192
2193// NOTE: all releases aligned with macOS 11.4 export __pthread_late_init(), but
2194// it is only currently called on TARGET_OS_OSX.
2195void
2196__pthread_late_init(const char *envp[], const char *apple[],
2197 const struct ProgramVars *vars)
2198{
2199#if _PTHREAD_CONFIG_JIT_WRITE_PROTECT
2200 _pthread_jit_write_protect_late_init();
2201#endif // _PTHREAD_CONFIG_JIT_WRITE_PROTECT
2202}
2203
2204#endif // !VARIANT_DYLD
2205
2206void
2207_pthread_main_thread_init(pthread_t p, mach_port_name_t main_thread_port)
2208{
2209 TAILQ_INIT(&__pthread_head);
2210 _pthread_lock_init(&_pthread_list_lock);
2211 _pthread_lock_init(&p->lock);
2212 p->__cleanup_stack = NULL;
2213 p->tl_join_ctx = NULL;
2214 p->tl_exit_gate = MACH_PORT_NULL;
2215
2216 if (main_thread_port != MACH_PORT_NULL) {
2217 _pthread_tsd_slot(p, MACH_THREAD_SELF) = main_thread_port;
2218 } else {
2219 // Can't get thread port from kernel or we are forking, fallback to mach_thread_self
2220 _pthread_tsd_slot(p, MACH_THREAD_SELF) = mach_thread_self();
2221 }
2222
2223 mach_port_t mig_reply_port = MACH_PORT_NULL;
2224 mach_port_options_t opts = {
2225 .flags = MPO_REPLY_PORT,
2226 };
2227 mach_port_construct(mach_task_self(), &opts, NULL, &mig_reply_port);
2228 _pthread_tsd_slot(p, MIG_REPLY) = mig_reply_port;
2229
2230 _pthread_tsd_slot(p, MACH_SPECIAL_REPLY) = MACH_PORT_NULL;
2231 _pthread_tsd_slot(p, SEMAPHORE_CACHE) = SEMAPHORE_NULL;
2232
2233 // Initialize the list of threads with the new main thread.
2234 TAILQ_INSERT_HEAD(&__pthread_head, p, tl_plist);
2235 _pthread_count = 1;
2236
2237 _pthread_introspection_thread_start(p);
2238}
2239
2240void
2241_pthread_main_thread_postfork_init(pthread_t p)
2242{
2243 _pthread_main_thread_init(p, MACH_PORT_NULL);
2244 _pthread_set_self_internal(p);
2245}
2246
2247__attribute__((disable_tail_calls))
2248int
2249sched_yield(void)
2250{
2251 if (os_likely(pthread_yield_to_zero)) {
2252 swtch_pri(0);
2253 } else {
2254 thread_switch(THREAD_NULL, SWITCH_OPTION_NONE, 0);
2255 }
2256 return 0;
2257}
2258
2259int
2260_pthread_yield_to_enqueuer_4dispatch(unsigned long tsd_slot, void *expected_val,
2261 unsigned int timeout_ms)
2262{
2263 mach_port_t enqueuer_port = MACH_PORT_NULL;
2264
2265 _pthread_lock_lock(&_pthread_list_lock);
2266
2267 /* If there are multiple enqueuers to the same queue, this logic does not
2268 * protect us from repeatedly yielding to the same thread which may not be
2269 * able to unblock us. But that is only a major problem if that thread is
2270 * constantly enqueueing to the same queue in a loop. We've not encountered
2271 * that scenario enough to handle that here.
2272 */
2273 pthread_t p = NULL;
2274 TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
2275 if (expected_val) {
2276 if (p->tsd[tsd_slot] == expected_val) {
2277 enqueuer_port = _pthread_tsd_slot(p, MACH_THREAD_SELF);
2278 break;
2279 }
2280 } else {
2281 if (p->tsd[tsd_slot] != NULL) {
2282 enqueuer_port = _pthread_tsd_slot(p, MACH_THREAD_SELF);
2283 break;
2284 }
2285 }
2286 }
2287 _pthread_lock_unlock(&_pthread_list_lock);
2288
2289 /* This will either do a directed yield if we found a potential enqueuer, or
2290 * do a suppress.
2291 *
2292 * Technically this is racey since we are referencing the enqueuer_port
2293 * without taking an additional reference to it under the list_lock. But
2294 * thread_switch falls back to a suppressed yield if the port is not an
2295 * actual valid thread_port anymore so that's a better win for perf rather
2296 * than taking refs
2297 */
2298 thread_switch(enqueuer_port, SWITCH_OPTION_OSLOCK_DEPRESS,
2299 (mach_msg_timeout_t) timeout_ms);
2300
2301 return 0;
2302}
2303
2304// Libsystem knows about this symbol and exports it to libsyscall
2305int
2306pthread_current_stack_contains_np(const void *addr, size_t length)
2307{
2308 uintptr_t begin = (uintptr_t) addr, end;
2309 uintptr_t stack_base = (uintptr_t) _pthread_self_direct()->stackbottom;
2310 uintptr_t stack_top = (uintptr_t) _pthread_self_direct()->stackaddr;
2311
2312 if (stack_base == stack_top) {
2313 return -ENOTSUP;
2314 }
2315
2316 if (__builtin_add_overflow(begin, length, &end)) {
2317 return -EINVAL;
2318 }
2319
2320 return stack_base <= begin && end <= stack_top;
2321}
2322
2323// Libsystem knows about this symbol and exports it to libsyscall
2324
2325void
2326_pthread_clear_qos_tsd(mach_port_t port)
2327{
2328 pthread_priority_t pp = _pthread_unspecified_priority();
2329
2330 if (port == MACH_PORT_NULL || _pthread_mach_thread_self_direct() == port) {
2331 /* Clear the current thread's TSD, that can be done inline. */
2332 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS, pp);
2333 } else {
2334 pthread_t p;
2335
2336 _pthread_lock_lock(&_pthread_list_lock);
2337
2338 TAILQ_FOREACH(p, &__pthread_head, tl_plist) {
2339 mach_port_t kp = _pthread_tsd_slot(p, MACH_THREAD_SELF);
2340 if (port == kp) {
2341 _pthread_tsd_slot(p, PTHREAD_QOS_CLASS) = pp;
2342 break;
2343 }
2344 }
2345
2346 _pthread_lock_unlock(&_pthread_list_lock);
2347 }
2348}
2349
2350#pragma mark pthread/stack_np.h public interface
2351
2352#if defined(__i386__) || defined(__x86_64__) || defined(__arm__) || defined(__arm64__)
2353#if __ARM64_ARCH_8_32__
2354/*
2355 * arm64_32 uses 64-bit sizes for the frame pointer and
2356 * return address of a stack frame.
2357 */
2358typedef uint64_t frame_data_addr_t;
2359#else
2360typedef uintptr_t frame_data_addr_t;
2361#endif
2362
2363struct frame_data {
2364 frame_data_addr_t frame_addr_next;
2365 frame_data_addr_t ret_addr;
2366};
2367#else
2368#error ********** Unimplemented architecture
2369#endif
2370
2371uintptr_t
2372pthread_stack_frame_decode_np(uintptr_t frame_addr, uintptr_t *return_addr)
2373{
2374 struct frame_data *frame = (struct frame_data *)frame_addr;
2375
2376#if __has_feature(ptrauth_calls)
2377 /* In the case of Swift async backtraces, the context pointer read from
2378 * the OS stack is signed and will be passed here as `frame_addr`.
2379 * There is no harm in generally stripping this pointer which solves
2380 * this usecase.
2381 */
2382 frame = ptrauth_strip(frame, ptrauth_key_process_dependent_data);
2383#endif
2384
2385 if (return_addr) {
2386#if __has_feature(ptrauth_calls)
2387 *return_addr = (uintptr_t)ptrauth_strip((void *)frame->ret_addr,
2388 ptrauth_key_return_address);
2389#else
2390 *return_addr = (uintptr_t)frame->ret_addr;
2391#endif /* __has_feature(ptrauth_calls) */
2392 }
2393
2394 uintptr_t next_frame = frame->frame_addr_next;
2395#if __has_feature(ptrauth_calls)
2396 next_frame = (uintptr_t)ptrauth_strip((void *)next_frame,
2397 ptrauth_key_frame_pointer);
2398#endif /* __has_feature(ptrauth_calls) */
2399#if __LP64__
2400 /*
2401 * On our 64 bit platforms the top byte of the frame pointer
2402 * in userspace should always be zeros. Some ABIs might use
2403 * this byte to stash extended information about the frame
2404 * (for example the Swift concurrency ABI does this). On arm64
2405 * TBI makes dereferencing it transparent, but people might
2406 * want to do arithmetic on the resulting pointer.
2407 */
2408 /*
2409 * Strip the top byte of the pointer, unconditionally.
2410 */
2411 next_frame = next_frame & ~0xFF00000000000000ul;
2412#endif /* __LP64__ */
2413
2414 return next_frame;
2415}
2416
2417#pragma mark pthread workqueue support routines
2418
2419void
2420_pthread_bsdthread_init(struct _pthread_registration_data *data)
2421{
2422 bzero(data, sizeof(*data));
2423 data->version = sizeof(struct _pthread_registration_data);
2424 data->dispatch_queue_offset = __PTK_LIBDISPATCH_KEY0 * sizeof(void *);
2425 data->return_to_kernel_offset = __TSD_RETURN_TO_KERNEL * sizeof(void *);
2426 data->tsd_offset = offsetof(struct pthread_s, tsd);
2427 data->mach_thread_self_offset = __TSD_MACH_THREAD_SELF * sizeof(void *);
2428 data->joinable_offset_bits = CHAR_BIT * (offsetof(struct pthread_s, tl_policy) + 1);
2429 data->wq_quantum_expiry_offset = __PTK_LIBDISPATCH_KEY10 * sizeof(void *);
2430
2431 int rv = __bsdthread_register(thread_start, start_wqthread, (int)PTHREAD_SIZE,
2432 (void*)data, (uintptr_t)sizeof(*data), data->dispatch_queue_offset);
2433
2434 if (rv > 0) {
2435 int required_features =
2436 PTHREAD_FEATURE_FINEPRIO |
2437 PTHREAD_FEATURE_BSDTHREADCTL |
2438 PTHREAD_FEATURE_SETSELF |
2439 PTHREAD_FEATURE_QOS_MAINTENANCE |
2440 PTHREAD_FEATURE_QOS_DEFAULT;
2441 if ((rv & required_features) != required_features) {
2442 PTHREAD_INTERNAL_CRASH(rv, "Missing required kernel support");
2443 }
2444 __pthread_supported_features = rv;
2445 }
2446
2447 /*
2448 * TODO: differentiate between (-1, EINVAL) after fork (which has the side
2449 * effect of resetting the child's stack_addr_hint before bailing out) and
2450 * (-1, EINVAL) because of invalid arguments. We'd probably like to treat
2451 * the latter as fatal.
2452 *
2453 * <rdar://problem/36451838>
2454 */
2455
2456 pthread_priority_t main_qos = (pthread_priority_t)data->main_qos;
2457
2458 if (_pthread_priority_thread_qos(main_qos) != THREAD_QOS_UNSPECIFIED) {
2459 _pthread_set_main_qos(main_qos);
2460 _pthread_tsd_slot(main_thread(), PTHREAD_QOS_CLASS) = main_qos;
2461 }
2462
2463 if (data->stack_addr_hint) {
2464 __pthread_stack_hint = data->stack_addr_hint;
2465 }
2466
2467 if (__libdispatch_workerfunction != NULL) {
2468 // prepare the kernel for workq action
2469 (void)__workq_open();
2470 }
2471}
2472
2473OS_NOINLINE
2474static void
2475_pthread_wqthread_legacy_worker_wrap(pthread_workqueue_function2_arg_t workq_function2_arg)
2476{
2477 /* Old thread priorities are inverted from where we have them in
2478 * the new flexible priority scheme. The highest priority is zero,
2479 * up to 2, with background at 3.
2480 */
2481 pthread_workqueue_function_t func = (pthread_workqueue_function_t)__libdispatch_workerfunction;
2482 bool overcommit = (workq_function2_arg & _PTHREAD_PRIORITY_OVERCOMMIT_FLAG);
2483 int opts = overcommit ? WORKQ_ADDTHREADS_OPTION_OVERCOMMIT : 0;
2484
2485 switch (_pthread_priority_thread_qos(workq_function2_arg)) {
2486 case THREAD_QOS_USER_INITIATED:
2487 return (*func)(WORKQ_HIGH_PRIOQUEUE, opts, NULL);
2488 case THREAD_QOS_LEGACY:
2489 /* B&I builders can't pass a QOS_CLASS_DEFAULT thread to dispatch, for fear of the QoS being
2490 * picked up by NSThread (et al) and transported around the system. So change the TSD to
2491 * make this thread look like QOS_CLASS_USER_INITIATED even though it will still run as legacy.
2492 */
2493 _pthread_setspecific_direct(_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS,
2494 _pthread_priority_make_from_thread_qos(THREAD_QOS_USER_INITIATED, 0, 0));
2495 return (*func)(WORKQ_DEFAULT_PRIOQUEUE, opts, NULL);
2496 case THREAD_QOS_UTILITY:
2497 return (*func)(WORKQ_LOW_PRIOQUEUE, opts, NULL);
2498 case THREAD_QOS_BACKGROUND:
2499 return (*func)(WORKQ_BG_PRIOQUEUE, opts, NULL);
2500 }
2501 PTHREAD_INTERNAL_CRASH(workq_function2_arg, "Invalid pthread priority for the legacy interface");
2502}
2503
2504OS_ALWAYS_INLINE
2505static inline pthread_priority_t
2506_pthread_wqthread_priority(int flags)
2507{
2508 pthread_priority_t pp = 0;
2509 thread_qos_t qos;
2510
2511 if (flags & WQ_FLAG_THREAD_KEVENT) {
2512 pp |= _PTHREAD_PRIORITY_NEEDS_UNBIND_FLAG;
2513 }
2514 if (flags & WQ_FLAG_THREAD_EVENT_MANAGER) {
2515 return pp | _PTHREAD_PRIORITY_EVENT_MANAGER_FLAG;
2516 }
2517
2518 if (flags & WQ_FLAG_THREAD_OVERCOMMIT) {
2519 pp |= _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
2520 }
2521
2522 if (flags & WQ_FLAG_THREAD_PRIO_QOS) {
2523 qos = (thread_qos_t)(flags & WQ_FLAG_THREAD_PRIO_MASK);
2524 pp = _pthread_priority_make_from_thread_qos(qos, 0, pp);
2525 } else if (flags & WQ_FLAG_THREAD_PRIO_SCHED) {
2526 pp |= _PTHREAD_PRIORITY_SCHED_PRI_MASK;
2527 pp |= (flags & WQ_FLAG_THREAD_PRIO_MASK);
2528 } else {
2529 PTHREAD_INTERNAL_CRASH(flags, "Missing priority");
2530 }
2531 return pp;
2532}
2533
2534OS_NOINLINE
2535static void
2536_pthread_wqthread_setup(pthread_t self, mach_port_t kport, void *stacklowaddr,
2537 int flags)
2538{
2539 void *stackaddr = self;
2540 size_t stacksize = (uintptr_t)self - (uintptr_t)stacklowaddr;
2541
2542 _pthread_struct_init(self, &_pthread_attr_default, stackaddr, stacksize,
2543 PTHREAD_ALLOCADDR(stackaddr, stacksize),
2544 PTHREAD_ALLOCSIZE(stackaddr, stacksize));
2545
2546 _pthread_tsd_slot(self, MACH_THREAD_SELF) = kport;
2547 self->wqthread = 1;
2548 self->wqkillset = 0;
2549 self->tl_joinable = false;
2550
2551 // Update the running thread count and set childrun bit.
2552 if (os_unlikely((flags & WQ_FLAG_THREAD_TSD_BASE_SET) == 0)) {
2553 PTHREAD_INTERNAL_CRASH(flags,
2554 "thread_set_tsd_base() wasn't called by the kernel");
2555 }
2556 _pthread_set_self_internal(self);
2557 __pthread_add_thread(self, kport, false);
2558 __pthread_started_thread(self);
2559}
2560
2561OS_NORETURN OS_NOINLINE
2562static void
2563_pthread_wqthread_exit(pthread_t self)
2564{
2565 const thread_qos_t WORKQ_THREAD_QOS_CLEANUP = THREAD_QOS_LEGACY;
2566 pthread_priority_t pp = _pthread_tsd_slot(self, PTHREAD_QOS_CLASS);
2567 thread_qos_t qos;
2568
2569 qos = _pthread_priority_thread_qos(pp);
2570 if (qos == THREAD_QOS_UNSPECIFIED || qos > WORKQ_THREAD_QOS_CLEANUP) {
2571 // Reset QoS to something low for the cleanup process
2572 pp = _pthread_priority_make_from_thread_qos(WORKQ_THREAD_QOS_CLEANUP, 0, 0);
2573 _pthread_tsd_slot(self, PTHREAD_QOS_CLASS) = pp;
2574 }
2575
2576 _pthread_exit(self, NULL);
2577}
2578
2579// workqueue entry point from kernel
2580void
2581_pthread_wqthread(pthread_t self, mach_port_t kport, void *stacklowaddr,
2582 void *keventlist, int flags, int nkevents)
2583{
2584 if ((flags & WQ_FLAG_THREAD_REUSE) == 0) {
2585 _pthread_wqthread_setup(self, kport, stacklowaddr, flags);
2586 }
2587
2588 pthread_priority_t pp;
2589
2590 if (flags & WQ_FLAG_THREAD_OUTSIDEQOS) {
2591 self->wq_outsideqos = 1;
2592 pp = _pthread_priority_make_from_thread_qos(THREAD_QOS_LEGACY, 0,
2593 _PTHREAD_PRIORITY_FALLBACK_FLAG);
2594 } else {
2595 self->wq_outsideqos = 0;
2596 pp = _pthread_wqthread_priority(flags);
2597 }
2598
2599 self->tsd[_PTHREAD_TSD_SLOT_PTHREAD_QOS_CLASS] = (void *)pp;
2600
2601 // Only used for conveying to libdispatch that the thread is coming out to
2602 // handle a cooperative request.
2603 pthread_workqueue_function2_arg_t workq_function2_arg = pp;
2604 if (flags & WQ_FLAG_THREAD_COOPERATIVE) {
2605 workq_function2_arg |= _PTHREAD_PRIORITY_COOPERATIVE_FLAG;
2606 }
2607
2608 // avoid spills on the stack hard to keep used stack space minimal
2609 if (os_unlikely(nkevents == WORKQ_EXIT_THREAD_NKEVENT)) {
2610 _pthread_wqthread_exit(self);
2611 } else if (flags & WQ_FLAG_THREAD_WORKLOOP) {
2612 kqueue_id_t *kqidptr = (kqueue_id_t *)keventlist - 1;
2613 self->fun = (void *(*)(void*))__libdispatch_workloopfunction;
2614 self->arg = keventlist;
2615 self->wq_nevents = nkevents;
2616 (*__libdispatch_workloopfunction)(kqidptr, &self->arg, &self->wq_nevents);
2617 __workq_kernreturn(WQOPS_THREAD_WORKLOOP_RETURN, self->arg, self->wq_nevents, 0);
2618 } else if (flags & WQ_FLAG_THREAD_KEVENT) {
2619 self->fun = (void *(*)(void*))__libdispatch_keventfunction;
2620 self->arg = keventlist;
2621 self->wq_nevents = nkevents;
2622 (*__libdispatch_keventfunction)(&self->arg, &self->wq_nevents);
2623 __workq_kernreturn(WQOPS_THREAD_KEVENT_RETURN, self->arg, self->wq_nevents, 0);
2624 } else {
2625 self->fun = (void *(*)(void*))__libdispatch_workerfunction;
2626 self->arg = (void *)(uintptr_t)workq_function2_arg;
2627 self->wq_nevents = 0;
2628 if (os_likely(__workq_newapi)) {
2629 (*__libdispatch_workerfunction)(workq_function2_arg);
2630 } else {
2631 _pthread_wqthread_legacy_worker_wrap(workq_function2_arg);
2632 }
2633 __workq_kernreturn(WQOPS_THREAD_RETURN, NULL, 0, 0);
2634 }
2635
2636 _os_set_crash_log_cause_and_message(self->err_no,
2637 "BUG IN LIBPTHREAD: __workq_kernreturn returned");
2638 /*
2639 * 52858993: we should never return but the compiler insists on outlining,
2640 * so the __builtin_trap() is in _start_wqthread in pthread_asm.s
2641 */
2642}
2643
2644#pragma mark pthread workqueue API for libdispatch
2645
2646_Static_assert(WORKQ_KEVENT_EVENT_BUFFER_LEN == WQ_KEVENT_LIST_LEN,
2647 "Kernel and userland should agree on the event list size");
2648
2649#define PTHREAD_WORKLOOP_CREATE_OPTIONS_MASK \
2650 (uint64_t)(PTHREAD_WORKLOOP_CREATE_WITH_BOUND_THREAD)
2651
2652void
2653pthread_workqueue_setdispatchoffset_np(int offset)
2654{
2655 __workq_kernreturn(WQOPS_QUEUE_NEWSPISUPP, NULL, offset, 0x00);
2656}
2657
2658int
2659pthread_workqueue_setup(struct pthread_workqueue_config *cfg, size_t cfg_size)
2660{
2661 int rv = EBUSY;
2662 struct workq_dispatch_config wdc_cfg;
2663 size_t min_size = 0;
2664
2665 if (cfg_size < sizeof(uint32_t)) {
2666 return EINVAL;
2667 }
2668
2669 switch (cfg->version) {
2670 case 1:
2671 min_size = offsetof(struct pthread_workqueue_config, queue_label_offs);
2672 break;
2673 case 2:
2674 min_size = sizeof(struct pthread_workqueue_config);
2675 break;
2676 default:
2677 return EINVAL;
2678 }
2679
2680 if (!cfg || cfg_size < min_size) {
2681 return EINVAL;
2682 }
2683
2684 if (cfg->flags & ~PTHREAD_WORKQUEUE_CONFIG_SUPPORTED_FLAGS ||
2685 cfg->version < PTHREAD_WORKQUEUE_CONFIG_MIN_SUPPORTED_VERSION) {
2686 return ENOTSUP;
2687 }
2688
2689 if (__libdispatch_workerfunction == NULL) {
2690 __workq_newapi = true;
2691
2692 wdc_cfg.wdc_version = WORKQ_DISPATCH_CONFIG_VERSION;
2693 wdc_cfg.wdc_flags = 0;
2694 wdc_cfg.wdc_queue_serialno_offs = cfg->queue_serialno_offs;
2695#if WORKQ_DISPATCH_CONFIG_VERSION >= 2
2696 wdc_cfg.wdc_queue_label_offs = cfg->queue_label_offs;
2697#endif
2698
2699 // Tell the kernel about dispatch internals
2700 rv = (int) __workq_kernreturn(WQOPS_SETUP_DISPATCH, &wdc_cfg, sizeof(wdc_cfg), 0);
2701 if (rv == -1) {
2702 return errno;
2703 } else {
2704 __libdispatch_keventfunction = cfg->kevent_cb;
2705 __libdispatch_workloopfunction = cfg->workloop_cb;
2706 __libdispatch_workerfunction = cfg->workq_cb;
2707
2708 // Prepare the kernel for workq action
2709 (void)__workq_open();
2710 if (__is_threaded == 0) {
2711 __is_threaded = 1;
2712 }
2713
2714 return 0;
2715 }
2716 }
2717
2718 return rv;
2719}
2720
2721int
2722_pthread_workqueue_init_with_workloop(pthread_workqueue_function2_t queue_func,
2723 pthread_workqueue_function_kevent_t kevent_func,
2724 pthread_workqueue_function_workloop_t workloop_func,
2725 int offset, int flags)
2726{
2727 struct pthread_workqueue_config cfg = {
2728 .version = PTHREAD_WORKQUEUE_CONFIG_VERSION,
2729 .flags = 0,
2730 .workq_cb = queue_func,
2731 .kevent_cb = kevent_func,
2732 .workloop_cb = workloop_func,
2733 .queue_serialno_offs = offset,
2734 .queue_label_offs = 0,
2735 };
2736
2737 return pthread_workqueue_setup(&cfg, sizeof(cfg));
2738}
2739
2740int
2741_pthread_workqueue_init_with_kevent(pthread_workqueue_function2_t queue_func,
2742 pthread_workqueue_function_kevent_t kevent_func,
2743 int offset, int flags)
2744{
2745 return _pthread_workqueue_init_with_workloop(queue_func, kevent_func, NULL, offset, flags);
2746}
2747
2748int
2749_pthread_workqueue_init(pthread_workqueue_function2_t func, int offset, int flags)
2750{
2751 return _pthread_workqueue_init_with_kevent(func, NULL, offset, flags);
2752}
2753
2754int
2755pthread_workqueue_setdispatch_np(pthread_workqueue_function_t worker_func)
2756{
2757 struct pthread_workqueue_config cfg = {
2758 .version = PTHREAD_WORKQUEUE_CONFIG_VERSION,
2759 .flags = 0,
2760 .workq_cb = (uint64_t)(pthread_workqueue_function2_t)worker_func,
2761 .kevent_cb = 0,
2762 .workloop_cb = 0,
2763 .queue_serialno_offs = 0,
2764 .queue_label_offs = 0,
2765 };
2766
2767 return pthread_workqueue_setup(&cfg, sizeof(cfg));
2768}
2769
2770int
2771_pthread_workqueue_supported(void)
2772{
2773 if (os_unlikely(!__pthread_supported_features)) {
2774 PTHREAD_INTERNAL_CRASH(0, "libpthread has not been initialized");
2775 }
2776
2777 return __pthread_supported_features;
2778}
2779
2780int
2781pthread_workqueue_addthreads_np(int queue_priority, int options, int numthreads)
2782{
2783 int res = 0;
2784
2785 // Cannot add threads without a worker function registered.
2786 if (__libdispatch_workerfunction == NULL) {
2787 return EPERM;
2788 }
2789
2790 pthread_priority_t kp = 0;
2791 int compat_priority = queue_priority & WQ_FLAG_THREAD_PRIO_MASK;
2792 int flags = 0;
2793
2794 if (options & WORKQ_ADDTHREADS_OPTION_OVERCOMMIT) {
2795 flags = _PTHREAD_PRIORITY_OVERCOMMIT_FLAG;
2796 }
2797
2798#pragma clang diagnostic push
2799#pragma clang diagnostic ignored "-Wdeprecated-declarations"
2800 kp = _pthread_qos_class_encode_workqueue(compat_priority, flags);
2801#pragma clang diagnostic pop
2802
2803 res = __workq_kernreturn(WQOPS_QUEUE_REQTHREADS, NULL, numthreads, (int)kp);
2804 if (res == -1) {
2805 res = errno;
2806 }
2807 return res;
2808}
2809
2810bool
2811_pthread_workqueue_should_narrow(pthread_priority_t pri)
2812{
2813 int res = __workq_kernreturn(WQOPS_SHOULD_NARROW, NULL, (int)pri, 0);
2814 if (res == -1) {
2815 return false;
2816 }
2817 return res;
2818}
2819
2820int
2821_pthread_workqueue_addthreads(int numthreads, pthread_priority_t priority)
2822{
2823 int res = 0;
2824
2825 if (__libdispatch_workerfunction == NULL) {
2826 return EPERM;
2827 }
2828
2829#if TARGET_OS_OSX
2830 // <rdar://problem/37687655> Legacy simulators fail to boot
2831 //
2832 // Older sims set the deprecated _PTHREAD_PRIORITY_ROOTQUEUE_FLAG wrongly,
2833 // which is aliased to _PTHREAD_PRIORITY_SCHED_PRI_FLAG and that XNU
2834 // validates and rejects.
2835 //
2836 // As a workaround, forcefully unset this bit that cannot be set here
2837 // anyway.
2838 priority &= ~_PTHREAD_PRIORITY_SCHED_PRI_FLAG;
2839#endif
2840
2841 res = __workq_kernreturn(WQOPS_QUEUE_REQTHREADS, NULL, numthreads, (int)priority);
2842 if (res == -1) {
2843 res = errno;
2844 }
2845 return res;
2846}
2847
2848int
2849_pthread_workqueue_add_cooperativethreads(int numthreads, pthread_priority_t priority)
2850{
2851 int res = 0;
2852
2853 if (__libdispatch_workerfunction == NULL) {
2854 return EPERM;
2855 }
2856
2857 res = __workq_kernreturn(WQOPS_QUEUE_REQTHREADS2, NULL, numthreads, (int)priority);
2858 if (res == -1) {
2859 res = errno;
2860 }
2861 return res;
2862}
2863
2864int
2865_pthread_workqueue_set_event_manager_priority(pthread_priority_t priority)
2866{
2867 int res = __workq_kernreturn(WQOPS_SET_EVENT_MANAGER_PRIORITY, NULL, (int)priority, 0);
2868 if (res == -1) {
2869 res = errno;
2870 }
2871 return res;
2872}
2873
2874int
2875_pthread_workqueue_allow_send_signals(int preserve_signum)
2876{
2877 // For chroot, the syscall would just return EINVAL until the
2878 // build platform moves to SunburstE.
2879 sigset_t set = sigmask(preserve_signum);
2880 return __bsdthread_ctl(BSDTHREAD_CTL_WORKQ_ALLOW_SIGMASK, set, 0, 0);
2881}
2882
2883int
2884_pthread_workloop_create(uint64_t workloop_id, uint64_t options, pthread_attr_t *attr)
2885{
2886 struct kqueue_workloop_params params = {
2887 .kqwlp_version = sizeof(struct kqueue_workloop_params),
2888 .kqwlp_id = workloop_id,
2889 .kqwlp_flags = 0,
2890 };
2891
2892 if ((!attr) || (options & ~PTHREAD_WORKLOOP_CREATE_OPTIONS_MASK)) {
2893 return EINVAL;
2894 }
2895
2896 if (attr->schedset) {
2897 params.kqwlp_flags |= KQ_WORKLOOP_CREATE_SCHED_PRI;
2898 params.kqwlp_sched_pri = attr->param.sched_priority;
2899 }
2900
2901 if (attr->policyset) {
2902 params.kqwlp_flags |= KQ_WORKLOOP_CREATE_SCHED_POL;
2903 params.kqwlp_sched_pol = attr->policy;
2904 }
2905
2906 if (attr->cpupercentset) {
2907 params.kqwlp_flags |= KQ_WORKLOOP_CREATE_CPU_PERCENT;
2908 params.kqwlp_cpu_percent = attr->cpupercent;
2909 params.kqwlp_cpu_refillms = attr->refillms;
2910 }
2911
2912 if (MACH_PORT_VALID(attr->work_interval_port)) {
2913 params.kqwlp_flags |= KQ_WORKLOOP_CREATE_WORK_INTERVAL;
2914 params.kqwl_wi_port = attr->work_interval_port;
2915 }
2916
2917 if (options & PTHREAD_WORKLOOP_CREATE_WITH_BOUND_THREAD) {
2918 params.kqwlp_flags |= KQ_WORKLOOP_CREATE_WITH_BOUND_THREAD;
2919 }
2920
2921 int res = __kqueue_workloop_ctl(KQ_WORKLOOP_CREATE, 0, ¶ms,
2922 sizeof(params));
2923 if (res == -1) {
2924 res = errno;
2925 }
2926 return res;
2927}
2928
2929int
2930_pthread_workloop_destroy(uint64_t workloop_id)
2931{
2932 struct kqueue_workloop_params params = {
2933 .kqwlp_version = sizeof(struct kqueue_workloop_params),
2934 .kqwlp_id = workloop_id,
2935 };
2936
2937 int res = __kqueue_workloop_ctl(KQ_WORKLOOP_DESTROY, 0, ¶ms,
2938 sizeof(params));
2939 if (res == -1) {
2940 res = errno;
2941 }
2942 return res;
2943}
2944
2945#pragma mark Introspection SPI for libpthread.
2946
2947static pthread_introspection_hook_t _pthread_introspection_hook;
2948
2949pthread_introspection_hook_t
2950pthread_introspection_hook_install(pthread_introspection_hook_t hook)
2951{
2952 pthread_introspection_hook_t prev;
2953 prev = _pthread_atomic_xchg_ptr((void**)&_pthread_introspection_hook, hook);
2954 return prev;
2955}
2956
2957static inline void
2958_pthread_introspection_call_hook(unsigned int event,
2959 pthread_t thread, void *addr, size_t size)
2960{
2961 pthread_t self = pthread_self();
2962 uint16_t old = self->introspection;
2963 self->introspection = (uint16_t)event;
2964 _pthread_introspection_hook(event, thread, addr, size);
2965 self->introspection = old;
2966}
2967
2968OS_NOINLINE
2969static void
2970_pthread_introspection_hook_callout_thread_create(pthread_t t)
2971{
2972 _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_CREATE, t, t,
2973 PTHREAD_SIZE);
2974}
2975
2976static inline void
2977_pthread_introspection_thread_create(pthread_t t)
2978{
2979 if (os_fastpath(!_pthread_introspection_hook)) return;
2980 _pthread_introspection_hook_callout_thread_create(t);
2981}
2982
2983OS_NOINLINE
2984static void
2985_pthread_introspection_hook_callout_thread_start(pthread_t t)
2986{
2987 size_t freesize;
2988 void *freeaddr;
2989 if (t == main_thread()) {
2990 size_t stacksize = t->stackaddr - t->stackbottom;
2991 freesize = stacksize + t->guardsize;
2992 freeaddr = t->stackaddr - freesize;
2993 } else {
2994 freesize = t->freesize - PTHREAD_SIZE;
2995 freeaddr = t->freeaddr;
2996 }
2997 _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_START, t,
2998 freeaddr, freesize);
2999}
3000
3001static inline void
3002_pthread_introspection_thread_start(pthread_t t)
3003{
3004 if (os_fastpath(!_pthread_introspection_hook)) return;
3005 _pthread_introspection_hook_callout_thread_start(t);
3006}
3007
3008OS_NOINLINE
3009static void
3010_pthread_introspection_hook_callout_thread_terminate(pthread_t t)
3011{
3012 size_t freesize;
3013 void *freeaddr;
3014 if (t == main_thread()) {
3015 size_t stacksize = t->stackaddr - t->stackbottom;
3016 freesize = stacksize + t->guardsize;
3017 freeaddr = t->stackaddr - freesize;
3018 } else {
3019 freesize = t->freesize - PTHREAD_SIZE;
3020 freeaddr = t->freeaddr;
3021 }
3022 _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_TERMINATE, t,
3023 freeaddr, freesize);
3024}
3025
3026static inline void
3027_pthread_introspection_thread_terminate(pthread_t t)
3028{
3029 if (os_fastpath(!_pthread_introspection_hook)) return;
3030 _pthread_introspection_hook_callout_thread_terminate(t);
3031}
3032
3033OS_NOINLINE
3034static void
3035_pthread_introspection_hook_callout_thread_destroy(pthread_t t)
3036{
3037 _pthread_introspection_call_hook(PTHREAD_INTROSPECTION_THREAD_DESTROY, t, t,
3038 PTHREAD_SIZE);
3039}
3040
3041static inline void
3042_pthread_introspection_thread_destroy(pthread_t t)
3043{
3044 if (os_fastpath(!_pthread_introspection_hook)) return;
3045 _pthread_introspection_hook_callout_thread_destroy(t);
3046}
3047
3048#pragma mark libplatform shims
3049#if !VARIANT_DYLD
3050
3051#include <platform/string.h>
3052
3053// pthread_setup initializes large structures to 0,
3054// which the compiler turns into a library call to memset.
3055//
3056// To avoid linking against Libc, provide a simple wrapper
3057// that calls through to the libplatform primitives
3058
3059#undef memset
3060PTHREAD_NOEXPORT
3061void *
3062memset(void *b, int c, size_t len)
3063{
3064 return _platform_memset(b, c, len);
3065}
3066
3067#undef bzero
3068PTHREAD_NOEXPORT
3069void
3070bzero(void *s, size_t n)
3071{
3072 _platform_bzero(s, n);
3073}
3074
3075#undef memcpy
3076PTHREAD_NOEXPORT
3077void *
3078memcpy(void* a, const void* b, unsigned long s)
3079{
3080 return _platform_memmove(a, b, s);
3081}
3082
3083#endif // !VARIANT_DYLD