osfmk/kern/thread.h
1/*
2 * Copyright (c) 2000-2019 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_FREE_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988,1987 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: thread.h
60 * Author: Avadis Tevanian, Jr.
61 *
62 * This file contains the structure definitions for threads.
63 *
64 */
65/*
66 * Copyright (c) 1993 The University of Utah and
67 * the Computer Systems Laboratory (CSL). All rights reserved.
68 *
69 * Permission to use, copy, modify and distribute this software and its
70 * documentation is hereby granted, provided that both the copyright
71 * notice and this permission notice appear in all copies of the
72 * software, derivative works or modified versions, and any portions
73 * thereof, and that both notices appear in supporting documentation.
74 *
75 * THE UNIVERSITY OF UTAH AND CSL ALLOW FREE USE OF THIS SOFTWARE IN ITS "AS
76 * IS" CONDITION. THE UNIVERSITY OF UTAH AND CSL DISCLAIM ANY LIABILITY OF
77 * ANY KIND FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
78 *
79 * CSL requests users of this software to return to csl-dist@cs.utah.edu any
80 * improvements that they make and grant CSL redistribution rights.
81 *
82 */
83
84#ifndef _KERN_THREAD_H_
85#define _KERN_THREAD_H_
86
87#include <mach/kern_return.h>
88#include <mach/mach_types.h>
89#include <mach/mach_param.h>
90#include <mach/message.h>
91#include <mach/boolean.h>
92#include <mach/vm_param.h>
93#include <mach/thread_info.h>
94#include <mach/thread_status.h>
95#include <mach/exception_types.h>
96
97#include <kern/kern_types.h>
98#include <vm/vm_kern.h>
99#include <sys/cdefs.h>
100#include <sys/_types/_size_t.h>
101
102#ifdef MACH_KERNEL_PRIVATE
103#include <mach_assert.h>
104#include <mach_ldebug.h>
105
106#include <ipc/ipc_types.h>
107
108#include <mach/port.h>
109#include <kern/cpu_number.h>
110#include <kern/smp.h>
111#include <kern/smr_types.h>
112#include <kern/queue.h>
113
114#include <kern/timer.h>
115#include <kern/simple_lock.h>
116#include <kern/locks.h>
117#include <kern/sched.h>
118#include <kern/sched_prim.h>
119#include <mach/sfi_class.h>
120#include <kern/thread_call.h>
121#include <kern/thread_group.h>
122#include <kern/timer_call.h>
123#include <kern/task.h>
124#include <kern/exception.h>
125#include <kern/affinity.h>
126#include <kern/debug.h>
127#include <kern/block_hint.h>
128#include <kern/recount.h>
129#include <kern/turnstile.h>
130#include <kern/mpsc_queue.h>
131
132#if CONFIG_EXCLAVES
133#include <mach/exclaves.h>
134#endif /* CONFIG_EXCLAVES */
135
136#include <kern/waitq.h>
137#include <san/kasan.h>
138#include <san/kcov_data.h>
139#include <os/refcnt.h>
140
141#include <ipc/ipc_kmsg.h>
142
143#include <machine/atomic.h>
144#include <machine/cpu_data.h>
145#include <machine/thread.h>
146
147#endif /* MACH_KERNEL_PRIVATE */
148#ifdef XNU_KERNEL_PRIVATE
149/* priority queue static asserts fail for __ARM64_ARCH_8_32__ kext builds */
150#include <kern/priority_queue.h>
151#endif /* XNU_KERNEL_PRIVATE */
152
153__BEGIN_DECLS
154
155#ifdef XNU_KERNEL_PRIVATE
156#if CONFIG_TASKWATCH
157/* Taskwatch related. TODO: find this a better home */
158typedef struct task_watcher task_watch_t;
159#endif /* CONFIG_TASKWATCH */
160
161/* Thread tags; for easy identification. */
162__options_closed_decl(thread_tag_t, uint16_t, {
163 THREAD_TAG_MAINTHREAD = 0x01,
164 THREAD_TAG_CALLOUT = 0x02,
165 THREAD_TAG_IOWORKLOOP = 0x04,
166 THREAD_TAG_PTHREAD = 0x10,
167 THREAD_TAG_WORKQUEUE = 0x20,
168 THREAD_TAG_USER_JOIN = 0x40,
169 THREAD_TAG_AIO_WORKQUEUE = 0x80,
170});
171
172typedef struct thread_ro *thread_ro_t;
173
174/*!
175 * @struct thread_ro
176 *
177 * @brief
178 * A structure allocated in a read only zone that safely
179 * represents the linkages of a thread to its cred, proc, task, ...
180 *
181 * @discussion
182 * The lifetime of a @c thread_ro structure is 1:1 with that
183 * of a @c thread_t or a @c uthread_t and holding a thread reference
184 * always allows to dereference this structure safely.
185 */
186struct thread_ro {
187 struct thread *tro_owner;
188#if MACH_BSD
189 __xnu_struct_group(thread_ro_creds, tro_creds, {
190 /*
191 * @c tro_cred holds the current thread credentials.
192 *
193 * For most threads, this is a cache of the proc's
194 * credentials that has been updated at the last
195 * syscall boundary via current_cached_proc_cred_update().
196 *
197 * If the thread assumed a different identity using settid(),
198 * then the proc cached credential lives in @c tro_realcred
199 * instead.
200 */
201 struct ucred *tro_cred;
202 struct ucred *tro_realcred;
203 });
204 struct proc *tro_proc;
205 struct proc_ro *tro_proc_ro;
206#endif
207 struct task *tro_task;
208
209 struct ipc_port *tro_ports[THREAD_SELF_PORT_COUNT]; /* no right */
210#if CONFIG_CSR
211 struct ipc_port *tro_settable_self_port; /* send right */
212#endif /* CONFIG_CSR */
213
214 struct exception_action *tro_exc_actions;
215};
216
217/*
218 * Flags for `thread set status`.
219 */
220__options_decl(thread_set_status_flags_t, uint32_t, {
221 TSSF_FLAGS_NONE = 0,
222
223 /* Translate the state to user. */
224 TSSF_TRANSLATE_TO_USER = 0x01,
225
226 /* Translate the state to user. Preserve flags */
227 TSSF_PRESERVE_FLAGS = 0x02,
228
229 /* Check kernel signed flag */
230 TSSF_CHECK_USER_FLAGS = 0x04,
231
232 /* Allow only user state PTRS */
233 TSSF_ALLOW_ONLY_USER_PTRS = 0x08,
234
235 /* Generate random diversifier and stash it */
236 TSSF_RANDOM_USER_DIV = 0x10,
237
238 /* Stash sigreturn token */
239 TSSF_STASH_SIGRETURN_TOKEN = 0x20,
240
241 /* Check sigreturn token */
242 TSSF_CHECK_SIGRETURN_TOKEN = 0x40,
243
244 /* Allow only matching sigreturn token */
245 TSSF_ALLOW_ONLY_MATCHING_TOKEN = 0x80,
246
247 /* Stash diversifier from thread */
248 TSSF_THREAD_USER_DIV = 0x100,
249
250 /* Check for entitlement */
251 TSSF_CHECK_ENTITLEMENT = 0x200,
252
253 /* Stash diversifier from task */
254 TSSF_TASK_USER_DIV = 0x400,
255
256 /* Only take the PC from the new thread state */
257 TSSF_ONLY_PC = 0x800,
258});
259
260/*
261 * Size in bits of compact thread id (ctid).
262 */
263#define CTID_SIZE_BIT 20
264typedef uint32_t ctid_t;
265
266#endif /* XNU_KERNEL_PRIVATE */
267#ifdef MACH_KERNEL_PRIVATE
268
269extern zone_t thread_ro_zone;
270
271__options_decl(thread_work_interval_flags_t, uint32_t, {
272 TH_WORK_INTERVAL_FLAGS_NONE = 0x0,
273#if CONFIG_SCHED_AUTO_JOIN
274 /* Flags to indicate status about work interval thread is currently part of */
275 TH_WORK_INTERVAL_FLAGS_AUTO_JOIN_LEAK = 0x1,
276#endif /* CONFIG_SCHED_AUTO_JOIN */
277 TH_WORK_INTERVAL_FLAGS_HAS_WORKLOAD_ID = 0x2,
278 TH_WORK_INTERVAL_FLAGS_RT_ALLOWED = 0x4,
279});
280
281#if CONFIG_EXCLAVES
282/* Thread exclaves interrupt-safe state bits (ORd) */
283__options_decl(thread_exclaves_intstate_flags_t, uint32_t, {
284 /* Thread is currently executing in secure kernel or exclaves userspace
285 * or was interrupted/preempted while doing so. */
286 TH_EXCLAVES_EXECUTION = 0x1,
287});
288
289__options_decl(thread_exclaves_state_flags_t, uint16_t, {
290/* Thread exclaves state bits (ORd) */
291 /* Thread is handling RPC from a client in xnu or Darwin userspace (but
292 * may have returned to xnu due to an exclaves scheduler request or having
293 * upcalled). Must not re-enter exclaves via RPC or return to Darwin
294 * userspace. */
295 TH_EXCLAVES_RPC = 0x1,
296 /* Thread has made an upcall RPC request back into xnu while handling RPC
297 * into exclaves from a client in xnu or Darwin userspace. Must not
298 * re-enter exclaves via RPC or return to Darwin userspace. */
299 TH_EXCLAVES_UPCALL = 0x2,
300 /* Thread has made an exclaves scheduler request (such as a wait or wake)
301 * from the xnu scheduler while handling RPC into exclaves from a client in
302 * xnu or Darwin userspace. Must not re-enter exclaves via RPC or return to
303 * Darwin userspace. */
304 TH_EXCLAVES_SCHEDULER_REQUEST = 0x4,
305 /* Thread is calling into xnu proxy server directly (but may have
306 * returned to xnu due to an exclaves scheduler request or having
307 * upcalled). Must not re-enter exclaves or return to Darwin userspace.
308 */
309 TH_EXCLAVES_XNUPROXY = 0x8,
310 /* Thread is calling into the exclaves scheduler directly.
311 * Must not re-enter exclaves or return to Darwin userspace.
312 */
313 TH_EXCLAVES_SCHEDULER_CALL = 0x10,
314 /* Thread has called the stop upcall and once the thread returns from
315 * downcall, exit_with_reason needs to be called on the task.
316 */
317 TH_EXCLAVES_STOP_UPCALL_PENDING = 0x20,
318 /* Thread is expecting that an exclaves-side thread may be spawned.
319 */
320 TH_EXCLAVES_SPAWN_EXPECTED = 0x40,
321 /* Thread is resuming the panic thread.
322 * Must not re-enter exclaves or return to Darwin userspace.
323 */
324 TH_EXCLAVES_RESUME_PANIC_THREAD = 0x80,
325});
326#define TH_EXCLAVES_STATE_ANY ( \
327 TH_EXCLAVES_RPC | \
328 TH_EXCLAVES_UPCALL | \
329 TH_EXCLAVES_SCHEDULER_REQUEST | \
330 TH_EXCLAVES_XNUPROXY | \
331 TH_EXCLAVES_SCHEDULER_CALL | \
332 TH_EXCLAVES_RESUME_PANIC_THREAD)
333
334__options_decl(thread_exclaves_inspection_flags_t, uint16_t, {
335 /* Thread is on Stackshot's inspection queue */
336 TH_EXCLAVES_INSPECTION_STACKSHOT = 0x1,
337 /* Thread is on Kperf's inspection queue */
338 TH_EXCLAVES_INSPECTION_KPERF = 0x2,
339 /* Thread must not be inspected (may deadlock, etc.) - set by collector thread*/
340 TH_EXCLAVES_INSPECTION_NOINSPECT = 0x8000,
341});
342
343#endif /* CONFIG_EXCLAVES */
344
345typedef union thread_rr_state {
346 uint32_t trr_value;
347 struct {
348#define TRR_FAULT_NONE 0
349#define TRR_FAULT_PENDING 1
350#define TRR_FAULT_OBSERVED 2
351 /*
352 * Set to TRR_FAULT_PENDING with interrupts disabled
353 * by the thread when it is entering a user fault codepath.
354 *
355 * Moved to TRR_FAULT_OBSERVED from TRR_FAULT_PENDING:
356 * - by the thread if at IPI time,
357 * - or by task_restartable_ranges_synchronize() if the thread
358 * is interrupted (under the thread lock)
359 *
360 * Cleared by the thread when returning from a user fault
361 * codepath.
362 */
363 uint8_t trr_fault_state;
364
365 /*
366 * Set by task_restartable_ranges_synchronize()
367 * if trr_fault_state is TRR_FAULT_OBSERVED
368 * and a rendez vous at the AST is required.
369 *
370 * Set atomically if trr_fault_state == TRR_FAULT_OBSERVED,
371 * and trr_ipi_ack_pending == 0
372 */
373 uint8_t trr_sync_waiting;
374
375 /*
376 * Updated under the thread_lock(),
377 * set by task_restartable_ranges_synchronize()
378 * when the thread was IPIed and the caller is waiting
379 * for an ACK.
380 */
381 uint16_t trr_ipi_ack_pending;
382 };
383} thread_rr_state_t;
384
385struct thread {
386#if MACH_ASSERT
387#define THREAD_MAGIC 0x1234ABCDDCBA4321ULL
388 /* Ensure nothing uses &thread as a queue entry */
389 uint64_t thread_magic;
390#endif /* MACH_ASSERT */
391
392 /*
393 * NOTE: The runq field in the thread structure has an unusual
394 * locking protocol. If its value is PROCESSOR_NULL, then it is
395 * locked by the thread_lock, but if its value is something else
396 * then it is locked by the associated run queue lock. It is
397 * set to PROCESSOR_NULL without holding the thread lock, but the
398 * transition from PROCESSOR_NULL to non-null must be done
399 * under the thread lock and the run queue lock. To enforce the
400 * protocol, runq should only be accessed using the
401 * thread_get/set/clear_runq functions and locked variants below.
402 *
403 * New waitq APIs allow the 'links' and '__runq' fields to be
404 * anywhere in the thread structure.
405 */
406 union {
407 queue_chain_t runq_links; /* run queue links */
408 queue_chain_t wait_links; /* wait queue links */
409 struct mpsc_queue_chain mpsc_links; /* thread daemon mpsc links */
410 struct priority_queue_entry_sched wait_prioq_links; /* priority ordered waitq links */
411 };
412
413 event64_t wait_event; /* wait queue event */
414 struct { processor_t runq; } __runq; /* internally managed run queue assignment, see above comment */
415 waitq_t waitq; /* wait queue this thread is enqueued on */
416 struct turnstile *turnstile; /* thread's turnstile, protected by primitives interlock */
417 void *inheritor; /* inheritor of the primitive the thread will block on */
418 struct priority_queue_sched_max sched_inheritor_queue; /* Inheritor queue for kernel promotion */
419 struct priority_queue_sched_max base_inheritor_queue; /* Inheritor queue for user promotion */
420
421#if CONFIG_SCHED_EDGE
422 bool th_bound_cluster_enqueued;
423 bool th_shared_rsrc_enqueued[CLUSTER_SHARED_RSRC_TYPE_COUNT];
424 bool th_shared_rsrc_heavy_user[CLUSTER_SHARED_RSRC_TYPE_COUNT];
425 bool th_shared_rsrc_heavy_perf_control[CLUSTER_SHARED_RSRC_TYPE_COUNT];
426 /*
427 * Caution! These bits should only be written/read by the current,
428 * just previous, or thread_setrun()-ing processor, before this
429 * thread can be picked up to run by its next processor.
430 */
431 uint8_t
432 th_expired_quantum_on_lower_core:1,
433 th_expired_quantum_on_higher_core:1,
434 :0;
435#endif /* CONFIG_SCHED_EDGE */
436
437#if CONFIG_SCHED_CLUTCH
438 /*
439 * In the clutch scheduler, the threads are maintained in runqs at the clutch_bucket
440 * level (clutch_bucket defines a unique thread group and scheduling bucket pair). The
441 * thread is linked via a couple of linkages in the clutch bucket:
442 *
443 * - A stable priority queue linkage which is the main runqueue (based on sched_pri) for the clutch bucket
444 * - A regular priority queue linkage which is based on thread's base/promoted pri (used for clutch bucket priority calculation)
445 * - A queue linkage used for timesharing operations of threads at the scheduler tick
446 */
447 struct priority_queue_entry_stable th_clutch_runq_link;
448 struct priority_queue_entry_sched th_clutch_pri_link;
449 queue_chain_t th_clutch_timeshare_link;
450#endif /* CONFIG_SCHED_CLUTCH */
451
452 /* Data updated during assert_wait/thread_wakeup */
453 decl_simple_lock_data(, sched_lock); /* scheduling lock (thread_lock()) */
454 decl_simple_lock_data(, wake_lock); /* for thread stop / wait (wake_lock()) */
455 uint16_t options; /* options set by thread itself */
456#define TH_OPT_INTMASK 0x0003 /* interrupt / abort level */
457#define TH_OPT_VMPRIV 0x0004 /* may allocate reserved memory */
458#define TH_OPT_SYSTEM_CRITICAL 0x0010 /* Thread must always be allowed to run - even under heavy load */
459#define TH_OPT_PROC_CPULIMIT 0x0020 /* Thread has a task-wide CPU limit applied to it */
460#define TH_OPT_PRVT_CPULIMIT 0x0040 /* Thread has a thread-private CPU limit applied to it */
461#define TH_OPT_IDLE_THREAD 0x0080 /* Thread is a per-processor idle thread */
462#define TH_OPT_GLOBAL_FORCED_IDLE 0x0100 /* Thread performs forced idle for thermal control */
463#define TH_OPT_SCHED_VM_GROUP 0x0200 /* Thread belongs to special scheduler VM group */
464#define TH_OPT_HONOR_QLIMIT 0x0400 /* Thread will honor qlimit while sending mach_msg, regardless of MACH_SEND_ALWAYS */
465#define TH_OPT_SEND_IMPORTANCE 0x0800 /* Thread will allow importance donation from kernel rpc */
466#define TH_OPT_ZONE_PRIV 0x1000 /* Thread may use the zone replenish reserve */
467#define TH_OPT_IPC_TG_BLOCKED 0x2000 /* Thread blocked in sync IPC and has made the thread group blocked callout */
468#define TH_OPT_FORCED_LEDGER 0x4000 /* Thread has a forced CPU limit */
469#define TH_IN_MACH_EXCEPTION 0x8000 /* Thread is currently handling a mach exception */
470#if CONFIG_EXCLAVES
471#define TH_OPT_AOE 0x10000 /* Thread is an AOE exclave thread */
472#endif /* CONFIG_EXCLAVES */
473
474 bool wake_active; /* wake event on stop */
475 bool at_safe_point; /* thread_abort_safely allowed */
476 uint8_t sched_saved_run_weight;
477#if DEVELOPMENT || DEBUG
478 bool pmap_footprint_suspended;
479#endif /* DEVELOPMENT || DEBUG */
480
481
482 ast_t reason; /* why we blocked */
483 uint32_t quantum_remaining;
484 wait_result_t wait_result; /* outcome of wait -
485 * may be examined by this thread
486 * WITHOUT locking */
487 thread_rr_state_t t_rr_state; /* state for restartable ranges */
488 thread_continue_t continuation; /* continue here next dispatch */
489 void *parameter; /* continuation parameter */
490
491 /* Data updated/used in thread_invoke */
492 vm_offset_t kernel_stack; /* current kernel stack */
493 vm_offset_t reserved_stack; /* reserved kernel stack */
494
495 /*** Machine-dependent state ***/
496 struct machine_thread machine;
497
498#if KASAN
499 struct kasan_thread_data kasan_data;
500#endif
501#if CONFIG_KCOV
502 kcov_thread_data_t kcov_data;
503#endif
504
505 /* Thread state: */
506 int state;
507/*
508 * Thread states [bits or'ed]
509 * All but TH_WAIT_REPORT are encoded in SS_TH_FLAGS
510 * All are encoded in kcdata.py ('ths_state')
511 */
512#define TH_WAIT 0x01 /* queued for waiting */
513#define TH_SUSP 0x02 /* stopped or requested to stop */
514#define TH_RUN 0x04 /* running or on runq */
515#define TH_UNINT 0x08 /* waiting uninteruptibly */
516#define TH_TERMINATE 0x10 /* halted at termination */
517#define TH_TERMINATE2 0x20 /* added to termination queue */
518#define TH_WAIT_REPORT 0x40 /* the wait is using the sched_call,
519 * only set if TH_WAIT is also set */
520#define TH_IDLE 0x80 /* idling processor */
521#define TH_WAKING 0x100 /* between waitq remove and thread_go */
522
523 /* Scheduling information */
524 sched_mode_t sched_mode; /* scheduling mode */
525 sched_mode_t saved_mode; /* saved mode during forced mode demotion */
526
527 /* This thread's contribution to global sched counters */
528 sched_bucket_t th_sched_bucket;
529
530 sfi_class_id_t sfi_class; /* SFI class (XXX Updated on CSW/QE/AST) */
531 sfi_class_id_t sfi_wait_class; /* Currently in SFI wait for this class, protected by sfi_lock */
532
533 uint32_t sched_flags; /* current flag bits */
534#if CONFIG_SCHED_SMT
535#define TH_SFLAG_NO_SMT 0x0001 /* On an SMT CPU, this thread must be scheduled alone */
536#endif /* CONFIG_SCHED_SMT */
537
538#define TH_SFLAG_FAILSAFE 0x0002 /* fail-safe has tripped */
539#define TH_SFLAG_THROTTLED 0x0004 /* throttled thread forced to timeshare mode (may be applied in addition to failsafe) */
540
541/* unused TH_SFLAG_PROMOTED 0x0008 */
542#define TH_SFLAG_ABORT 0x0010 /* abort interruptible waits */
543#define TH_SFLAG_ABORTSAFELY 0x0020 /* ... but only those at safe point */
544#define TH_SFLAG_ABORTED_MASK (TH_SFLAG_ABORT | TH_SFLAG_ABORTSAFELY)
545#define TH_SFLAG_DEPRESS 0x0040 /* normal depress yield */
546#define TH_SFLAG_POLLDEPRESS 0x0080 /* polled depress yield */
547#define TH_SFLAG_DEPRESSED_MASK (TH_SFLAG_DEPRESS | TH_SFLAG_POLLDEPRESS)
548/* unused TH_SFLAG_PRI_UPDATE 0x0100 */
549#define TH_SFLAG_EAGERPREEMPT 0x0200 /* Any preemption of this thread should be treated as if AST_URGENT applied */
550#define TH_SFLAG_RW_PROMOTED 0x0400 /* promote reason: blocking with RW lock held */
551#define TH_SFLAG_BASE_PRI_FROZEN 0x0800 /* (effective) base_pri is frozen */
552#define TH_SFLAG_WAITQ_PROMOTED 0x1000 /* promote reason: waitq wakeup (generally for IPC receive) */
553
554#if __AMP__
555/* unused TH_SFLAG_ECORE_ONLY 0x2000 */
556/* unused TH_SFLAG_PCORE_ONLY 0x4000 */
557#endif
558
559#define TH_SFLAG_EXEC_PROMOTED 0x8000 /* promote reason: thread is in an exec */
560
561#define TH_SFLAG_THREAD_GROUP_AUTO_JOIN 0x10000 /* thread has been auto-joined to thread group */
562#if __AMP__
563/* unused TH_SFLAG_BOUND_SOFT 0x20000 */
564#endif /* __AMP__ */
565
566#if CONFIG_PREADOPT_TG
567#define TH_SFLAG_REEVALUTE_TG_HIERARCHY_LATER 0x40000 /* thread needs to reevaluate its TG hierarchy */
568#endif
569
570#define TH_SFLAG_FLOOR_PROMOTED 0x80000 /* promote reason: boost requested */
571
572/* 'promote reasons' that request a priority floor only, not a custom priority */
573#define TH_SFLAG_PROMOTE_REASON_MASK (TH_SFLAG_RW_PROMOTED | TH_SFLAG_WAITQ_PROMOTED | TH_SFLAG_EXEC_PROMOTED | TH_SFLAG_FLOOR_PROMOTED)
574
575#define TH_SFLAG_RT_DISALLOWED 0x100000 /* thread wants RT but may not have joined a work interval that allows it */
576#define TH_SFLAG_DEMOTED_MASK (TH_SFLAG_THROTTLED | TH_SFLAG_FAILSAFE | TH_SFLAG_RT_DISALLOWED) /* saved_mode contains previous sched_mode */
577#define TH_SFLAG_RT_CPULIMIT 0x200000 /* thread should have a CPU limit applied. */
578
579#define TH_SFLAG_FAILSAFE_REPORTED 0x400000 /* whether the kernel has already logged that thread triggered the failsafe (to prevent log spam) */
580
581 int16_t sched_pri; /* scheduled (current) priority */
582 int16_t base_pri; /* effective base priority (equal to req_base_pri unless TH_SFLAG_BASE_PRI_FROZEN) */
583 int16_t req_base_pri; /* requested base priority */
584 int16_t max_priority; /* copy of max base priority */
585 int16_t task_priority; /* copy of task base priority */
586 uint16_t priority_floor_count; /* number of push to boost the floor priority */
587 int16_t suspend_count; /* Kernel holds on this thread */
588
589 int iotier_override; /* atomic operations to set, cleared on ret to user */
590 os_ref_atomic_t ref_count; /* number of references to me */
591
592 uint32_t rwlock_count; /* Number of lck_rw_t locks held by thread */
593 struct smrq_slist_head smr_stack;
594#ifdef DEBUG_RW
595 rw_lock_debug_t rw_lock_held; /* rw_locks currently held by the thread */
596#endif /* DEBUG_RW */
597
598 integer_t importance; /* task-relative importance */
599
600 /* Priority depression expiration */
601 integer_t depress_timer_active;
602 timer_call_t depress_timer;
603
604 /* real-time parameters */
605 struct { /* see mach/thread_policy.h */
606 uint32_t period;
607 uint32_t computation;
608 uint32_t constraint;
609 bool preemptible;
610 uint8_t priority_offset; /* base_pri = BASEPRI_RTQUEUES + priority_offset */
611 uint64_t deadline;
612 } realtime;
613
614 uint64_t last_run_time; /* time when thread was switched away from */
615 uint64_t last_made_runnable_time; /* time when thread was unblocked or preempted */
616 uint64_t last_basepri_change_time; /* time when thread was last changed in basepri while runnable */
617 uint64_t same_pri_latency;
618 /*
619 * workq_quantum_deadline is the workq thread's next runtime deadline. This
620 * value is set to 0 if the thread has no such deadline applicable to it.
621 *
622 * The synchronization for this field is due to how this field is modified
623 * 1) This field is always modified on the thread by itself or on the thread
624 * when it is not running/runnable
625 * 2) Change of this field is immediately followed by a
626 * corresponding change to the AST_KEVENT to either set or clear the
627 * AST_KEVENT_WORKQ_QUANTUM_EXPIRED bit
628 *
629 * workq_quantum_deadline can be modified by the thread on itself during
630 * interrupt context. However, due to (2) and due to the fact that the
631 * change to the AST_KEVENT is volatile, this forces the compiler to
632 * guarantee the order between the write to workq_quantum_deadline and the
633 * kevent field and therefore guarantees the correct synchronization.
634 */
635 uint64_t workq_quantum_deadline;
636
637#if WORKQ_QUANTUM_HISTORY_DEBUG
638
639#define WORKQ_QUANTUM_HISTORY_COUNT 16
640 struct workq_quantum_history {
641 uint64_t time;
642 uint64_t deadline;
643 bool arm;
644 } workq_quantum_history[WORKQ_QUANTUM_HISTORY_COUNT];
645 uint64_t workq_quantum_history_index;
646
647#define WORKQ_QUANTUM_HISTORY_WRITE_ENTRY(thread, ...) ({\
648 thread_t __th = (thread); \
649 uint64_t __index = os_atomic_inc_orig(&thread->workq_quantum_history_index, relaxed); \
650 struct workq_quantum_history _wq_quantum_history = { mach_approximate_time(), __VA_ARGS__}; \
651 __th->workq_quantum_history[__index % WORKQ_QUANTUM_HISTORY_COUNT] = \
652 (struct workq_quantum_history) _wq_quantum_history; \
653 })
654#else /* WORKQ_QUANTUM_HISTORY_DEBUG */
655#define WORKQ_QUANTUM_HISTORY_WRITE_ENTRY(thread, ...)
656#endif /* WORKQ_QUANTUM_HISTORY_DEBUG */
657
658#define THREAD_NOT_RUNNABLE (~0ULL)
659
660#if CONFIG_THREAD_GROUPS
661 struct thread_group *thread_group;
662#endif
663
664 /* Data used during setrun/dispatch */
665 processor_t bound_processor; /* bound to a processor? */
666 processor_t last_processor; /* processor last dispatched on */
667 processor_t chosen_processor; /* Where we want to run this thread */
668
669 /* Fail-safe computation since last unblock or qualifying yield */
670 uint64_t computation_metered;
671 uint64_t computation_epoch;
672 uint64_t computation_interrupt_epoch;
673 uint64_t safe_release; /* when to release fail-safe */
674
675 /* Call out from scheduler */
676 void (*sched_call)(int type, thread_t thread);
677
678 /* Statistics and timesharing calculations */
679#if defined(CONFIG_SCHED_TIMESHARE_CORE)
680 natural_t sched_stamp; /* last scheduler tick */
681 natural_t sched_usage; /* timesharing cpu usage [sched] */
682 natural_t pri_shift; /* usage -> priority from pset */
683 natural_t cpu_usage; /* instrumented cpu usage [%cpu] */
684 natural_t cpu_delta; /* accumulated cpu_usage delta */
685#endif /* CONFIG_SCHED_TIMESHARE_CORE */
686
687 uint32_t c_switch; /* total context switches */
688 uint32_t p_switch; /* total processor switches */
689 uint32_t ps_switch; /* total pset switches */
690
691 /* Timing data structures */
692 uint64_t sched_time_save; /* saved time for scheduler tick */
693 uint64_t vtimer_user_save; /* saved values for vtimers */
694 uint64_t vtimer_prof_save;
695 uint64_t vtimer_rlim_save;
696 uint64_t vtimer_qos_save;
697
698 timer_data_t runnable_timer; /* time the thread is runnable (including running) */
699
700 struct recount_thread th_recount; /* resource accounting */
701
702#if CONFIG_SCHED_SFI
703 /* Timing for wait state */
704 uint64_t wait_sfi_begin_time; /* start time for thread waiting in SFI */
705#endif
706
707 /*
708 * Processor/cache affinity
709 * - affinity_threads links task threads with the same affinity set
710 */
711 queue_chain_t affinity_threads;
712 affinity_set_t affinity_set;
713
714#if CONFIG_TASKWATCH
715 task_watch_t *taskwatch; /* task watch */
716#endif /* CONFIG_TASKWATCH */
717
718 /* Various bits of state to stash across a continuation, exclusive to the current thread block point */
719 union {
720 struct {
721 /* set before ipc_mqueue_receive() as implicit arguments */
722 mach_msg_recv_bufs_t recv_bufs; /* receive context */
723 mach_msg_option64_t option; /* 64 bits options for receive */
724 ipc_object_t object; /* object received on */
725
726 /* set by ipc_mqueue_receive() as implicit results */
727 mach_msg_return_t state; /* receive state */
728 mach_port_seqno_t seqno; /* seqno of recvd message */
729 mach_msg_size_t msize; /* actual size for the msg */
730 mach_msg_size_t asize; /* actual size for aux data */
731 mach_port_name_t receiver_name; /* the receive port name */
732 struct ipc_kmsg *XNU_PTRAUTH_SIGNED_PTR("thread.ith_kmsg") kmsg; /* received message */
733 } receive;
734 struct {
735 struct semaphore *waitsemaphore; /* semaphore ref */
736 struct semaphore *signalsemaphore; /* semaphore ref */
737 int options; /* semaphore options */
738 kern_return_t result; /* primary result */
739 mach_msg_continue_t continuation;
740 } sema;
741 struct {
742#define THREAD_SAVE_IOKIT_TLS_COUNT 8
743 void *tls[THREAD_SAVE_IOKIT_TLS_COUNT];
744 } iokit;
745 } saved;
746
747 /* Only user threads can cause Mach exceptions, only kernel threads can be thread call threads */
748 union {
749 /* Thread call thread's state structure, stored on its stack */
750 struct thread_call_thread_state *thc_state;
751
752 /* Structure to save information about Mach exception */
753 struct {
754 int os_reason;
755 exception_type_t exception_type;
756 mach_exception_code_t code;
757 mach_exception_subcode_t subcode;
758 } mach_exc_info;
759 };
760
761 /* User level suspensions */
762 int32_t user_stop_count;
763
764 /* IPC data structures */
765#if IMPORTANCE_INHERITANCE
766 natural_t ith_assertions; /* assertions pending drop */
767#endif
768 circle_queue_head_t ith_messages; /* messages to reap */
769 mach_port_t ith_kernel_reply_port; /* reply port for kernel RPCs */
770
771 /* VM Fault Tolerance */
772 bool th_vm_faults_disabled;
773
774 /* Ast/Halt data structures */
775 bool recover; /* True if page faulted in recoverable IO */
776
777#if DEBUG || DEVELOPMENT
778 struct thread_test_context *th_test_ctx; /* thread-specific data for kernel tests */
779#endif
780
781 queue_chain_t threads; /* global list of all threads */
782
783 /* Activation */
784 queue_chain_t task_threads;
785
786 /* Task membership */
787#if __x86_64__ || __arm__
788 struct task *t_task;
789#endif
790 struct thread_ro *t_tro;
791 vm_map_t map;
792 thread_t handoff_thread;
793
794 /* Timed wait expiration */
795 timer_call_t wait_timer;
796 uint16_t wait_timer_active; /* is the call running */
797 bool wait_timer_armed; /* should the wait be cleared */
798
799 /* Miscellaneous bits guarded by mutex */
800 uint32_t
801 active:1, /* Thread is active and has not been terminated */
802 ipc_active:1, /* IPC with the thread ports is allowed */
803 started:1, /* Thread has been started after creation */
804 static_param:1, /* Disallow policy parameter changes */
805 inspection:1, /* TRUE when task is being inspected by crash reporter */
806 policy_reset:1, /* Disallow policy parameter changes on terminating threads */
807 suspend_parked:1, /* thread parked in thread_suspended */
808 corpse_dup:1, /* TRUE when thread is an inactive duplicate in a corpse */
809 :0;
810
811 /* Pending thread ast(s) */
812 os_atomic(ast_t) ast;
813
814 decl_lck_mtx_data(, mutex);
815
816 struct ipc_port *ith_special_reply_port; /* ref to special reply port */
817
818#if CONFIG_DTRACE
819 uint16_t t_dtrace_flags; /* DTrace thread states */
820#define TH_DTRACE_EXECSUCCESS 0x01
821 uint16_t t_dtrace_inprobe; /* Executing under dtrace_probe */
822 uint32_t t_dtrace_predcache; /* DTrace per thread predicate value hint */
823 int64_t t_dtrace_tracing; /* Thread time under dtrace_probe() */
824 int64_t t_dtrace_vtime;
825#endif
826
827 clock_sec_t t_page_creation_time;
828 uint32_t t_page_creation_count;
829 uint32_t t_page_creation_throttled;
830#if (DEVELOPMENT || DEBUG)
831 uint64_t t_page_creation_throttled_hard;
832 uint64_t t_page_creation_throttled_soft;
833#endif /* DEVELOPMENT || DEBUG */
834 int t_pagein_error; /* for vm_fault(), holds error from vnop_pagein() */
835
836 mach_port_name_t ith_voucher_name;
837 ipc_voucher_t ith_voucher;
838
839#ifdef KPERF
840/* The high 8 bits are the number of frames to sample of a user callstack. */
841#define T_KPERF_CALLSTACK_DEPTH_OFFSET (24)
842#define T_KPERF_SET_CALLSTACK_DEPTH(DEPTH) (((uint32_t)(DEPTH)) << T_KPERF_CALLSTACK_DEPTH_OFFSET)
843#define T_KPERF_GET_CALLSTACK_DEPTH(FLAGS) ((FLAGS) >> T_KPERF_CALLSTACK_DEPTH_OFFSET)
844#define T_KPERF_ACTIONID_OFFSET (18)
845#define T_KPERF_SET_ACTIONID(AID) (((uint32_t)(AID)) << T_KPERF_ACTIONID_OFFSET)
846#define T_KPERF_GET_ACTIONID(FLAGS) ((FLAGS) >> T_KPERF_ACTIONID_OFFSET)
847#endif
848
849#define T_KPERF_AST_CALLSTACK 0x1 /* dump a callstack on thread's next AST */
850#define T_KPERF_AST_DISPATCH 0x2 /* dump a name on thread's next AST */
851#define T_KPC_ALLOC 0x4 /* thread needs a kpc_buf allocated */
852
853#define T_KPERF_AST_ALL \
854 (T_KPERF_AST_CALLSTACK | T_KPERF_AST_DISPATCH | T_KPC_ALLOC)
855/* only go up to T_KPERF_ACTIONID_OFFSET - 1 */
856
857#ifdef KPERF
858 uint32_t kperf_ast;
859 uint32_t kperf_pet_gen; /* last generation of PET that sampled this thread*/
860 uint32_t kperf_c_switch; /* last dispatch detection */
861 uint32_t kperf_pet_cnt; /* how many times a thread has been sampled by PET */
862#if CONFIG_EXCLAVES
863 uint32_t kperf_exclaves_ast;
864#endif
865#endif
866
867#ifdef CONFIG_CPU_COUNTERS
868 /* accumulated performance counters for this thread */
869 uint64_t *kpc_buf;
870#endif /* CONFIG_CPU_COUNTERS */
871
872#if HYPERVISOR
873 /* hypervisor virtual CPU object associated with this thread */
874 void *hv_thread_target;
875#endif /* HYPERVISOR */
876
877 /* Statistics accumulated per-thread and aggregated per-task */
878 uint32_t syscalls_unix;
879 uint32_t syscalls_mach;
880 ledger_t t_ledger;
881 ledger_t t_threadledger; /* per thread ledger */
882 ledger_t t_bankledger; /* ledger to charge someone */
883 uint64_t t_deduct_bank_ledger_time; /* cpu time to be deducted from bank ledger */
884 uint64_t t_deduct_bank_ledger_energy; /* energy to be deducted from bank ledger */
885
886 uint64_t thread_id; /* system wide unique thread-id */
887 uint32_t ctid; /* system wide compact thread-id */
888 uint32_t ctsid; /* this thread ts ID */
889
890 /* policy is protected by the thread mutex */
891 struct thread_requested_policy requested_policy;
892 struct thread_effective_policy effective_policy;
893
894 /* usynch override is protected by the task lock, eventually will be thread mutex */
895 struct thread_qos_override {
896 struct thread_qos_override *override_next;
897 uint32_t override_contended_resource_count;
898 int16_t override_qos;
899 int16_t override_resource_type;
900 user_addr_t override_resource;
901 } *overrides;
902
903 uint32_t kevent_overrides;
904 uint8_t user_promotion_basepri;
905 uint8_t kern_promotion_schedpri;
906 _Atomic uint16_t kevent_ast_bits;
907
908 io_stat_info_t thread_io_stats; /* per-thread I/O statistics */
909
910 uint32_t thread_callout_interrupt_wakeups;
911 uint32_t thread_callout_platform_idle_wakeups;
912 uint32_t thread_timer_wakeups_bin_1;
913 uint32_t thread_timer_wakeups_bin_2;
914 thread_tag_t thread_tag;
915
916 /*
917 * callout_* fields are only set for thread call threads whereas mach_exc_* are set
918 * by user threads on themselves while taking a Mach exception. So it's okay for them to
919 * share this bitfield.
920 */
921 uint16_t
922 callout_woken_from_icontext:1,
923 callout_woken_from_platform_idle:1,
924 callout_woke_thread:1,
925 mach_exc_fatal:1,
926 mach_exc_ktriage:1,
927 thread_bitfield_unused:11;
928
929#define THREAD_BOUND_CLUSTER_NONE (UINT32_MAX)
930 /*
931 * Cluster to which the thread is soft-bound for scheduling. The thread will always run
932 * on its bound cluster unless that cluster has been derecommended for scheduling.
933 */
934 uint32_t th_bound_cluster_id;
935
936#if CONFIG_THREAD_GROUPS
937#if CONFIG_PREADOPT_TG
938 /* The preadopt thread group is set on the thread
939 *
940 * a) By another thread when it is a creator and it is scheduled with the
941 * thread group on the TR
942 * b) On itself when it binds a thread request and becomes a
943 * servicer or when it rebinds to the thread request
944 * c) On itself when it processes knotes and finds the first
945 * EVFILT_MACHPORT event to deliver to userspace
946 *
947 * Note that this is a full reference owned by the thread_t and not a
948 * borrowed reference.
949 *
950 * This reference is cleared from the thread_t by the thread itself at the
951 * following times:
952 * a) When it explicitly adopts a work interval or a bank voucher
953 * b) If it still exists on the thread, after it has unbound and is about
954 * to park
955 * c) During thread termination if one still exists
956 * d) When a different preadoption thread group is set on the thread
957 *
958 * It is modified under the thread lock.
959 */
960 struct thread_group *preadopt_thread_group;
961
962 /* This field here is present in order to make sure that the t->thread_group
963 * is always pointing to a valid thread group and isn't a dangling pointer.
964 *
965 * Consider the following scenario:
966 * a) t->thread_group points to the preadoption thread group
967 * b) The preadoption thread group is modified on the thread but we are
968 * unable to resolve the hierarchy immediately due to the current state of
969 * the thread
970 *
971 * In order to make sure that t->thread_group points to a valid thread
972 * group until we can resolve the hierarchy again, we save the existing
973 * thread_group it points to in old_preadopt_thread_group. The next time a
974 * hierarchy resolution is done, we know that t->thread_group will not point
975 * to this field anymore so we can clear it.
976 *
977 * This field is always going to take the reference that was previously in
978 * preadopt_thread_group so it will have a full +1
979 */
980 struct thread_group *old_preadopt_thread_group;
981#endif /* CONFIG_PREADOPT_TG */
982
983 /* This is a borrowed reference to the TG from the ith_voucher and is saved
984 * here since we may not always be in the right context to able to do the
985 * lookups.
986 *
987 * It is set always set on self under the thread lock */
988 struct thread_group *bank_thread_group;
989
990 /* Whether this is the autojoin thread group or the work interval thread
991 * group depends on whether the thread's sched_flags has the
992 * TH_SFLAG_THREAD_GROUP_AUTO_JOIN bit set */
993 union {
994 /* This is a borrowed reference to the auto join thread group from the
995 * work_interval. It is set with the thread lock held */
996 struct thread_group *auto_join_thread_group;
997 /* This is a borrowed reference to the explicit work_interval thread group
998 * and is always set on self */
999 struct thread_group *work_interval_thread_group;
1000 };
1001#endif /* CONFIG_THREAD_GROUPS */
1002
1003 /* work interval (if any) associated with the thread. Only modified by
1004 * current thread on itself or when another thread when the thread is held
1005 * off of runq */
1006 struct work_interval *th_work_interval;
1007 thread_work_interval_flags_t th_work_interval_flags;
1008
1009#if SCHED_TRACE_THREAD_WAKEUPS
1010 uintptr_t thread_wakeup_bt[64];
1011#endif
1012 turnstile_update_flags_t inheritor_flags; /* inheritor flags for inheritor field */
1013 block_hint_t pending_block_hint;
1014 block_hint_t block_hint; /* What type of primitive last caused us to block. */
1015 uint32_t decompressions; /* Per-thread decompressions counter to be added to per-task decompressions counter */
1016 int thread_region_page_shift; /* Page shift that this thread would like to use when */
1017 /* introspecting a task. This is currently being used */
1018 /* by footprint which uses a thread for each task being inspected. */
1019#if CONFIG_SCHED_RT_ALLOW
1020 /* Used when a thread is requested to set/clear its own CPU limit */
1021 uint32_t
1022 t_ledger_req_action:2,
1023 t_ledger_req_percentage:7,
1024 t_ledger_req_interval_ms:16,
1025 :0;
1026#endif /* CONFIG_SCHED_RT_ALLOW */
1027
1028#if CONFIG_IOSCHED
1029 void *decmp_upl;
1030#endif /* CONFIG_IOSCHED */
1031 struct knote *ith_knote; /* knote fired for rcv */
1032
1033#if CONFIG_SPTM
1034 /* TXM thread stack associated with this thread */
1035 uintptr_t txm_thread_stack;
1036#endif
1037
1038#if CONFIG_EXCLAVES
1039 /* Per-thread IPC context for exclaves communication. Only modified by the
1040 * current thread on itself. */
1041 exclaves_ctx_t th_exclaves_ipc_ctx;
1042 /* Thread exclaves interrupt-safe state. Only mutated by the current thread
1043 * on itself with interrupts disabled, and only ever read by the current
1044 * thread (with no locking), including from interrupt context, or during
1045 * debug/stackshot. */
1046 thread_exclaves_intstate_flags_t th_exclaves_intstate;
1047 /* Thread exclaves state. Only mutated by the current thread on itself, and
1048 * only ever read by the current thread (with no locking). Unsafe to read
1049 * from interrupt context. */
1050 thread_exclaves_state_flags_t th_exclaves_state;
1051 /* Thread stackshot state. Prevents returning to Exclave world until after
1052 * an external agent has triggered inspection (likely via Exclave stackshot),
1053 * and woken this thread. */
1054 thread_exclaves_inspection_flags_t _Atomic th_exclaves_inspection_state;
1055 /* Task for which conclave teardown is being called by this thread. Used
1056 * for context by conclave crash info upcall to find the task for appending
1057 * the conclave crash info. */
1058 task_t conclave_stop_task;
1059 /* Queue of threads being inspected by Stackshot.
1060 * Modified under exclaves_collect_mtx. */
1061 queue_chain_t th_exclaves_inspection_queue_stackshot;
1062 /* Queue of threads being inspected by kperf.
1063 * Modified under exclaves_collect_mtx. */
1064 queue_chain_t th_exclaves_inspection_queue_kperf;
1065#endif /* CONFIG_EXCLAVES */
1066};
1067
1068#define ith_receive saved.receive
1069/* arguments */
1070#define ith_recv_bufs saved.receive.recv_bufs
1071#define ith_object saved.receive.object
1072#define ith_option saved.receive.option
1073/* results */
1074#define ith_state saved.receive.state
1075#define ith_seqno saved.receive.seqno
1076#define ith_msize saved.receive.msize
1077#define ith_asize saved.receive.asize
1078#define ith_receiver_name saved.receive.receiver_name
1079#define ith_kmsg saved.receive.kmsg
1080
1081#define sth_waitsemaphore saved.sema.waitsemaphore
1082#define sth_signalsemaphore saved.sema.signalsemaphore
1083#define sth_options saved.sema.options
1084#define sth_result saved.sema.result
1085#define sth_continuation saved.sema.continuation
1086
1087#define ITH_KNOTE_NULL ((void *)NULL)
1088#define ITH_KNOTE_PSEUDO ((void *)0xdeadbeef)
1089/*
1090 * The ith_knote is used during message delivery, and can safely be interpreted
1091 * only when used for one of these codepaths, which the test for the msgt_name
1092 * being RECEIVE or SEND_ONCE is about.
1093 */
1094#define ITH_KNOTE_VALID(kn, msgt_name) \
1095 (((kn) != ITH_KNOTE_NULL && (kn) != ITH_KNOTE_PSEUDO) && \
1096 ((msgt_name) == MACH_MSG_TYPE_PORT_RECEIVE || \
1097 (msgt_name) == MACH_MSG_TYPE_PORT_SEND_ONCE))
1098
1099#if MACH_ASSERT
1100#define assert_thread_magic(thread) assertf((thread)->thread_magic == THREAD_MAGIC, \
1101 "bad thread magic 0x%llx for thread %p, expected 0x%llx", \
1102 (thread)->thread_magic, (thread), THREAD_MAGIC)
1103#else
1104#define assert_thread_magic(thread) do { (void)(thread); } while (0)
1105#endif
1106
1107extern thread_t thread_bootstrap(void);
1108
1109extern void thread_machine_init_template(void);
1110
1111extern void thread_init(void);
1112
1113extern void thread_daemon_init(void);
1114
1115extern void thread_reference(
1116 thread_t thread);
1117
1118extern void thread_deallocate(
1119 thread_t thread);
1120
1121extern void thread_inspect_deallocate(
1122 thread_inspect_t thread);
1123
1124extern void thread_read_deallocate(
1125 thread_read_t thread);
1126
1127extern kern_return_t thread_terminate(
1128 thread_t thread);
1129
1130extern void thread_terminate_self(void);
1131
1132extern kern_return_t thread_terminate_internal(
1133 thread_t thread);
1134
1135extern void thread_start(
1136 thread_t thread) __attribute__ ((noinline));
1137
1138extern void thread_start_in_assert_wait(
1139 thread_t thread,
1140 struct waitq *waitq,
1141 event64_t event,
1142 wait_interrupt_t interruptible) __attribute__ ((noinline));
1143
1144extern void thread_terminate_enqueue(
1145 thread_t thread);
1146
1147extern void thread_exception_enqueue(
1148 task_t task,
1149 thread_t thread,
1150 exception_type_t etype);
1151
1152extern void thread_backtrace_enqueue(
1153 kcdata_object_t obj,
1154 exception_port_t ports[static BT_EXC_PORTS_COUNT],
1155 exception_type_t etype);
1156
1157extern void thread_copy_resource_info(
1158 thread_t dst_thread,
1159 thread_t src_thread);
1160
1161extern void thread_terminate_crashed_threads(void);
1162
1163extern void thread_stack_enqueue(
1164 thread_t thread);
1165
1166extern void thread_hold(
1167 thread_t thread);
1168
1169extern void thread_release(
1170 thread_t thread);
1171
1172extern void thread_corpse_continue(void) __dead2;
1173
1174extern boolean_t thread_is_active(thread_t thread);
1175
1176extern lck_grp_t thread_lck_grp;
1177
1178/* Locking for scheduler state, always acquired with interrupts disabled (splsched()) */
1179#define thread_lock_init(th) simple_lock_init(&(th)->sched_lock, 0)
1180#define thread_lock(th) simple_lock(&(th)->sched_lock, &thread_lck_grp)
1181#define thread_unlock(th) simple_unlock(&(th)->sched_lock)
1182#define thread_lock_assert(th, x) simple_lock_assert(&(th)->sched_lock, (x))
1183
1184#define wake_lock_init(th) simple_lock_init(&(th)->wake_lock, 0)
1185#define wake_lock(th) simple_lock(&(th)->wake_lock, &thread_lck_grp)
1186#define wake_unlock(th) simple_unlock(&(th)->wake_lock)
1187
1188#define thread_should_halt_fast(thread) (!(thread)->active)
1189
1190extern void stack_alloc(
1191 thread_t thread);
1192
1193extern void stack_handoff(
1194 thread_t from,
1195 thread_t to);
1196
1197extern void stack_free(
1198 thread_t thread);
1199
1200extern void stack_free_reserved(
1201 thread_t thread);
1202
1203extern boolean_t stack_alloc_try(
1204 thread_t thread);
1205
1206extern void stack_collect(void);
1207
1208extern kern_return_t thread_info_internal(
1209 thread_t thread,
1210 thread_flavor_t flavor,
1211 thread_info_t thread_info_out,
1212 mach_msg_type_number_t *thread_info_count);
1213
1214extern kern_return_t kernel_thread_create(
1215 thread_continue_t continuation,
1216 void *parameter,
1217 integer_t priority,
1218 thread_t *new_thread);
1219
1220extern kern_return_t kernel_thread_start_priority(
1221 thread_continue_t continuation,
1222 void *parameter,
1223 integer_t priority,
1224 thread_t *new_thread);
1225
1226extern void machine_stack_attach(
1227 thread_t thread,
1228 vm_offset_t stack);
1229
1230extern vm_offset_t machine_stack_detach(
1231 thread_t thread);
1232
1233extern void machine_stack_handoff(
1234 thread_t old,
1235 thread_t new);
1236
1237extern thread_t machine_switch_context(
1238 thread_t old_thread,
1239 thread_continue_t continuation,
1240 thread_t new_thread);
1241
1242extern void machine_load_context(
1243 thread_t thread) __attribute__((noreturn));
1244
1245extern void machine_thread_state_initialize(
1246 thread_t thread);
1247
1248extern kern_return_t machine_thread_set_state(
1249 thread_t thread,
1250 thread_flavor_t flavor,
1251 thread_state_t state,
1252 mach_msg_type_number_t count);
1253
1254extern mach_vm_address_t machine_thread_pc(
1255 thread_t thread);
1256
1257extern void machine_thread_reset_pc(
1258 thread_t thread,
1259 mach_vm_address_t pc);
1260
1261extern boolean_t machine_thread_on_core(
1262 thread_t thread);
1263
1264extern boolean_t machine_thread_on_core_allow_invalid(
1265 thread_t thread);
1266
1267extern kern_return_t machine_thread_get_state(
1268 thread_t thread,
1269 thread_flavor_t flavor,
1270 thread_state_t state,
1271 mach_msg_type_number_t *count);
1272
1273extern kern_return_t machine_thread_state_convert_from_user(
1274 thread_t thread,
1275 thread_flavor_t flavor,
1276 thread_state_t tstate,
1277 mach_msg_type_number_t count,
1278 thread_state_t old_tstate,
1279 mach_msg_type_number_t old_count,
1280 thread_set_status_flags_t tssf_flags);
1281
1282extern kern_return_t machine_thread_state_convert_to_user(
1283 thread_t thread,
1284 thread_flavor_t flavor,
1285 thread_state_t tstate,
1286 mach_msg_type_number_t *count,
1287 thread_set_status_flags_t tssf_flags);
1288
1289extern kern_return_t machine_thread_dup(
1290 thread_t self,
1291 thread_t target,
1292 boolean_t is_corpse);
1293
1294extern void machine_thread_init(void);
1295
1296extern void machine_thread_template_init(thread_t thr_template);
1297
1298#if __has_feature(ptrauth_calls)
1299extern bool machine_thread_state_is_debug_flavor(int flavor);
1300#endif /* __has_feature(ptrauth_calls) */
1301
1302
1303extern void machine_thread_create(
1304 thread_t thread,
1305 task_t task,
1306 bool first_thread);
1307
1308extern kern_return_t machine_thread_process_signature(
1309 thread_t thread,
1310 task_t task);
1311
1312extern void machine_thread_switch_addrmode(
1313 thread_t thread);
1314
1315extern void machine_thread_destroy(
1316 thread_t thread);
1317
1318extern void machine_set_current_thread(
1319 thread_t thread);
1320
1321extern kern_return_t machine_thread_get_kern_state(
1322 thread_t thread,
1323 thread_flavor_t flavor,
1324 thread_state_t tstate,
1325 mach_msg_type_number_t *count);
1326
1327extern kern_return_t machine_thread_inherit_taskwide(
1328 thread_t thread,
1329 task_t parent_task);
1330
1331extern kern_return_t machine_thread_set_tsd_base(
1332 thread_t thread,
1333 mach_vm_offset_t tsd_base);
1334
1335#define thread_mtx_try(thread) lck_mtx_try_lock(&(thread)->mutex)
1336#define thread_mtx_held(thread) lck_mtx_assert(&(thread)->mutex, LCK_MTX_ASSERT_OWNED)
1337
1338extern void thread_apc_ast(thread_t thread);
1339
1340extern void thread_update_qos_cpu_time(thread_t thread);
1341
1342void act_machine_sv_free(thread_t, int);
1343
1344vm_offset_t min_valid_stack_address(void);
1345vm_offset_t max_valid_stack_address(void);
1346
1347#if CONFIG_SCHED_SMT
1348extern bool thread_no_smt(thread_t thread);
1349extern bool processor_active_thread_no_smt(processor_t processor);
1350#endif /* CONFIG_SCHED_SMT */
1351
1352extern void thread_set_options(uint32_t thopt);
1353
1354#if CONFIG_THREAD_GROUPS
1355struct thread_group *thread_get_current_voucher_thread_group(thread_t thread);
1356#endif /* CONFIG_THREAD_GROUPS */
1357
1358#if CONFIG_COALITIONS
1359uint64_t thread_get_current_voucher_resource_coalition_id(thread_t thread);
1360#endif /* CONFIG_COALITIONS */
1361
1362#endif /* MACH_KERNEL_PRIVATE */
1363#if BSD_KERNEL_PRIVATE
1364
1365/* Duplicated from osfmk/kern/ipc_tt.h */
1366__options_decl(port_intrans_options_t, uint32_t, {
1367 PORT_INTRANS_OPTIONS_NONE = 0x0000,
1368 PORT_INTRANS_THREAD_IN_CURRENT_TASK = 0x0001,
1369 PORT_INTRANS_THREAD_NOT_CURRENT_THREAD = 0x0002,
1370
1371 PORT_INTRANS_SKIP_TASK_EVAL = 0x0004,
1372 PORT_INTRANS_ALLOW_CORPSE_TASK = 0x0008,
1373});
1374
1375extern thread_t port_name_to_thread(
1376 mach_port_name_t port_name,
1377 port_intrans_options_t options);
1378
1379#endif /* BSD_KERNEL_PRIVATE */
1380#ifdef XNU_KERNEL_PRIVATE
1381
1382extern void thread_require(
1383 thread_t thread);
1384
1385extern void thread_deallocate_safe(
1386 thread_t thread);
1387
1388extern uint64_t thread_rettokern_addr(
1389 thread_t thread);
1390
1391extern uint64_t thread_wqquantum_addr(
1392 thread_t thread);
1393
1394extern integer_t thread_kern_get_pri(thread_t thr) __pure2;
1395
1396extern void thread_kern_set_pri(thread_t thr, integer_t pri);
1397
1398extern integer_t thread_kern_get_kernel_maxpri(void) __pure2;
1399
1400uint16_t thread_set_tag(thread_t thread, uint16_t tag);
1401uint16_t thread_get_tag(thread_t thread);
1402
1403__options_decl(shared_rsrc_policy_agent_t, uint32_t, {
1404 SHARED_RSRC_POLICY_AGENT_DISPATCH = 0,
1405 SHARED_RSRC_POLICY_AGENT_SYSCTL = 1,
1406 SHARED_RSRC_POLICY_AGENT_PERFCTL_CSW = 2,
1407 SHARED_RSRC_POLICY_AGENT_PERFCTL_QUANTUM = 3,
1408});
1409
1410boolean_t thread_shared_rsrc_policy_get(thread_t thread, cluster_shared_rsrc_type_t type);
1411kern_return_t thread_shared_rsrc_policy_set(thread_t thread, uint32_t index, cluster_shared_rsrc_type_t type, shared_rsrc_policy_agent_t agent);
1412kern_return_t thread_shared_rsrc_policy_clear(thread_t thread, cluster_shared_rsrc_type_t type, shared_rsrc_policy_agent_t agent);
1413
1414#ifdef MACH_KERNEL_PRIVATE
1415static inline thread_tag_t
1416thread_set_tag_internal(thread_t thread, thread_tag_t tag)
1417{
1418 return os_atomic_or_orig(&thread->thread_tag, tag, relaxed);
1419}
1420
1421static inline thread_tag_t
1422thread_get_tag_internal(thread_t thread)
1423{
1424 return thread->thread_tag;
1425}
1426#endif /* MACH_KERNEL_PRIVATE */
1427
1428extern uint64_t thread_last_run_time(
1429 thread_t thread);
1430
1431extern kern_return_t thread_state_initialize(
1432 thread_t thread);
1433
1434extern kern_return_t thread_setstatus(
1435 thread_t thread,
1436 int flavor,
1437 thread_state_t tstate,
1438 mach_msg_type_number_t count);
1439
1440extern kern_return_t thread_setstatus_from_user(
1441 thread_t thread,
1442 int flavor,
1443 thread_state_t tstate,
1444 mach_msg_type_number_t count,
1445 thread_state_t old_tstate,
1446 mach_msg_type_number_t old_count,
1447 thread_set_status_flags_t flags);
1448
1449extern kern_return_t thread_getstatus(
1450 thread_t thread,
1451 int flavor,
1452 thread_state_t tstate,
1453 mach_msg_type_number_t *count);
1454
1455extern kern_return_t thread_getstatus_to_user(
1456 thread_t thread,
1457 int flavor,
1458 thread_state_t tstate,
1459 mach_msg_type_number_t *count,
1460 thread_set_status_flags_t flags);
1461
1462extern kern_return_t thread_create_with_continuation(
1463 task_t task,
1464 thread_t *new_thread,
1465 thread_continue_t continuation);
1466
1467extern kern_return_t main_thread_create_waiting(
1468 task_t task,
1469 thread_continue_t continuation,
1470 event_t event,
1471 thread_t *new_thread);
1472
1473extern kern_return_t thread_create_workq_waiting(
1474 task_t task,
1475 thread_continue_t thread_return,
1476 thread_t *new_thread,
1477 bool is_permanently_bound);
1478
1479extern kern_return_t thread_create_aio_workq_waiting(
1480 task_t task,
1481 thread_continue_t thread_return,
1482 thread_t *new_thread);
1483
1484extern void thread_yield_internal(
1485 mach_msg_timeout_t interval);
1486
1487extern void thread_yield_to_preemption(void);
1488
1489extern void thread_depress_timer_setup(
1490 thread_t self);
1491
1492/*
1493 * Thread-private CPU limits: apply a private CPU limit to this thread only. Available actions are:
1494 *
1495 * 1) Block. Prevent CPU consumption of the thread from exceeding the limit.
1496 * 2) Exception. Generate a resource consumption exception when the limit is exceeded.
1497 * 3) Disable. Remove any existing CPU limit.
1498 */
1499#define THREAD_CPULIMIT_BLOCK 0x1
1500#define THREAD_CPULIMIT_EXCEPTION 0x2
1501#define THREAD_CPULIMIT_DISABLE 0x3
1502
1503struct _thread_ledger_indices {
1504 int cpu_time;
1505};
1506
1507extern struct _thread_ledger_indices thread_ledgers;
1508
1509extern int thread_get_cpulimit(int *action, uint8_t *percentage, uint64_t *interval_ns);
1510extern int thread_set_cpulimit(int action, uint8_t percentage, uint64_t interval_ns);
1511
1512extern uint64_t thread_cpulimit_remaining(uint64_t now);
1513extern bool thread_cpulimit_interval_has_expired(uint64_t now);
1514extern void thread_cpulimit_restart(uint64_t now);
1515
1516extern void thread_read_times(
1517 thread_t thread,
1518 time_value_t *user_time,
1519 time_value_t *system_time,
1520 time_value_t *runnable_time);
1521
1522extern void thread_read_times_unsafe(
1523 thread_t thread,
1524 time_value_t *user_time,
1525 time_value_t *system_time,
1526 time_value_t *runnable_time);
1527
1528extern uint64_t thread_get_runtime_self(void);
1529
1530extern void thread_setuserstack(
1531 thread_t thread,
1532 mach_vm_offset_t user_stack);
1533
1534extern user_addr_t thread_adjuserstack(
1535 thread_t thread,
1536 int adjust);
1537
1538
1539extern void thread_setentrypoint(
1540 thread_t thread,
1541 mach_vm_offset_t entry);
1542
1543extern kern_return_t thread_set_tsd_base(
1544 thread_t thread,
1545 mach_vm_offset_t tsd_base);
1546
1547extern kern_return_t thread_setsinglestep(
1548 thread_t thread,
1549 int on);
1550
1551extern kern_return_t thread_userstack(
1552 thread_t,
1553 int,
1554 thread_state_t,
1555 unsigned int,
1556 mach_vm_offset_t *,
1557 int *,
1558 boolean_t);
1559
1560extern kern_return_t thread_entrypoint(
1561 thread_t,
1562 int,
1563 thread_state_t,
1564 unsigned int,
1565 mach_vm_offset_t *);
1566
1567extern kern_return_t thread_userstackdefault(
1568 mach_vm_offset_t *,
1569 boolean_t);
1570
1571extern kern_return_t thread_wire_internal(
1572 host_priv_t host_priv,
1573 thread_t thread,
1574 boolean_t wired,
1575 boolean_t *prev_state);
1576
1577
1578extern kern_return_t thread_dup(thread_t);
1579
1580extern kern_return_t thread_dup2(thread_t, thread_t);
1581
1582#if !defined(_SCHED_CALL_T_DEFINED)
1583#define _SCHED_CALL_T_DEFINED
1584typedef void (*sched_call_t)(
1585 int type,
1586 thread_t thread);
1587#endif
1588
1589#define SCHED_CALL_BLOCK 0x1
1590#define SCHED_CALL_UNBLOCK 0x2
1591
1592extern void thread_sched_call(
1593 thread_t thread,
1594 sched_call_t call);
1595
1596extern boolean_t thread_is_static_param(
1597 thread_t thread);
1598
1599extern task_t get_threadtask(thread_t) __pure2;
1600
1601extern task_t get_threadtask_early(thread_t) __pure2;
1602
1603/*
1604 * Thread is running within a 64-bit address space.
1605 */
1606#define thread_is_64bit_addr(thd) \
1607 task_has_64Bit_addr(get_threadtask(thd))
1608
1609/*
1610 * Thread is using 64-bit machine state.
1611 */
1612#define thread_is_64bit_data(thd) \
1613 task_has_64Bit_data(get_threadtask(thd))
1614
1615struct uthread;
1616
1617#if defined(__x86_64__)
1618extern int thread_task_has_ldt(thread_t);
1619#endif
1620extern void set_thread_pagein_error(thread_t, int);
1621extern event_t workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/pthread/
1622extern event_t aio_workq_thread_init_and_wq_lock(task_t, thread_t); // bsd/aio/
1623
1624struct proc;
1625struct uthread;
1626struct image_params;
1627extern const size_t uthread_size;
1628extern thread_ro_t get_thread_ro_unchecked(thread_t) __pure2;
1629extern thread_ro_t get_thread_ro(thread_t) __pure2;
1630extern thread_ro_t current_thread_ro_unchecked(void) __pure2;
1631extern thread_ro_t current_thread_ro(void) __pure2;
1632extern void clear_thread_ro_proc(thread_t);
1633extern struct uthread *get_bsdthread_info(thread_t) __pure2;
1634extern thread_t get_machthread(struct uthread *) __pure2;
1635extern uint64_t uthread_tid(struct uthread *) __pure2;
1636extern user_addr_t thread_get_sigreturn_token(thread_t thread);
1637extern uint32_t thread_get_sigreturn_diversifier(thread_t thread);
1638extern void uthread_init(task_t, struct uthread *, thread_ro_t, int);
1639extern void uthread_cleanup_name(struct uthread *uthread);
1640extern void uthread_cleanup(struct uthread *, thread_ro_t);
1641extern void uthread_cred_ref(struct ucred *);
1642extern void uthread_cred_free(struct ucred *);
1643extern void uthread_destroy(struct uthread *);
1644extern void uthread_reset_proc_refcount(struct uthread *);
1645
1646extern void uthread_set_exec_data(struct uthread *uth, struct image_params *imgp);
1647extern bool uthread_is64bit(struct uthread *uth) __pure2;
1648#if PROC_REF_DEBUG
1649extern void uthread_init_proc_refcount(struct uthread *);
1650extern void uthread_destroy_proc_refcount(struct uthread *);
1651extern void uthread_assert_zero_proc_refcount(struct uthread *);
1652#else
1653#define uthread_init_proc_refcount(uth) ((void)(uth))
1654#define uthread_destroy_proc_refcount(uth) ((void)(uth))
1655#define uthread_assert_zero_proc_refcount(uth) ((void)(uth))
1656#endif
1657#if CONFIG_DEBUG_SYSCALL_REJECTION
1658extern uint64_t uthread_get_syscall_rejection_flags(void *);
1659extern uint64_t *uthread_get_syscall_rejection_mask(void *);
1660extern uint64_t *uthread_get_syscall_rejection_once_mask(void *);
1661extern bool uthread_syscall_rejection_is_enabled(void *);
1662#endif /* CONFIG_DEBUG_SYSCALL_REJECTION */
1663extern mach_port_name_t uthread_joiner_port(struct uthread *);
1664extern user_addr_t uthread_joiner_address(struct uthread *);
1665extern void uthread_joiner_wake(task_t task, struct uthread *);
1666
1667extern boolean_t thread_should_halt(
1668 thread_t thread);
1669
1670extern boolean_t thread_should_abort(
1671 thread_t);
1672
1673extern bool current_thread_in_kernel_fault(void);
1674
1675extern int is_64signalregset(void);
1676
1677extern void act_set_kperf(thread_t);
1678extern void act_set_astledger(thread_t thread);
1679extern void act_set_astledger_async(thread_t thread);
1680extern void act_set_io_telemetry_ast(thread_t);
1681extern void act_set_macf_telemetry_ast(thread_t);
1682extern void act_set_astproc_resource(thread_t);
1683
1684extern vm_offset_t thread_get_kernel_stack(thread_t);
1685
1686extern kern_return_t thread_process_signature(thread_t thread, task_t task);
1687
1688extern uint32_t dtrace_get_thread_predcache(thread_t);
1689extern int64_t dtrace_get_thread_vtime(thread_t);
1690extern int64_t dtrace_get_thread_tracing(thread_t);
1691extern uint16_t dtrace_get_thread_inprobe(thread_t);
1692extern int dtrace_get_thread_last_cpu_id(thread_t);
1693extern vm_offset_t dtrace_get_kernel_stack(thread_t);
1694#define dtrace_get_kernel_stack thread_get_kernel_stack
1695extern void dtrace_set_thread_predcache(thread_t, uint32_t);
1696extern void dtrace_set_thread_vtime(thread_t, int64_t);
1697extern void dtrace_set_thread_tracing(thread_t, int64_t);
1698extern void dtrace_set_thread_inprobe(thread_t, uint16_t);
1699extern void dtrace_thread_bootstrap(void);
1700extern void dtrace_thread_didexec(thread_t);
1701
1702extern int64_t dtrace_calc_thread_recent_vtime(thread_t);
1703
1704
1705extern kern_return_t thread_set_wq_state32(
1706 thread_t thread,
1707 thread_state_t tstate);
1708
1709extern kern_return_t thread_set_wq_state64(
1710 thread_t thread,
1711 thread_state_t tstate);
1712
1713extern vm_offset_t kernel_stack_mask;
1714extern vm_offset_t kernel_stack_size;
1715extern vm_offset_t kernel_stack_depth_max;
1716
1717extern void mach_exception_ast(thread_t);
1718extern void fd_guard_ast(thread_t,
1719 mach_exception_code_t, mach_exception_subcode_t);
1720extern void vn_guard_ast(thread_t,
1721 mach_exception_code_t, mach_exception_subcode_t);
1722extern void mach_port_guard_ast(thread_t,
1723 mach_exception_code_t, mach_exception_subcode_t);
1724extern void virt_memory_guard_ast(thread_t,
1725 mach_exception_code_t, mach_exception_subcode_t);
1726extern void thread_ast_mach_exception(thread_t,
1727 int, exception_type_t, mach_exception_code_t, mach_exception_subcode_t, bool, bool);
1728extern void thread_guard_violation(thread_t,
1729 mach_exception_code_t, mach_exception_subcode_t, bool);
1730extern void thread_update_io_stats(thread_t, int size, int io_flags);
1731
1732extern kern_return_t thread_set_voucher_name(mach_port_name_t name);
1733extern kern_return_t thread_get_voucher_origin_pid(thread_t thread, int32_t *pid);
1734extern kern_return_t thread_get_voucher_origin_proximate_pid(thread_t thread,
1735 int32_t *origin_pid, int32_t *proximate_pid);
1736extern kern_return_t thread_get_current_voucher_origin_pid(int32_t *pid);
1737
1738extern void thread_enable_send_importance(thread_t thread, boolean_t enable);
1739
1740/*
1741 * Translate signal context data pointer to userspace representation
1742 */
1743
1744extern kern_return_t machine_thread_siguctx_pointer_convert_to_user(
1745 thread_t thread,
1746 user_addr_t *uctxp);
1747
1748extern void machine_tecs(thread_t thr);
1749
1750typedef enum cpuvn {
1751 CPUVN_CI = 1
1752} cpuvn_e;
1753
1754extern int machine_csv(cpuvn_e cve);
1755#if defined(__x86_64__)
1756extern void machine_thread_set_insn_copy_optout(thread_t thr);
1757#endif
1758
1759/*
1760 * Translate array of function pointer syscall arguments from userspace representation
1761 */
1762
1763extern kern_return_t machine_thread_function_pointers_convert_from_user(
1764 thread_t thread,
1765 user_addr_t *fptrs,
1766 uint32_t count);
1767
1768/*
1769 * Get the duration of the given thread's last wait.
1770 */
1771uint64_t thread_get_last_wait_duration(thread_t thread);
1772
1773#if CONFIG_SCHED_SMT
1774extern bool thread_get_no_smt(void);
1775#endif /* CONFIG_SCHED_SMT */
1776#if defined(__x86_64__)
1777extern bool curtask_get_insn_copy_optout(void);
1778extern void curtask_set_insn_copy_optout(void);
1779#endif /* defined(__x86_64__) */
1780
1781/*! @function ctid_get_thread
1782 * @abstract translates a ctid_t to thread_t
1783 * @discussion ctid are system wide compact thread-id
1784 * associated to thread_t at thread creation
1785 * and recycled at thread termination. If a ctid is
1786 * referenced past the corresponding thread termination,
1787 * it is considered stale, and the behavior is not defined.
1788 * Note that this call does not acquire a reference on the thread,
1789 * so as soon as the matching thread terminates, the ctid
1790 * will become stale, and it could be re-used and associated with
1791 * another thread. You must externally guarantee that the thread
1792 * will not exit while you are using its ctid.
1793 * @result thread_t corresponding to ctid
1794 */
1795extern thread_t ctid_get_thread(ctid_t ctid);
1796
1797/*! @function ctid_get_thread
1798 * @abstract translates a ctid_t to thread_t
1799 * @discussion Unsafe variant of ctid_get_thread() to be used
1800 * when the caller can't guarantee the liveness of this ctid_t.
1801 * may return NULL or a freed thread_t.
1802 */
1803extern thread_t ctid_get_thread_unsafe(ctid_t ctid);
1804
1805/*!
1806 * @function thread_get_ctid
1807 * @abstract returns the ctid of thread.
1808 * @param thread to find the corresponding ctid.
1809 * @discussion the ctid provided will become stale after the matching thread
1810 * terminates.
1811 * @result uint32_t ctid.
1812 */
1813extern ctid_t thread_get_ctid(thread_t thread);
1814
1815#endif /* XNU_KERNEL_PRIVATE */
1816#ifdef KERNEL_PRIVATE
1817
1818typedef struct thread_pri_floor {
1819 thread_t thread;
1820} thread_pri_floor_t;
1821
1822#ifdef MACH_KERNEL_PRIVATE
1823extern void thread_floor_boost_ast(thread_t thread);
1824extern void thread_floor_boost_set_promotion_locked(thread_t thread);
1825#endif /* MACH_KERNEL_PRIVATE */
1826
1827/*! @function thread_priority_floor_start
1828 * @abstract boost the current thread priority to floor.
1829 * @discussion Increase the priority of the current thread to at least MINPRI_FLOOR.
1830 * The boost will be mantained until a corresponding thread_priority_floor_end()
1831 * is called. Every call of thread_priority_floor_start() needs to have a corresponding
1832 * call to thread_priority_floor_end() from the same thread.
1833 * No thread can return to userspace before calling thread_priority_floor_end().
1834 *
1835 * NOTE: avoid to use this function. Try to use gate_t or sleep_with_inheritor()
1836 * instead.
1837 * @result a token to be given to the corresponding thread_priority_floor_end()
1838 */
1839extern thread_pri_floor_t thread_priority_floor_start(void);
1840/*! @function thread_priority_floor_end
1841 * @abstract ends the floor boost.
1842 * @param token the token obtained from thread_priority_floor_start()
1843 * @discussion ends the priority floor boost started with thread_priority_floor_start()
1844 */
1845extern void thread_priority_floor_end(thread_pri_floor_t *token);
1846
1847#if defined(__x86_64__)
1848extern void thread_set_no_smt(bool set);
1849#endif /* __x86_64__ */
1850
1851extern void thread_mtx_lock(thread_t thread);
1852
1853extern void thread_mtx_unlock(thread_t thread);
1854
1855extern uint64_t thread_dispatchqaddr(
1856 thread_t thread);
1857
1858bool thread_is_eager_preempt(thread_t thread);
1859void thread_set_eager_preempt(thread_t thread);
1860void thread_clear_eager_preempt(thread_t thread);
1861void thread_set_honor_qlimit(thread_t thread);
1862void thread_clear_honor_qlimit(thread_t thread);
1863extern ipc_port_t convert_thread_to_port(thread_t);
1864extern ipc_port_t convert_thread_to_port_immovable(thread_t);
1865extern ipc_port_t convert_thread_inspect_to_port(thread_inspect_t);
1866extern ipc_port_t convert_thread_read_to_port(thread_read_t);
1867extern void convert_thread_array_to_ports(thread_act_array_t, size_t, mach_thread_flavor_t);
1868extern boolean_t is_external_pageout_thread(void);
1869extern boolean_t is_vm_privileged(void);
1870extern boolean_t set_vm_privilege(boolean_t);
1871extern kern_allocation_name_t thread_set_allocation_name(kern_allocation_name_t new_name);
1872extern void *thread_iokit_tls_get(uint32_t index);
1873extern void thread_iokit_tls_set(uint32_t index, void * data);
1874extern int thread_self_region_page_shift(void);
1875extern void thread_self_region_page_shift_set(int pgshift);
1876extern kern_return_t thread_create_immovable(task_t task, thread_t *new_thread);
1877extern kern_return_t thread_terminate_immovable(thread_t thread);
1878
1879struct thread_attr_for_ipc_propagation;
1880extern kern_return_t thread_get_ipc_propagate_attr(thread_t thread, struct thread_attr_for_ipc_propagation *attr);
1881extern size_t thread_get_current_exec_path(char *path, size_t size);
1882#endif /* KERNEL_PRIVATE */
1883#ifdef XNU_KERNEL_PRIVATE
1884
1885extern void
1886thread_get_thread_name(thread_t th, char* name);
1887
1888/* Read the runq assignment, under the thread lock. */
1889extern processor_t thread_get_runq(thread_t thread);
1890
1891/*
1892 * Read the runq assignment, under both the thread lock and
1893 * the pset lock corresponding to the last non-null assignment.
1894 */
1895extern processor_t thread_get_runq_locked(thread_t thread);
1896
1897/*
1898 * Set the runq assignment to a non-null value, under both the
1899 * thread lock and the pset lock corresponding to the new
1900 * assignment.
1901 */
1902extern void thread_set_runq_locked(thread_t thread, processor_t new_runq);
1903
1904/*
1905 * Set the runq assignment to PROCESSOR_NULL, under the pset
1906 * lock corresponding to the current non-null assignment.
1907 */
1908extern void thread_clear_runq(thread_t thread);
1909
1910/*
1911 * Set the runq assignment to PROCESSOR_NULL, under both the
1912 * thread lock and the pset lock corresponding to the current
1913 * non-null assignment.
1914 */
1915extern void thread_clear_runq_locked(thread_t thread);
1916
1917/*
1918 * Assert the runq assignment to be PROCESSOR_NULL, under
1919 * some guarantee that the runq will not change from null to
1920 * non-null, such as holding the thread lock.
1921 */
1922extern void thread_assert_runq_null(thread_t thread);
1923
1924/*
1925 * Assert the runq assignment to be non-null, under the pset
1926 * lock corresponding to the current non-null assignment.
1927 */
1928extern void thread_assert_runq_nonnull(thread_t thread);
1929
1930extern bool thread_supports_cooperative_workqueue(thread_t thread);
1931extern void thread_arm_workqueue_quantum(thread_t thread);
1932extern void thread_disarm_workqueue_quantum(thread_t thread);
1933
1934extern void thread_evaluate_workqueue_quantum_expiry(thread_t thread);
1935extern bool thread_has_expired_workqueue_quantum(thread_t thread, bool should_trace);
1936
1937#if CONFIG_SPTM
1938
1939extern void
1940thread_associate_txm_thread_stack(uintptr_t thread_stack);
1941
1942extern void
1943thread_disassociate_txm_thread_stack(uintptr_t thread_stack);
1944
1945extern uintptr_t
1946thread_get_txm_thread_stack(void);
1947
1948#endif /* CONFIG_SPTM */
1949
1950/* Kernel side prototypes for MIG routines */
1951extern kern_return_t thread_get_exception_ports(
1952 thread_t thread,
1953 exception_mask_t exception_mask,
1954 exception_mask_array_t masks,
1955 mach_msg_type_number_t *CountCnt,
1956 exception_port_array_t ports,
1957 exception_behavior_array_t behaviors,
1958 thread_state_flavor_array_t flavors);
1959
1960extern kern_return_t thread_get_special_port(
1961 thread_inspect_t thread,
1962 int which,
1963 ipc_port_t *portp);
1964
1965extern uint64_t thread_c_switch(thread_t thread);
1966
1967#endif /* XNU_KERNEL_PRIVATE */
1968
1969/*! @function thread_has_thread_name
1970 * @abstract Checks if a thread has a name.
1971 * @discussion This function takes one input, a thread, and returns
1972 * a boolean value indicating if that thread already has a name associated
1973 * with it.
1974 * @param th The thread to inspect.
1975 * @result TRUE if the thread has a name, FALSE otherwise.
1976 */
1977extern boolean_t thread_has_thread_name(thread_t th);
1978
1979/*! @function thread_set_thread_name
1980 * @abstract Set a thread's name.
1981 * @discussion This function takes two input parameters: a thread to name,
1982 * and the name to apply to the thread. The name will be copied over to
1983 * the thread in order to better identify the thread. If the name is
1984 * longer than MAXTHREADNAMESIZE - 1, it will be truncated.
1985 * @param th The thread to be named.
1986 * @param name The name to apply to the thread.
1987 */
1988extern void thread_set_thread_name(thread_t th, const char* name);
1989
1990#if !MACH_KERNEL_PRIVATE || !defined(current_thread)
1991extern thread_t current_thread(void) __pure2;
1992#endif
1993
1994
1995extern uint64_t thread_tid(thread_t thread) __pure2;
1996
1997extern void thread_reference(
1998 thread_t thread);
1999
2000extern void thread_deallocate(
2001 thread_t thread);
2002
2003/*! @function kernel_thread_start
2004 * @abstract Create a kernel thread.
2005 * @discussion This function takes three input parameters, namely reference
2006 * to the function that the thread should execute, caller specified data
2007 * and a reference which is used to return the newly created kernel
2008 * thread. The function returns KERN_SUCCESS on success or an appropriate
2009 * kernel code type indicating the error. It may be noted that the caller
2010 * is responsible for explicitly releasing the reference to the created
2011 * thread when no longer needed. This should be done by calling
2012 * thread_deallocate(new_thread).
2013 * @param continuation A C-function pointer where the thread will begin execution.
2014 * @param parameter Caller specified data to be passed to the new thread.
2015 * @param new_thread Reference to the new thread is returned in this parameter.
2016 * @result Returns KERN_SUCCESS on success or an appropriate kernel code type.
2017 */
2018
2019extern kern_return_t kernel_thread_start(
2020 thread_continue_t continuation,
2021 void *parameter,
2022 thread_t *new_thread);
2023
2024__END_DECLS
2025
2026#endif /* _KERN_THREAD_H_ */