osfmk/arm64/status.c

xnu source @ f6217f8 2025-10-16 · 2725 lines · view on GitHub
1/*
* Copyright (c) 2007-2024 Apple Inc. All rights reserved.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_START@
*
* This file contains Original Code and/or Modifications of Original Code
* as defined in and that are subject to the Apple Public Source License
* Version 2.0 (the 'License'). You may not use this file except in
* compliance with the License. The rights granted to you under the License
* may not be used to create, or enable the creation or redistribution of,
* unlawful or unlicensed copies of an Apple operating system, or to
* circumvent, violate, or enable the circumvention or violation of, any
* terms of an Apple operating system software license agreement.
*
* Please obtain a copy of the License at
* http://www.opensource.apple.com/apsl/ and read it before using this file.
*
* The Original Code and all software distributed under the License are
* distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
* Please see the License for the specific language governing rights and
* limitations under the License.
*
* @APPLE_OSREFERENCE_LICENSE_HEADER_END@
*/
28#include <debug.h>
29#include <mach/mach_types.h>
30#include <mach/kern_return.h>
31#include <mach/thread_status.h>
32#include <kern/thread.h>
33#include <kern/kalloc.h>
34#include <arm/vmparam.h>
35#include <arm/cpu_data_internal.h>
36#include <arm/misc_protos.h>
37#include <arm64/machine_machdep.h>
38#include <arm64/proc_reg.h>
39#include <sys/random.h>
40#if __has_feature(ptrauth_calls)
41#include <ptrauth.h>
42#endif
43
44#include <libkern/coreanalytics/coreanalytics.h>
45
46
47struct arm_vfpv2_state {
__uint32_t __r[32];
__uint32_t __fpscr;
50};
51
52typedef struct arm_vfpv2_state arm_vfpv2_state_t;
53
54#define ARM_VFPV2_STATE_COUNT \
((mach_msg_type_number_t)(sizeof (arm_vfpv2_state_t)/sizeof(uint32_t)))
56
57/*
* Forward definitions
*/
60void thread_set_child(thread_t child, int pid);
61static void free_debug_state(thread_t thread);
62user_addr_t thread_get_sigreturn_token(thread_t thread);
63uint32_t thread_get_sigreturn_diversifier(thread_t thread);
64
65/*
* Maps state flavor to number of words in the state:
*/
68/* __private_extern__ */
69unsigned int _MachineStateCount[THREAD_STATE_FLAVORS] = {
[ARM_UNIFIED_THREAD_STATE] = ARM_UNIFIED_THREAD_STATE_COUNT,
[ARM_VFP_STATE] = ARM_VFP_STATE_COUNT,
[ARM_EXCEPTION_STATE] = ARM_EXCEPTION_STATE_COUNT,
[ARM_DEBUG_STATE] = ARM_DEBUG_STATE_COUNT,
[ARM_THREAD_STATE64] = ARM_THREAD_STATE64_COUNT,
[ARM_EXCEPTION_STATE64] = ARM_EXCEPTION_STATE64_COUNT,
[ARM_EXCEPTION_STATE64_V2] = ARM_EXCEPTION_STATE64_V2_COUNT,
[ARM_THREAD_STATE32] = ARM_THREAD_STATE32_COUNT,
[ARM_DEBUG_STATE32] = ARM_DEBUG_STATE32_COUNT,
[ARM_DEBUG_STATE64] = ARM_DEBUG_STATE64_COUNT,
[ARM_NEON_STATE] = ARM_NEON_STATE_COUNT,
[ARM_NEON_STATE64] = ARM_NEON_STATE64_COUNT,
[ARM_PAGEIN_STATE] = ARM_PAGEIN_STATE_COUNT,
/*
* Mach exception ports don't currently support SME state flavors.
* In case exception_deliver tries to access them anyway, give
* them bogus sizes that will ensure the access fails.
*/
[ARM_SME_STATE ... ARM_SME2_STATE] = 0,
89};
90
91extern zone_t ads_zone;
92
93#if __arm64__
94/*
* Copy values from saved_state to ts64.
*/
97void
98saved_state_to_thread_state64(const arm_saved_state_t * saved_state,
  arm_thread_state64_t *    ts64)
100{
uint32_t i;
102
assert(is_saved_state64(saved_state));
104
ts64->fp = get_saved_state_fp(saved_state);
ts64->lr = get_saved_state_lr(saved_state);
ts64->sp = get_saved_state_sp(saved_state);
ts64->pc = get_saved_state_pc(saved_state);
ts64->cpsr = get_saved_state_cpsr(saved_state);
for (i = 0; i < 29; i++) {
ts64->x[i] = get_saved_state_reg(saved_state, i);
}
113}
114
115/*
* Copy values from ts64 to saved_state.
*
* For safety, CPSR is sanitized as follows:
*
* - ts64->cpsr.{N,Z,C,V} are copied as-is into saved_state->cpsr
* - ts64->cpsr.M is ignored, and saved_state->cpsr.M is reset to EL0
* - All other saved_state->cpsr bits are preserved as-is
*/
124void
125thread_state64_to_saved_state(const arm_thread_state64_t * ts64,
  arm_saved_state_t *          saved_state)
127{
uint32_t i;
129#if __has_feature(ptrauth_calls)
uint64_t intr = ml_pac_safe_interrupts_disable();
131#endif /* __has_feature(ptrauth_calls) */
132
assert(is_saved_state64(saved_state));
134
const uint32_t CPSR_COPY_MASK = PSR64_USER_MASK;
const uint32_t CPSR_ZERO_MASK = PSR64_MODE_MASK;
const uint32_t CPSR_PRESERVE_MASK = ~(CPSR_COPY_MASK | CPSR_ZERO_MASK);
138#if __has_feature(ptrauth_calls)
/* BEGIN IGNORE CODESTYLE */
MANIPULATE_SIGNED_USER_THREAD_STATE(saved_state,
"and	w2, w2, %w[preserve_mask]"	"\n"
"mov	w6, %w[cpsr]"			"\n"
"and	w6, w6, %w[copy_mask]"		"\n"
"orr	w2, w2, w6"			"\n"
"str	w2, [x0, %[SS64_CPSR]]"		"\n",
[cpsr] "r"(ts64->cpsr),
[preserve_mask] "i"(CPSR_PRESERVE_MASK),
[copy_mask] "i"(CPSR_COPY_MASK)
);
/* END IGNORE CODESTYLE */
/*
* Make writes to ts64->cpsr visible first, since it's useful as a
* canary to detect thread-state corruption.
*/
__builtin_arm_dmb(DMB_ST);
156#else
uint32_t new_cpsr = get_saved_state_cpsr(saved_state);
new_cpsr &= CPSR_PRESERVE_MASK;
new_cpsr |= (ts64->cpsr & CPSR_COPY_MASK);
set_user_saved_state_cpsr(saved_state, new_cpsr);
161#endif /* __has_feature(ptrauth_calls) */
set_saved_state_fp(saved_state, ts64->fp);
set_user_saved_state_lr(saved_state, ts64->lr);
set_saved_state_sp(saved_state, ts64->sp);
set_user_saved_state_pc(saved_state, ts64->pc);
for (i = 0; i < 29; i++) {
set_user_saved_state_reg(saved_state, i, ts64->x[i]);
}
169
170#if __has_feature(ptrauth_calls)
ml_pac_safe_interrupts_restore(intr);
172#endif /* __has_feature(ptrauth_calls) */
173}
174
175#endif /* __arm64__ */
176
177static kern_return_t
178handle_get_arm32_thread_state(thread_state_t            tstate,
  mach_msg_type_number_t *  count,
  const arm_saved_state_t * saved_state)
181{
if (*count < ARM_THREAD_STATE32_COUNT) {
return KERN_INVALID_ARGUMENT;
}
if (!is_saved_state32(saved_state)) {
return KERN_INVALID_ARGUMENT;
}
188
(void)saved_state_to_thread_state32(saved_state, (arm_thread_state32_t *)tstate);
*count = ARM_THREAD_STATE32_COUNT;
return KERN_SUCCESS;
192}
193
194static kern_return_t
195handle_get_arm64_thread_state(thread_state_t            tstate,
  mach_msg_type_number_t *  count,
  const arm_saved_state_t * saved_state)
198{
if (*count < ARM_THREAD_STATE64_COUNT) {
return KERN_INVALID_ARGUMENT;
}
if (!is_saved_state64(saved_state)) {
return KERN_INVALID_ARGUMENT;
}
205
(void)saved_state_to_thread_state64(saved_state, (arm_thread_state64_t *)tstate);
*count = ARM_THREAD_STATE64_COUNT;
return KERN_SUCCESS;
209}
210
211
212static kern_return_t
213handle_get_arm_thread_state(thread_state_t            tstate,
  mach_msg_type_number_t *  count,
  const arm_saved_state_t * saved_state)
216{
/* In an arm64 world, this flavor can be used to retrieve the thread
* state of a 32-bit or 64-bit thread into a unified structure, but we
* need to support legacy clients who are only aware of 32-bit, so
* check the count to see what the client is expecting.
*/
if (*count < ARM_UNIFIED_THREAD_STATE_COUNT) {
return handle_get_arm32_thread_state(tstate, count, saved_state);
}
225
arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *) tstate;
bzero(unified_state, sizeof(*unified_state));
228#if __arm64__
if (is_saved_state64(saved_state)) {
unified_state->ash.flavor = ARM_THREAD_STATE64;
unified_state->ash.count = ARM_THREAD_STATE64_COUNT;
(void)saved_state_to_thread_state64(saved_state, thread_state64(unified_state));
} else
234#endif
{
unified_state->ash.flavor = ARM_THREAD_STATE32;
unified_state->ash.count = ARM_THREAD_STATE32_COUNT;
(void)saved_state_to_thread_state32(saved_state, thread_state32(unified_state));
}
*count = ARM_UNIFIED_THREAD_STATE_COUNT;
return KERN_SUCCESS;
242}
243
244
245static kern_return_t
246handle_set_arm32_thread_state(const thread_state_t   tstate,
  mach_msg_type_number_t count,
  arm_saved_state_t *    saved_state)
249{
if (count != ARM_THREAD_STATE32_COUNT) {
return KERN_INVALID_ARGUMENT;
}
253
(void)thread_state32_to_saved_state((const arm_thread_state32_t *)tstate, saved_state);
return KERN_SUCCESS;
256}
257
258static kern_return_t
259handle_set_arm64_thread_state(const thread_state_t   tstate,
  mach_msg_type_number_t count,
  arm_saved_state_t *    saved_state)
262{
if (count != ARM_THREAD_STATE64_COUNT) {
return KERN_INVALID_ARGUMENT;
}
266
(void)thread_state64_to_saved_state((const arm_thread_state64_t *)tstate, saved_state);
return KERN_SUCCESS;
269}
270
271
272static kern_return_t
273handle_set_arm_thread_state(const thread_state_t   tstate,
  mach_msg_type_number_t count,
  arm_saved_state_t *    saved_state)
276{
/* In an arm64 world, this flavor can be used to set the thread state of a
* 32-bit or 64-bit thread from a unified structure, but we need to support
* legacy clients who are only aware of 32-bit, so check the count to see
* what the client is expecting.
*/
if (count < ARM_UNIFIED_THREAD_STATE_COUNT) {
if (!is_saved_state32(saved_state)) {
	return KERN_INVALID_ARGUMENT;
}
return handle_set_arm32_thread_state(tstate, count, saved_state);
}
288
const arm_unified_thread_state_t *unified_state = (const arm_unified_thread_state_t *) tstate;
290#if __arm64__
if (is_thread_state64(unified_state)) {
if (!is_saved_state64(saved_state)) {
	return KERN_INVALID_ARGUMENT;
}
(void)thread_state64_to_saved_state(const_thread_state64(unified_state), saved_state);
} else
297#endif
{
if (!is_saved_state32(saved_state)) {
	return KERN_INVALID_ARGUMENT;
}
(void)thread_state32_to_saved_state(const_thread_state32(unified_state), saved_state);
}
304
return KERN_SUCCESS;
306}
307
308
309#if HAS_ARM_FEAT_SME
310static kern_return_t
311handle_get_arm_sme_state(thread_state_t tstate, mach_msg_type_number_t *count, thread_t thread)
312{
if (!arm_sme_version()) {
return KERN_INVALID_ARGUMENT;
}
316
if (*count < ARM_SME_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
320
arm_sme_state_t *state_out = (arm_sme_state_t *)tstate;
state_out->svl_b = arm_sme_svl_b();
323
/*
* Saved-state TPIDR2_EL0, ZA, and ZT0 are only updated on context-switch.  If
* the thread is currently active on this CPU, those saved-state values may be
* stale and must be resynchronized with the live CPU state.
*
* We don't need to do this for threads active on other CPUs, since thread_get_state()
* forcibly preempts them before calling machine_thread_get_state().
*/
if (thread == current_thread()) {
thread->machine.tpidr2_el0 = __builtin_arm_rsr64("TPIDR2_EL0");
}
state_out->tpidr2_el0 = thread->machine.tpidr2_el0;
336
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
/* This thread has never used SME before, so all SME state is invalid */
state_out->svcr = 0;
} else {
state_out->svcr = saved_state->svcr;
}
344
*count = ARM_SME_STATE_COUNT;
return KERN_SUCCESS;
347}
348
349static kern_return_t
350handle_get_arm_sve_z_state(thread_state_t tstate, mach_msg_type_number_t *count, thread_t thread, size_t z_offset)
351{
if (*count < ARM_SVE_Z_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
355
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_SM)) {
return KERN_INVALID_ARGUMENT;
}
363
arm_sve_z_state_t *state_out = (arm_sve_z_state_t *)tstate;
const size_t elem_size = saved_state->svl_b;
const uint8_t *saved_state_z = const_arm_sme_z(&saved_state->context) + z_offset * elem_size;
367
const size_t padding_size = sizeof(state_out->z[0]) - elem_size;
for (int i = 0; i < ARRAY_COUNT(state_out->z); i++) {
bcopy(saved_state_z, state_out->z[i], elem_size);
if (padding_size) {
	bzero(state_out->z[i] + elem_size, padding_size);
}
saved_state_z += elem_size;
}
376
*count = ARM_SVE_Z_STATE_COUNT;
return KERN_SUCCESS;
379}
380
381static kern_return_t
382handle_get_arm_sve_p_state(thread_state_t tstate, mach_msg_type_number_t *count, thread_t thread)
383{
if (*count < ARM_SVE_P_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
387
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_SM)) {
return KERN_INVALID_ARGUMENT;
}
395
arm_sve_p_state_t *state_out = (arm_sve_p_state_t *)tstate;
const uint8_t *saved_state_p = const_arm_sme_p(&saved_state->context, saved_state->svl_b);
398
const size_t elem_size = saved_state->svl_b / 8;
const size_t padding_size = sizeof(state_out->p[0]) - elem_size;
for (int i = 0; i < ARRAY_COUNT(state_out->p); i++) {
bcopy(saved_state_p, state_out->p[i], elem_size);
if (padding_size) {
	bzero(state_out->p[i] + elem_size, padding_size);
}
saved_state_p += elem_size;
}
408
*count = ARM_SVE_P_STATE_COUNT;
return KERN_SUCCESS;
411}
412
413static kern_return_t
414handle_get_arm_za_state(thread_state_t tstate, mach_msg_type_number_t *count, thread_t thread)
415{
if (*count < ARM_SME_ZA_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
419
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_ZA)) {
return KERN_INVALID_ARGUMENT;
}
427
if (thread == current_thread()) {
arm_save_sme_za(&saved_state->context, saved_state->svl_b);
}
431
const uint8_t *saved_state_za = const_arm_sme_za(&saved_state->context, saved_state->svl_b);
size_t za_size = saved_state->svl_b * saved_state->svl_b;
arm_sme_za_state_t *state_out = (arm_sme_za_state_t *)tstate;
assert(za_size <= sizeof(state_out->za));
436
bcopy(saved_state_za, state_out->za, za_size);
if (za_size < sizeof(state_out->za)) {
bzero(state_out->za + za_size, sizeof(state_out->za) - za_size);
}
441
*count = ARM_SME_ZA_STATE_COUNT;
return KERN_SUCCESS;
444}
445
446static kern_return_t
447handle_set_arm_sme_state(const thread_state_t tstate, mach_msg_type_number_t count, thread_t thread)
448{
if (!arm_sme_version()) {
return KERN_INVALID_ARGUMENT;
}
452
if (count < ARM_SME_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
456
const arm_sme_state_t *state_in = (const arm_sme_state_t *)tstate;
uint16_t svl_b = arm_sme_svl_b();
if (state_in->svl_b != svl_b) {
/* arm_sme_state_t::svl_b is currently read-only */
return KERN_INVALID_ARGUMENT;
}
463
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
kern_return_t err = machine_thread_sme_state_alloc(thread);
if (err) {
	return err;
}
saved_state = machine_thread_get_sme_state(thread);
assert(saved_state);
}
473
saved_state->svcr = state_in->svcr & (SVCR_SM | SVCR_ZA);
thread->machine.tpidr2_el0 = state_in->tpidr2_el0;
/*
* Like in handle_get_arm_sme_state(), if we're accessing live TPIDR2_EL0, ZA,
* or ZT0 state then we need to explicitly synchronize the saved-state with
* hardware.
*/
if (thread == current_thread()) {
__builtin_arm_wsr64("TPIDR2_EL0", state_in->tpidr2_el0);
}
return KERN_SUCCESS;
485}
486
487static kern_return_t
488handle_set_arm_sve_z_state(const thread_state_t tstate, mach_msg_type_number_t count, thread_t thread, size_t z_offset)
489{
if (count < ARM_SVE_Z_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
493
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_SM)) {
return KERN_INVALID_ARGUMENT;
}
501
const arm_sve_z_state_t *state_in = (const arm_sve_z_state_t *)tstate;
const size_t elem_size = saved_state->svl_b;
uint8_t *saved_state_z = arm_sme_z(&saved_state->context) + z_offset * elem_size;
505
for (int i = 0; i < ARRAY_COUNT(state_in->z); i++) {
bcopy(state_in->z[i], saved_state_z, elem_size);
saved_state_z += elem_size;
}
510
return KERN_SUCCESS;
512}
513
514static kern_return_t
515handle_set_arm_sve_p_state(const thread_state_t tstate, mach_msg_type_number_t count, thread_t thread)
516{
if (count < ARM_SVE_P_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
520
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_SM)) {
return KERN_INVALID_ARGUMENT;
}
528
const arm_sve_p_state_t *state_in = (const arm_sve_p_state_t *)tstate;
uint8_t *saved_state_p = arm_sme_p(&saved_state->context, saved_state->svl_b);
531
const size_t elem_size = saved_state->svl_b / 8;
for (int i = 0; i < ARRAY_COUNT(state_in->p); i++) {
bcopy(state_in->p[i], saved_state_p, elem_size);
saved_state_p += elem_size;
}
537
return KERN_SUCCESS;
539}
540
541static kern_return_t
542handle_set_arm_za_state(const thread_state_t tstate, mach_msg_type_number_t count, thread_t thread)
543{
if (count < ARM_SME_ZA_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
547
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
if (!(saved_state->svcr & SVCR_ZA)) {
return KERN_INVALID_ARGUMENT;
}
555
uint8_t *saved_state_za = arm_sme_za(&saved_state->context, saved_state->svl_b);
size_t za_size = saved_state->svl_b * saved_state->svl_b;
const arm_sme_za_state_t *state_in = (const arm_sme_za_state_t *)tstate;
assert(za_size <= sizeof(state_in->za));
560
bcopy(state_in->za, saved_state_za, za_size);
if (thread == current_thread()) {
arm_load_sme_za(&saved_state->context, saved_state->svl_b);
}
565
return KERN_SUCCESS;
567}
568
569#if HAS_ARM_FEAT_SME2
570static kern_return_t
571handle_get_arm_sme2_state(thread_state_t tstate, mach_msg_type_number_t *count, thread_t thread)
572{
if (arm_sme_version() < ARM_FEAT_SME2) {
return KERN_INVALID_ARGUMENT;
}
576
if (*count < ARM_SME2_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
580
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
585
if ((saved_state->svcr & SVCR_ZA) == 0) {
return KERN_INVALID_ARGUMENT;
}
589
arm_sme2_state_t *sme2_tstate = (arm_sme2_state_t *)tstate;
if (thread == current_thread()) {
arm_save_sme_zt0(&saved_state->context);
}
memcpy(sme2_tstate->zt0, saved_state->context.zt0, sizeof(saved_state->context.zt0));
595
*count = ARM_SME2_STATE_COUNT;
return KERN_SUCCESS;
598}
599
600static kern_return_t
601handle_set_arm_sme2_state(const thread_state_t tstate, mach_msg_type_number_t count, thread_t thread)
602{
if (arm_sme_version() < ARM_FEAT_SME2) {
return KERN_INVALID_ARGUMENT;
}
606
arm_sme_saved_state_t *saved_state = machine_thread_get_sme_state(thread);
if (!saved_state) {
return KERN_INVALID_ARGUMENT;
}
611
if (count < ARM_SME2_STATE_COUNT) {
return KERN_INVALID_ARGUMENT;
}
615
if ((saved_state->svcr & SVCR_ZA) == 0) {
return KERN_INVALID_ARGUMENT;
}
619
const arm_sme2_state_t *sme2_tstate = (const arm_sme2_state_t *)tstate;
memcpy(saved_state->context.zt0, sme2_tstate->zt0, sizeof(saved_state->context.zt0));
if (thread == current_thread()) {
arm_load_sme_zt0(&saved_state->context);
}
return KERN_SUCCESS;
626}
627#endif /* HAS_ARM_FEAT_SME2 */
628#endif /* HAS_ARM_FEAT_SME */
629
630#if __has_feature(ptrauth_calls)
631
632static inline uint32_t
633thread_generate_sigreturn_token(
void *ptr,
thread_t thread)
636{
user64_addr_t token = (user64_addr_t)ptr;
token ^= (user64_addr_t)thread_get_sigreturn_token(thread);
token = (user64_addr_t)pmap_sign_user_ptr((void*)token,
   ptrauth_key_process_independent_data, ptrauth_string_discriminator("nonce"),
   thread->machine.jop_pid);
token >>= 32;
return (uint32_t)token;
644}
645#endif //__has_feature(ptrauth_calls)
646
647/*
* Translate thread state arguments to userspace representation
*/
650
651kern_return_t
652machine_thread_state_convert_to_user(
thread_t thread,
thread_flavor_t flavor,
thread_state_t tstate,
mach_msg_type_number_t *count,
thread_set_status_flags_t tssf_flags)
658{
659#if __has_feature(ptrauth_calls)
arm_thread_state64_t *ts64;
bool preserve_flags = !!(tssf_flags & TSSF_PRESERVE_FLAGS);
bool stash_sigreturn_token = !!(tssf_flags & TSSF_STASH_SIGRETURN_TOKEN);
bool random_div = !!(tssf_flags & TSSF_RANDOM_USER_DIV);
bool thread_div = !!(tssf_flags & TSSF_THREAD_USER_DIV);
bool task_div = !!(tssf_flags & TSSF_TASK_USER_DIV);
uint32_t old_flags;
bool kernel_signed_pc = true;
bool kernel_signed_lr = true;
uint32_t userland_diversifier = 0;
670
switch (flavor) {
case ARM_THREAD_STATE:
{
arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *)tstate;
675
if (*count < ARM_UNIFIED_THREAD_STATE_COUNT || !is_thread_state64(unified_state)) {
	return KERN_SUCCESS;
}
ts64 = thread_state64(unified_state);
break;
}
case ARM_THREAD_STATE64:
{
if (*count < ARM_THREAD_STATE64_COUNT) {
	return KERN_SUCCESS;
}
ts64 = (arm_thread_state64_t *)tstate;
break;
}
default:
return KERN_SUCCESS;
}
693
// Note that kernel threads never have disable_user_jop set
if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
   !thread_is_64bit_addr(current_thread()) ||
   (thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) || !thread_is_64bit_addr(thread)
   ) {
ts64->flags = __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
return KERN_SUCCESS;
}
702
old_flags = ts64->flags;
ts64->flags = 0;
if (ts64->lr) {
// lr might contain an IB-signed return address (strip is a no-op on unsigned addresses)
uintptr_t stripped_lr = (uintptr_t)ptrauth_strip((void *)ts64->lr,
    ptrauth_key_return_address);
if (ts64->lr != stripped_lr) {
	// Need to allow already-signed lr value to round-trip as is
	ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
}
// Note that an IB-signed return address that happens to have a 0 signature value
// will round-trip correctly even if IA-signed again below (and IA-authd later)
}
716
if (arm_user_jop_disabled()) {
return KERN_SUCCESS;
}
720
if (preserve_flags) {
assert(random_div == false);
assert(thread_div == false);
724
/* Restore the diversifier and other opaque flags */
ts64->flags |= (old_flags & __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
userland_diversifier = old_flags & __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
if (!(old_flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC)) {
	kernel_signed_pc = false;
}
if (!(old_flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR)) {
	kernel_signed_lr = false;
}
} else {
/* Set a non zero userland diversifier */
if (random_div || task_div) {
	/* Still use random div in case of task_div to avoid leaking the secret key */
	do {
		read_random(&userland_diversifier, sizeof(userland_diversifier));
		userland_diversifier &=
		    __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
	} while (userland_diversifier == 0);
} else if (thread_div) {
	userland_diversifier = thread_get_sigreturn_diversifier(thread) &
	    __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK;
}
ts64->flags |= userland_diversifier;
}
749
if (kernel_signed_pc) {
ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC;
}
753
if (kernel_signed_lr) {
ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR;
}
757
758
if (ts64->pc) {
uint64_t discriminator = ptrauth_string_discriminator("pc");
if (!kernel_signed_pc && userland_diversifier != 0) {
	discriminator = ptrauth_blend_discriminator((void *)(long)userland_diversifier,
	    ptrauth_string_discriminator("pc"));
}
765
ts64->pc = (uintptr_t)pmap_sign_user_ptr((void*)ts64->pc,
    ptrauth_key_process_independent_code, discriminator,
    thread->machine.jop_pid);
}
if (ts64->lr && !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)) {
uint64_t discriminator = ptrauth_string_discriminator("lr");
if (!kernel_signed_lr && userland_diversifier != 0) {
	discriminator = ptrauth_blend_discriminator((void *)(long)userland_diversifier,
	    ptrauth_string_discriminator("lr"));
}
776
ts64->lr = (uintptr_t)pmap_sign_user_ptr((void*)ts64->lr,
    ptrauth_key_process_independent_code, discriminator,
    thread->machine.jop_pid);
}
if (ts64->sp) {
ts64->sp = (uintptr_t)pmap_sign_user_ptr((void*)ts64->sp,
    ptrauth_key_process_independent_data, ptrauth_string_discriminator("sp"),
    thread->machine.jop_pid);
}
if (ts64->fp) {
ts64->fp = (uintptr_t)pmap_sign_user_ptr((void*)ts64->fp,
    ptrauth_key_process_independent_data, ptrauth_string_discriminator("fp"),
    thread->machine.jop_pid);
}
791
/* Stash the sigreturn token */
if (stash_sigreturn_token) {
if (kernel_signed_pc) {
	uint32_t token = thread_generate_sigreturn_token((void *)ts64->pc, thread);
	__DARWIN_ARM_THREAD_STATE64_SET_SIGRETURN_TOKEN(ts64, token,
	    __DARWIN_ARM_THREAD_STATE64_SIGRETURN_PC_MASK);
}
799
if (kernel_signed_lr) {
	uint32_t token = thread_generate_sigreturn_token((void *)ts64->lr, thread);
	__DARWIN_ARM_THREAD_STATE64_SET_SIGRETURN_TOKEN(ts64, token,
	    __DARWIN_ARM_THREAD_STATE64_SIGRETURN_LR_MASK);
}
}
806
return KERN_SUCCESS;
808#else
// No conversion to userspace representation on this platform
(void)thread; (void)flavor; (void)tstate; (void)count; (void)tssf_flags;
return KERN_SUCCESS;
812#endif /* __has_feature(ptrauth_calls) */
813}
814
815#if __has_feature(ptrauth_calls)
816extern char *   proc_name_address(void *p);
817
818CA_EVENT(pac_thread_state_exception_event,
  CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name);
820
821static void
822machine_thread_state_check_pac_state(
arm_thread_state64_t *ts64,
arm_thread_state64_t *old_ts64)
825{
bool send_event = false;
task_t task = current_task();
void *proc = get_bsdtask_info(task);
char *proc_name = (char *) "unknown";
830
if (((ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC) &&
   ts64->pc != old_ts64->pc) || (!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) &&
   (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR) && (ts64->lr != old_ts64->lr ||
   (old_ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)))) {
send_event = true;
}
837
if (!send_event) {
return;
}
841
proc_name = proc_name_address(proc);
ca_event_t ca_event = CA_EVENT_ALLOCATE(pac_thread_state_exception_event);
CA_EVENT_TYPE(pac_thread_state_exception_event) * pexc_event = ca_event->data;
strlcpy(pexc_event->proc_name, proc_name, CA_PROCNAME_LEN);
CA_EVENT_SEND(ca_event);
847}
848
849CA_EVENT(pac_thread_state_sigreturn_event,
  CA_STATIC_STRING(CA_PROCNAME_LEN), proc_name);
851
852static bool
853machine_thread_state_check_sigreturn_token(
arm_thread_state64_t *ts64,
thread_t thread)
856{
task_t task = current_task();
void *proc = get_bsdtask_info(task);
char *proc_name = (char *) "unknown";
bool token_matched = true;
bool kernel_signed_pc = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC);
bool kernel_signed_lr = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR);
863
if (kernel_signed_pc) {
/* Compute the sigreturn token */
uint32_t token = thread_generate_sigreturn_token((void *)ts64->pc, thread);
if (!__DARWIN_ARM_THREAD_STATE64_CHECK_SIGRETURN_TOKEN(ts64, token,
    __DARWIN_ARM_THREAD_STATE64_SIGRETURN_PC_MASK)) {
	token_matched = false;
}
}
872
if (kernel_signed_lr) {
/* Compute the sigreturn token */
uint32_t token = thread_generate_sigreturn_token((void *)ts64->lr, thread);
if (!__DARWIN_ARM_THREAD_STATE64_CHECK_SIGRETURN_TOKEN(ts64, token,
    __DARWIN_ARM_THREAD_STATE64_SIGRETURN_LR_MASK)) {
	token_matched = false;
}
}
881
if (token_matched) {
return true;
}
885
proc_name = proc_name_address(proc);
ca_event_t ca_event = CA_EVENT_ALLOCATE(pac_thread_state_sigreturn_event);
CA_EVENT_TYPE(pac_thread_state_sigreturn_event) * psig_event = ca_event->data;
strlcpy(psig_event->proc_name, proc_name, CA_PROCNAME_LEN);
CA_EVENT_SEND(ca_event);
return false;
892}
893
894#endif
895
896/*
* Translate thread state arguments from userspace representation
*/
899
900kern_return_t
901machine_thread_state_convert_from_user(
thread_t thread,
thread_flavor_t flavor,
thread_state_t tstate,
mach_msg_type_number_t count,
thread_state_t old_tstate,
mach_msg_type_number_t old_count,
thread_set_status_flags_t tssf_flags)
909{
arm_thread_state64_t *ts64;
arm_thread_state64_t *old_ts64 = NULL;
bool only_set_pc = !!(tssf_flags & TSSF_ONLY_PC);
913
switch (flavor) {
case ARM_THREAD_STATE:
{
arm_unified_thread_state_t *unified_state = (arm_unified_thread_state_t *)tstate;
918
if (count < ARM_UNIFIED_THREAD_STATE_COUNT || !is_thread_state64(unified_state)) {
	return KERN_SUCCESS;
}
ts64 = thread_state64(unified_state);
923
arm_unified_thread_state_t *old_unified_state = (arm_unified_thread_state_t *)old_tstate;
if (old_unified_state && old_count >= ARM_UNIFIED_THREAD_STATE_COUNT) {
	old_ts64 = thread_state64(old_unified_state);
}
break;
}
case ARM_THREAD_STATE64:
{
if (count != ARM_THREAD_STATE64_COUNT) {
	return KERN_SUCCESS;
}
ts64 = (arm_thread_state64_t *)tstate;
936
if (old_count == ARM_THREAD_STATE64_COUNT) {
	old_ts64 = (arm_thread_state64_t *)old_tstate;
}
break;
}
default:
return KERN_SUCCESS;
}
945
if (only_set_pc) {
uint64_t new_pc = ts64->pc;
uint64_t new_flags = ts64->flags;
/* Only allow pc to be modified in new_state */
memcpy(ts64, old_ts64, sizeof(arm_thread_state64_t));
ts64->pc = new_pc;
ts64->flags = new_flags;
}
954
955#if __has_feature(ptrauth_calls)
956
void *userland_diversifier = NULL;
bool kernel_signed_pc;
bool kernel_signed_lr;
bool random_div = !!(tssf_flags & TSSF_RANDOM_USER_DIV);
bool thread_div = !!(tssf_flags & TSSF_THREAD_USER_DIV);
bool task_div = !!(tssf_flags & TSSF_TASK_USER_DIV);
963
// Note that kernel threads never have disable_user_jop set
if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
   !thread_is_64bit_addr(current_thread())) {
if ((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
    !thread_is_64bit_addr(thread)) {
	ts64->flags = __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
	return KERN_SUCCESS;
}
// A JOP-disabled process must not set thread state on a JOP-enabled process
return KERN_PROTECTION_FAILURE;
}
975
if (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH) {
if ((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
    !thread_is_64bit_addr(thread)
    ) {
	return KERN_SUCCESS;
}
// Disallow setting unsigned thread state on JOP-enabled processes.
// Ignore flag and treat thread state arguments as signed, ptrauth
// poisoning will cause resulting thread state to be invalid
ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_NO_PTRAUTH;
}
987
if (ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) {
// lr might contain an IB-signed return address (strip is a no-op on unsigned addresses)
uintptr_t stripped_lr = (uintptr_t)ptrauth_strip((void *)ts64->lr,
    ptrauth_key_return_address);
if (ts64->lr == stripped_lr) {
	// Don't allow unsigned pointer to be passed through as is. Ignore flag and
	// treat as IA-signed below (where auth failure may poison the value).
	ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
}
// Note that an IB-signed return address that happens to have a 0 signature value
// will also have been IA-signed (without this flag being set) and so will IA-auth
// correctly below.
}
1001
if (arm_user_jop_disabled()) {
return KERN_SUCCESS;
}
1005
kernel_signed_pc = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_PC);
kernel_signed_lr = !!(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_KERNEL_SIGNED_LR);
/*
* Replace pc/lr with old state if allow only
* user ptr flag is passed and ptrs are marked
* kernel signed.
*/
if ((tssf_flags & TSSF_CHECK_USER_FLAGS) &&
   (kernel_signed_pc || kernel_signed_lr)) {
if (old_ts64 && old_count == count) {
	/* Send a CA event if the thread state does not match */
	machine_thread_state_check_pac_state(ts64, old_ts64);
1018
	/* Check if user ptrs needs to be replaced */
	if ((tssf_flags & TSSF_ALLOW_ONLY_USER_PTRS) &&
	    kernel_signed_pc) {
		ts64->pc = old_ts64->pc;
	}
1024
	if ((tssf_flags & TSSF_ALLOW_ONLY_USER_PTRS) &&
	    !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) &&
	    kernel_signed_lr) {
		ts64->lr = old_ts64->lr;
		if (old_ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR) {
			ts64->flags |= __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
		} else {
			ts64->flags &= ~__DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR;
		}
	}
}
}
1037
/* Validate sigreturn token */
if (tssf_flags & TSSF_CHECK_SIGRETURN_TOKEN) {
bool token_matched = machine_thread_state_check_sigreturn_token(ts64, thread);
if ((tssf_flags & TSSF_ALLOW_ONLY_MATCHING_TOKEN) && !token_matched) {
	return KERN_PROTECTION_FAILURE;
}
}
1045
/* Get the userland diversifier */
if (random_div && old_ts64 && old_count == count) {
/* Get the random diversifier from the old thread state */
userland_diversifier = (void *)(long)(old_ts64->flags &
    __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
} else if (thread_div) {
userland_diversifier = (void *)(long)(thread_get_sigreturn_diversifier(thread) &
    __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
} else if (task_div) {
userland_diversifier =
    (void *)(long)((get_threadtask(thread)->hardened_exception_action.signed_pc_key) &
    __DARWIN_ARM_THREAD_STATE64_USER_DIVERSIFIER_MASK);
}
1059
if (ts64->pc) {
uint64_t discriminator = ptrauth_string_discriminator("pc");
if (!kernel_signed_pc && userland_diversifier != 0) {
	discriminator = ptrauth_blend_discriminator(userland_diversifier,
	    ptrauth_string_discriminator("pc"));
}
ts64->pc = (uintptr_t)pmap_auth_user_ptr((void*)ts64->pc,
    ptrauth_key_process_independent_code, discriminator,
    thread->machine.jop_pid);
}
if (ts64->lr && !(ts64->flags & __DARWIN_ARM_THREAD_STATE64_FLAGS_IB_SIGNED_LR)) {
uint64_t discriminator = ptrauth_string_discriminator("lr");
if (!kernel_signed_lr && userland_diversifier != 0) {
	discriminator = ptrauth_blend_discriminator(userland_diversifier,
	    ptrauth_string_discriminator("lr"));
}
ts64->lr = (uintptr_t)pmap_auth_user_ptr((void*)ts64->lr,
    ptrauth_key_process_independent_code, discriminator,
    thread->machine.jop_pid);
}
if (ts64->sp) {
ts64->sp = (uintptr_t)pmap_auth_user_ptr((void*)ts64->sp,
    ptrauth_key_process_independent_data, ptrauth_string_discriminator("sp"),
    thread->machine.jop_pid);
}
if (ts64->fp) {
ts64->fp = (uintptr_t)pmap_auth_user_ptr((void*)ts64->fp,
    ptrauth_key_process_independent_data, ptrauth_string_discriminator("fp"),
    thread->machine.jop_pid);
}
1090
return KERN_SUCCESS;
1092#else
// No conversion from userspace representation on this platform
(void)thread; (void)flavor; (void)tstate; (void)count;
(void)old_tstate; (void)old_count; (void)tssf_flags;
return KERN_SUCCESS;
1097#endif /* __has_feature(ptrauth_calls) */
1098}
1099
1100#if __has_feature(ptrauth_calls)
1101bool
1102machine_thread_state_is_debug_flavor(int flavor)
1103{
if (flavor == ARM_DEBUG_STATE ||
   flavor == ARM_DEBUG_STATE64 ||
   flavor == ARM_DEBUG_STATE32) {
return true;
}
return false;
1110}
1111#endif /* __has_feature(ptrauth_calls) */
1112
1113/*
* Translate signal context data pointer to userspace representation
*/
1116
1117kern_return_t
1118machine_thread_siguctx_pointer_convert_to_user(
thread_t thread,
user_addr_t *uctxp)
1121{
1122#if __has_feature(ptrauth_calls)
if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
   !thread_is_64bit_addr(current_thread())) {
assert((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) || !thread_is_64bit_addr(thread));
return KERN_SUCCESS;
}
1128
if (arm_user_jop_disabled()) {
return KERN_SUCCESS;
}
1132
if (*uctxp) {
*uctxp = (uintptr_t)pmap_sign_user_ptr((void*)*uctxp,
    ptrauth_key_process_independent_data, ptrauth_string_discriminator("uctx"),
    thread->machine.jop_pid);
}
1138
return KERN_SUCCESS;
1140#else
// No conversion to userspace representation on this platform
(void)thread; (void)uctxp;
return KERN_SUCCESS;
1144#endif /* __has_feature(ptrauth_calls) */
1145}
1146
1147/*
* Translate array of function pointer syscall arguments from userspace representation
*/
1150
1151kern_return_t
1152machine_thread_function_pointers_convert_from_user(
thread_t thread,
user_addr_t *fptrs,
uint32_t count)
1156{
1157#if __has_feature(ptrauth_calls)
if ((current_thread()->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
   !thread_is_64bit_addr(current_thread())) {
assert((thread->machine.arm_machine_flags & ARM_MACHINE_THREAD_DISABLE_USER_JOP) ||
    !thread_is_64bit_addr(thread));
return KERN_SUCCESS;
}
1164
if (arm_user_jop_disabled()) {
return KERN_SUCCESS;
}
1168
while (count--) {
if (*fptrs) {
	*fptrs = (uintptr_t)pmap_auth_user_ptr((void*)*fptrs,
	    ptrauth_key_function_pointer, 0, thread->machine.jop_pid);
}
fptrs++;
}
1176
return KERN_SUCCESS;
1178#else
// No conversion from userspace representation on this platform
(void)thread; (void)fptrs; (void)count;
return KERN_SUCCESS;
1182#endif /* __has_feature(ptrauth_calls) */
1183}
1184
1185/*
* Routine: machine_thread_get_state
*
*/
1189kern_return_t
1190machine_thread_get_state(thread_t                 thread,
  thread_flavor_t          flavor,
  thread_state_t           tstate,
  mach_msg_type_number_t * count)
1194{
switch (flavor) {
case THREAD_STATE_FLAVOR_LIST:
if (*count < 4) {
	return KERN_INVALID_ARGUMENT;
}
1200
tstate[0] = ARM_THREAD_STATE;
tstate[1] = ARM_VFP_STATE;
tstate[2] = ARM_EXCEPTION_STATE;
tstate[3] = ARM_DEBUG_STATE;
*count = 4;
break;
1207
case THREAD_STATE_FLAVOR_LIST_NEW:
if (*count < 4) {
	return KERN_INVALID_ARGUMENT;
}
1212
tstate[0] = ARM_THREAD_STATE;
tstate[1] = ARM_VFP_STATE;
tstate[2] = thread_is_64bit_data(thread) ? ARM_EXCEPTION_STATE64 : ARM_EXCEPTION_STATE;
tstate[3] = thread_is_64bit_data(thread) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32;
*count = 4;
break;
1219
case THREAD_STATE_FLAVOR_LIST_10_15:
if (*count < 5) {
	return KERN_INVALID_ARGUMENT;
}
1224
tstate[0] = ARM_THREAD_STATE;
tstate[1] = ARM_VFP_STATE;
tstate[2] = thread_is_64bit_data(thread) ? ARM_EXCEPTION_STATE64 : ARM_EXCEPTION_STATE;
tstate[3] = thread_is_64bit_data(thread) ? ARM_DEBUG_STATE64 : ARM_DEBUG_STATE32;
tstate[4] = ARM_PAGEIN_STATE;
*count = 5;
break;
1232
case ARM_THREAD_STATE:
{
kern_return_t rn = handle_get_arm_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
break;
}
case ARM_THREAD_STATE32:
{
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1246
kern_return_t rn = handle_get_arm32_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
break;
}
1253#if __arm64__
case ARM_THREAD_STATE64:
{
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1259
kern_return_t rn = handle_get_arm64_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
1264
break;
}
1267#endif
case ARM_EXCEPTION_STATE:{
struct arm_exception_state *state;
struct arm_saved_state32 *saved_state;
1271
if (*count < ARM_EXCEPTION_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1278
state = (struct arm_exception_state *) tstate;
saved_state = saved_state32(thread->machine.upcb);
1281
state->exception = saved_state->exception;
state->fsr = (uint32_t) saved_state->esr;
state->far = saved_state->far;
1285
*count = ARM_EXCEPTION_STATE_COUNT;
break;
}
case ARM_EXCEPTION_STATE64:{
struct arm_exception_state64 *state;
struct arm_saved_state64 *saved_state;
1292
if (*count < ARM_EXCEPTION_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1299
state = (struct arm_exception_state64 *) tstate;
saved_state = saved_state64(thread->machine.upcb);
1302
state->exception = 0;
state->far = saved_state->far;
state->esr = (uint32_t) saved_state->esr;
1306
*count = ARM_EXCEPTION_STATE64_COUNT;
break;
}
case ARM_EXCEPTION_STATE64_V2:{
struct arm_exception_state64_v2 *state;
struct arm_saved_state64 *saved_state;
1313
if (*count < ARM_EXCEPTION_STATE64_V2_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1320
state = (struct arm_exception_state64_v2 *) tstate;
saved_state = saved_state64(thread->machine.upcb);
1323
state->far = saved_state->far;
state->esr = saved_state->esr;
1326
*count = ARM_EXCEPTION_STATE64_V2_COUNT;
break;
}
case ARM_DEBUG_STATE:{
arm_legacy_debug_state_t *state;
arm_debug_state32_t *thread_state;
1333
if (*count < ARM_LEGACY_DEBUG_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1337
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1341
state = (arm_legacy_debug_state_t *) tstate;
thread_state = find_debug_state32(thread);
1344
if (thread_state == NULL) {
	bzero(state, sizeof(arm_legacy_debug_state_t));
} else {
	bcopy(thread_state, state, sizeof(arm_legacy_debug_state_t));
}
1350
*count = ARM_LEGACY_DEBUG_STATE_COUNT;
break;
}
case ARM_DEBUG_STATE32:{
arm_debug_state32_t *state;
arm_debug_state32_t *thread_state;
1357
if (*count < ARM_DEBUG_STATE32_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1361
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1365
state = (arm_debug_state32_t *) tstate;
thread_state = find_debug_state32(thread);
1368
if (thread_state == NULL) {
	bzero(state, sizeof(arm_debug_state32_t));
} else {
	bcopy(thread_state, state, sizeof(arm_debug_state32_t));
}
1374
*count = ARM_DEBUG_STATE32_COUNT;
break;
}
1378
case ARM_DEBUG_STATE64:{
arm_debug_state64_t *state;
arm_debug_state64_t *thread_state;
1382
if (*count < ARM_DEBUG_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1386
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1390
state = (arm_debug_state64_t *) tstate;
thread_state = find_debug_state64(thread);
1393
if (thread_state == NULL) {
	bzero(state, sizeof(arm_debug_state64_t));
} else {
	bcopy(thread_state, state, sizeof(arm_debug_state64_t));
}
1399
*count = ARM_DEBUG_STATE64_COUNT;
break;
}
1403
case ARM_VFP_STATE:{
struct arm_vfp_state *state;
arm_neon_saved_state32_t *thread_state;
unsigned int max;
1408
if (*count < ARM_VFP_STATE_COUNT) {
	if (*count < ARM_VFPV2_STATE_COUNT) {
		return KERN_INVALID_ARGUMENT;
	} else {
		*count =  ARM_VFPV2_STATE_COUNT;
	}
}
1416
if (*count == ARM_VFPV2_STATE_COUNT) {
	max = 32;
} else {
	max = 64;
}
1422
state = (struct arm_vfp_state *) tstate;
thread_state = neon_state32(thread->machine.uNeon);
/* ARM64 TODO: set fpsr and fpcr from state->fpscr */
1426
bcopy(thread_state, state, (max + 1) * sizeof(uint32_t));
*count = (max + 1);
break;
}
case ARM_NEON_STATE:{
arm_neon_state_t *state;
arm_neon_saved_state32_t *thread_state;
1434
if (*count < ARM_NEON_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1438
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1442
state = (arm_neon_state_t *)tstate;
thread_state = neon_state32(thread->machine.uNeon);
1445
assert(sizeof(*thread_state) == sizeof(*state));
bcopy(thread_state, state, sizeof(arm_neon_state_t));
1448
*count = ARM_NEON_STATE_COUNT;
break;
}
1452
case ARM_NEON_STATE64:{
arm_neon_state64_t *state;
const arm_neon_saved_state64_t *thread_state;
1456
if (*count < ARM_NEON_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1460
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1464
state = (arm_neon_state64_t *)tstate;
thread_state = neon_state64(thread->machine.uNeon);
1467
1468#if HAS_ARM_FEAT_SME
const arm_sme_saved_state_t *sme_ss = machine_thread_get_sme_state(thread);
/* Inside streaming SVE mode, reads from Q[i] access the lower 128 bits of Z[i] */
if (sme_ss && (sme_ss->svcr & SVCR_SM)) {
	const uint8_t *z = const_arm_sme_z(&sme_ss->context);
1473
	for (int i = 0; i < ARRAY_COUNT(state->q); i++) {
		bcopy(z, &state->q[i], sizeof(state->q[i]));
		z += sme_ss->svl_b;
	}
	state->fpcr = thread_state->fpcr;
	state->fpsr = thread_state->fpsr;
1480#else
if (0) {
1482#endif
} else {
	/* For now, these are identical */
	assert(sizeof(*state) == sizeof(*thread_state));
	bcopy(thread_state, state, sizeof(arm_neon_state64_t));
}
1488
1489
*count = ARM_NEON_STATE64_COUNT;
break;
}
1493
1494
case ARM_PAGEIN_STATE: {
arm_pagein_state_t *state;
1497
if (*count < ARM_PAGEIN_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1501
state = (arm_pagein_state_t *)tstate;
state->__pagein_error = thread->t_pagein_error;
1504
*count = ARM_PAGEIN_STATE_COUNT;
break;
}
1508
1509#if HAS_ARM_FEAT_SME
case ARM_SME_STATE:
return handle_get_arm_sme_state(tstate, count, thread);
1512
case ARM_SVE_Z_STATE1:
return handle_get_arm_sve_z_state(tstate, count, thread, 0);
1515
case ARM_SVE_Z_STATE2:
return handle_get_arm_sve_z_state(tstate, count, thread, 16);
1518
case ARM_SVE_P_STATE:
return handle_get_arm_sve_p_state(tstate, count, thread);
1521
case ARM_SME_ZA_STATE1:
return handle_get_arm_za_state(tstate, count, thread);
1524
case ARM_SME_ZA_STATE2 ... ARM_SME_ZA_STATE16:
/* Reserved for future use */
return KERN_INVALID_ARGUMENT;
1528
1529#if HAS_ARM_FEAT_SME2
case ARM_SME2_STATE:
return handle_get_arm_sme2_state(tstate, count, thread);
1532#endif
1533#endif
1534
default:
return KERN_INVALID_ARGUMENT;
}
return KERN_SUCCESS;
1539}
1540
1541
1542/*
* Routine: machine_thread_get_kern_state
*
*/
1546kern_return_t
1547machine_thread_get_kern_state(thread_t                 thread,
  thread_flavor_t          flavor,
  thread_state_t           tstate,
  mach_msg_type_number_t * count)
1551{
/*
* This works only for an interrupted kernel thread
*/
if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) {
return KERN_FAILURE;
}
1558
switch (flavor) {
case ARM_THREAD_STATE:
{
kern_return_t rn = handle_get_arm_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
if (rn) {
	return rn;
}
break;
}
case ARM_THREAD_STATE32:
{
kern_return_t rn = handle_get_arm32_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
if (rn) {
	return rn;
}
break;
}
1576#if __arm64__
case ARM_THREAD_STATE64:
{
kern_return_t rn = handle_get_arm64_thread_state(tstate, count, getCpuDatap()->cpu_int_state);
if (rn) {
	return rn;
}
break;
}
1585#endif
default:
return KERN_INVALID_ARGUMENT;
}
return KERN_SUCCESS;
1590}
1591
1592void
1593machine_thread_switch_addrmode(thread_t thread)
1594{
if (task_has_64Bit_data(get_threadtask(thread))) {
thread->machine.upcb->ash.flavor = ARM_SAVED_STATE64;
thread->machine.upcb->ash.count = ARM_SAVED_STATE64_COUNT;
thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64;
thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT;
1600
/*
 * Reinitialize the NEON state.
 */
bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns));
thread->machine.uNeon->ns_64.fpcr = FPCR_DEFAULT;
} else {
thread->machine.upcb->ash.flavor = ARM_SAVED_STATE32;
thread->machine.upcb->ash.count = ARM_SAVED_STATE32_COUNT;
thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
1611
/*
 * Reinitialize the NEON state.
 */
bzero(&thread->machine.uNeon->uns, sizeof(thread->machine.uNeon->uns));
thread->machine.uNeon->ns_32.fpcr = FPCR_DEFAULT_32;
}
1618}
1619
1620extern long long arm_debug_get(void);
1621
1622/*
* Routine: machine_thread_set_state
*
*/
1626kern_return_t
1627machine_thread_set_state(thread_t               thread,
  thread_flavor_t        flavor,
  thread_state_t         tstate,
  mach_msg_type_number_t count)
1631{
kern_return_t rn;
1633
switch (flavor) {
case ARM_THREAD_STATE:
rn = handle_set_arm_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
break;
1641
case ARM_THREAD_STATE32:
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1646
rn = handle_set_arm32_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
break;
1652
1653#if __arm64__
case ARM_THREAD_STATE64:
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1658
1659
rn = handle_set_arm64_thread_state(tstate, count, thread->machine.upcb);
if (rn) {
	return rn;
}
break;
1665#endif
case ARM_EXCEPTION_STATE:{
if (count != ARM_EXCEPTION_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1673
break;
}
case ARM_EXCEPTION_STATE64:{
if (count != ARM_EXCEPTION_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1683
break;
}
case ARM_EXCEPTION_STATE64_V2:{
if (count != ARM_EXCEPTION_STATE64_V2_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1693
break;
}
case ARM_DEBUG_STATE:
{
arm_legacy_debug_state_t *state;
boolean_t enabled = FALSE;
unsigned int    i;
1701
if (count != ARM_LEGACY_DEBUG_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1708
state = (arm_legacy_debug_state_t *) tstate;
1710
for (i = 0; i < 16; i++) {
	/* do not allow context IDs to be set */
	if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
	    || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
	    || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
	    || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
		return KERN_PROTECTION_FAILURE;
	}
	if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
	    || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
		enabled = TRUE;
	}
}
1724
if (!enabled) {
	free_debug_state(thread);
} else {
	arm_debug_state32_t *thread_state = find_or_allocate_debug_state32(thread);
1729
	if (thread_state == NULL) {
		return KERN_FAILURE;
	}
1733
	for (i = 0; i < 16; i++) {
		/* set appropriate privilege; mask out unknown bits */
		thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
		    | ARM_DBGBCR_MATCH_MASK
		    | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBGBCR_TYPE_IVA
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK;
		thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
		    | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBGWCR_ACCESS_CONTROL_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK;
	}
1754
	thread_state->mdscr_el1 = 0ULL;         // Legacy customers issuing ARM_DEBUG_STATE dont drive single stepping.
}
1757
if (thread == current_thread()) {
	arm_debug_set32(thread->machine.DebugData);
}
1761
break;
}
case ARM_DEBUG_STATE32:
/* ARM64_TODO  subtle bcr/wcr semantic differences e.g. wcr and ARM_DBGBCR_TYPE_IVA */
{
arm_debug_state32_t *state;
boolean_t enabled = FALSE;
unsigned int    i;
1770
if (count != ARM_DEBUG_STATE32_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1777
state = (arm_debug_state32_t *) tstate;
1779
if (state->mdscr_el1 & MDSCR_SS) {
	enabled = TRUE;
}
1783
for (i = 0; i < 16; i++) {
	/* do not allow context IDs to be set */
	if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
	    || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
	    || ((state->wcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
	    || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
		return KERN_PROTECTION_FAILURE;
	}
	if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
	    || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
		enabled = TRUE;
	}
}
1797
if (!enabled) {
	free_debug_state(thread);
} else {
	arm_debug_state32_t * thread_state = find_or_allocate_debug_state32(thread);
1802
	if (thread_state == NULL) {
		return KERN_FAILURE;
	}
1806
	if (state->mdscr_el1 & MDSCR_SS) {
		thread_state->mdscr_el1 |= MDSCR_SS;
	} else {
		thread_state->mdscr_el1 &= ~MDSCR_SS;
	}
1812
	for (i = 0; i < 16; i++) {
		/* set appropriate privilege; mask out unknown bits */
		thread_state->bcr[i] = (state->bcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
		    | ARM_DBGBCR_MATCH_MASK
		    | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBGBCR_TYPE_IVA
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK;
		thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
		    | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBGWCR_ACCESS_CONTROL_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK;
	}
}
1834
if (thread == current_thread()) {
	arm_debug_set32(thread->machine.DebugData);
}
1838
break;
}
1841
case ARM_DEBUG_STATE64:
{
arm_debug_state64_t *state;
boolean_t enabled = FALSE;
unsigned int i;
1847
if (count != ARM_DEBUG_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1854
state = (arm_debug_state64_t *) tstate;
1856
if (state->mdscr_el1 & MDSCR_SS) {
	enabled = TRUE;
}
1860
for (i = 0; i < 16; i++) {
	/* do not allow context IDs to be set */
	if (((state->bcr[i] & ARM_DBGBCR_TYPE_MASK) != ARM_DBGBCR_TYPE_IVA)
	    || ((state->bcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)
	    || ((state->wcr[i] & ARM_DBG_CR_LINKED_MASK) != ARM_DBG_CR_LINKED_UNLINKED)) {
		return KERN_PROTECTION_FAILURE;
	}
	if ((((state->bcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE))
	    || ((state->wcr[i] & ARM_DBG_CR_ENABLE_MASK) == ARM_DBG_CR_ENABLE_ENABLE)) {
		enabled = TRUE;
	}
}
1873
if (!enabled) {
	free_debug_state(thread);
} else {
	arm_debug_state64_t *thread_state = find_or_allocate_debug_state64(thread);
1878
	if (thread_state == NULL) {
		return KERN_FAILURE;
	}
1882
	if (state->mdscr_el1 & MDSCR_SS) {
		thread_state->mdscr_el1 |= MDSCR_SS;
	} else {
		thread_state->mdscr_el1 &= ~MDSCR_SS;
	}
1888
	for (i = 0; i < 16; i++) {
		/* set appropriate privilege; mask out unknown bits */
		thread_state->bcr[i] = (state->bcr[i] & (0         /* Was ARM_DBG_CR_ADDRESS_MASK_MASK deprecated in v8 */
		    | 0                             /* Was ARM_DBGBCR_MATCH_MASK, ignored in AArch64 state */
		    | ARM_DBG_CR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBGBCR_TYPE_IVA
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->bvr[i] = state->bvr[i] & ARM_DBG_VR_ADDRESS_MASK64;
		thread_state->wcr[i] = (state->wcr[i] & (ARM_DBG_CR_ADDRESS_MASK_MASK
		    | ARM_DBGWCR_BYTE_ADDRESS_SELECT_MASK
		    | ARM_DBGWCR_ACCESS_CONTROL_MASK
		    | ARM_DBG_CR_ENABLE_MASK))
		    | ARM_DBG_CR_LINKED_UNLINKED
		    | ARM_DBG_CR_SECURITY_STATE_BOTH
		    | ARM_DBG_CR_MODE_CONTROL_USER;
		thread_state->wvr[i] = state->wvr[i] & ARM_DBG_VR_ADDRESS_MASK64;
	}
}
1910
if (thread == current_thread()) {
	arm_debug_set64(thread->machine.DebugData);
}
1914
break;
}
1917
case ARM_VFP_STATE:{
struct arm_vfp_state *state;
arm_neon_saved_state32_t *thread_state;
unsigned int    max;
1922
if (count != ARM_VFP_STATE_COUNT && count != ARM_VFPV2_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1926
if (count == ARM_VFPV2_STATE_COUNT) {
	max = 32;
} else {
	max = 64;
}
1932
state = (struct arm_vfp_state *) tstate;
thread_state = neon_state32(thread->machine.uNeon);
/* ARM64 TODO: combine fpsr and fpcr into state->fpscr */
1936
bcopy(state, thread_state, (max + 1) * sizeof(uint32_t));
1938
thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
break;
}
1943
case ARM_NEON_STATE:{
arm_neon_state_t *state;
arm_neon_saved_state32_t *thread_state;
1947
if (count != ARM_NEON_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1951
if (thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1955
state = (arm_neon_state_t *)tstate;
thread_state = neon_state32(thread->machine.uNeon);
1958
assert(sizeof(*state) == sizeof(*thread_state));
bcopy(state, thread_state, sizeof(arm_neon_state_t));
1961
thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE32;
thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE32_COUNT;
break;
}
1966
case ARM_NEON_STATE64:{
arm_neon_state64_t *state;
arm_neon_saved_state64_t *thread_state;
1970
if (count != ARM_NEON_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
1974
if (!thread_is_64bit_data(thread)) {
	return KERN_INVALID_ARGUMENT;
}
1978
state = (arm_neon_state64_t *)tstate;
thread_state = neon_state64(thread->machine.uNeon);
1981
1982#if HAS_ARM_FEAT_SME
arm_sme_saved_state_t *sme_ss = machine_thread_get_sme_state(thread);
/*
 * Inside streaming SVE mode, writes to Q[i] modify the lower 128 bits
 * of Z[i], and zero out all bits > 128.
 */
if (sme_ss && (sme_ss->svcr & SVCR_SM)) {
	uint8_t *z = arm_sme_z(&sme_ss->context);
	const size_t elem_size = sizeof(state->q[0]);
	const size_t padding_size = sme_ss->svl_b - elem_size;
1992
	for (int i = 0; i < ARRAY_COUNT(state->q); i++) {
		bcopy(&state->q[i], z, elem_size);
		z += elem_size;
		bzero(z, padding_size);
		z += padding_size;
	}
	thread_state->fpcr = state->fpcr;
	thread_state->fpsr = state->fpsr;
2001#else
if (0) {
2003#endif
} else {
	assert(sizeof(*state) == sizeof(*thread_state));
	bcopy(state, thread_state, sizeof(arm_neon_state64_t));
}
2008
2009
thread->machine.uNeon->nsh.flavor = ARM_NEON_SAVED_STATE64;
thread->machine.uNeon->nsh.count = ARM_NEON_SAVED_STATE64_COUNT;
break;
}
2014
2015
2016#if HAS_ARM_FEAT_SME
case ARM_SME_STATE:
return handle_set_arm_sme_state(tstate, count, thread);
2019
case ARM_SVE_Z_STATE1:
return handle_set_arm_sve_z_state(tstate, count, thread, 0);
2022
case ARM_SVE_Z_STATE2:
return handle_set_arm_sve_z_state(tstate, count, thread, 16);
2025
case ARM_SVE_P_STATE:
return handle_set_arm_sve_p_state(tstate, count, thread);
2028
case ARM_SME_ZA_STATE1:
return handle_set_arm_za_state(tstate, count, thread);
2031
case ARM_SME_ZA_STATE2 ... ARM_SME_ZA_STATE16:
/* Reserved for future use */
return KERN_INVALID_ARGUMENT;
2035
2036#if HAS_ARM_FEAT_SME2
case ARM_SME2_STATE:
return handle_set_arm_sme2_state(tstate, count, thread);
2039#endif
2040#endif
2041
default:
return KERN_INVALID_ARGUMENT;
}
return KERN_SUCCESS;
2046}
2047
2048mach_vm_address_t
2049machine_thread_pc(thread_t thread)
2050{
struct arm_saved_state *ss = get_user_regs(thread);
return (mach_vm_address_t)get_saved_state_pc(ss);
2053}
2054
2055void
2056machine_thread_reset_pc(thread_t thread, mach_vm_address_t pc)
2057{
set_user_saved_state_pc(get_user_regs(thread), (register_t)pc);
2059}
2060
2061/*
* Routine: machine_thread_state_initialize
*
*/
2065void
2066machine_thread_state_initialize(thread_t thread)
2067{
arm_context_t *context = thread->machine.contextData;
2069
/*
* Should always be set up later. For a kernel thread, we don't care
* about this state. For a user thread, we'll set the state up in
* setup_wqthread, bsdthread_create, load_main(), or load_unixthread().
*/
2075
if (context != NULL) {
bzero(&context->ss.uss, sizeof(context->ss.uss));
bzero(&context->ns.uns, sizeof(context->ns.uns));
2079
if (context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64) {
	context->ns.ns_64.fpcr = FPCR_DEFAULT;
} else {
	context->ns.ns_32.fpcr = FPCR_DEFAULT_32;
}
context->ss.ss_64.cpsr = PSR64_USER64_DEFAULT;
}
2087
thread->machine.DebugData = NULL;
2089
2090#if defined(HAS_APPLE_PAC)
/* Sign the initial user-space thread state */
if (thread->machine.upcb != NULL) {
uint64_t intr = ml_pac_safe_interrupts_disable();
asm volatile (
                      "mov	x0, %[iss]"             "\n"
                      "mov	x1, #0"                 "\n"
                      "mov	w2, %w[usr]"            "\n"
                      "mov	x3, #0"                 "\n"
                      "mov	x4, #0"                 "\n"
                      "mov	x5, #0"                 "\n"
                      "msr	SPSel, #1"              "\n"
                      VERIFY_USER_THREAD_STATE_INSTR  "\n"
                      "mov	x6, lr"                 "\n"
                      "bl     _ml_sign_thread_state"  "\n"
                      "msr	SPSel, #0"              "\n"
                      "mov	lr, x6"                 "\n"
                      :
                      : [iss] "r"(thread->machine.upcb), [usr] "r"(thread->machine.upcb->ss_64.cpsr),
                        VERIFY_USER_THREAD_STATE_INPUTS
                      : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x17"
              );
ml_pac_safe_interrupts_restore(intr);
}
2114#endif /* defined(HAS_APPLE_PAC) */
2115}
2116
2117/*
* Routine: machine_thread_dup
*
*/
2121kern_return_t
2122machine_thread_dup(thread_t self,
  thread_t target,
  __unused boolean_t is_corpse)
2125{
struct arm_saved_state *self_saved_state;
struct arm_saved_state *target_saved_state;
2128
target->machine.cthread_self = self->machine.cthread_self;
2130
self_saved_state = self->machine.upcb;
target_saved_state = target->machine.upcb;
bcopy(self_saved_state, target_saved_state, sizeof(struct arm_saved_state));
2134#if defined(HAS_APPLE_PAC)
if (!is_corpse && is_saved_state64(self_saved_state)) {
check_and_sign_copied_user_thread_state(target_saved_state, self_saved_state);
}
2138#endif /* defined(HAS_APPLE_PAC) */
2139
arm_neon_saved_state_t *self_neon_state = self->machine.uNeon;
arm_neon_saved_state_t *target_neon_state = target->machine.uNeon;
bcopy(self_neon_state, target_neon_state, sizeof(*target_neon_state));
2143
2144#if HAVE_MACHINE_THREAD_MATRIX_STATE
if (self->machine.umatrix_hdr) {
machine_thread_matrix_state_dup(target);
}
2148#endif
2149
return KERN_SUCCESS;
2151}
2152
2153/*
* Routine: get_user_regs
*
*/
2157struct arm_saved_state *
2158get_user_regs(thread_t thread)
2159{
return thread->machine.upcb;
2161}
2162
2163/**
* Sets the value of NEON register Q[regno] inside the provided thread's
* saved-state.
*
* On SME-capable CPUs, this accessor follows the same aliasing rules as
* hardware.  If the thread is inside streaming SVE mode (SVCR.SM == 1), then
* writes to Q[regno] actually update Z[regno][127:0] and zero out
* Z[regno][SVL-1:128].
*
* @param thread thread
* @param regno which Q register to modify
* @param val the new value to store in Q[regno]
*/
2176void
2177set_user_neon_reg(thread_t thread, unsigned int regno, uint128_t val)
2178{
2179#if HAS_ARM_FEAT_SME
arm_sme_saved_state_t *sme_ss = machine_thread_get_sme_state(thread);
if (sme_ss && (sme_ss->svcr & SVCR_SM)) {
uint8_t *z_0 = arm_sme_z(&sme_ss->context);
uint8_t *z_dst = z_0 + (regno * sme_ss->svl_b);
bcopy(&val, z_dst, sizeof(val));
bzero(z_dst + sizeof(val), sme_ss->svl_b - sizeof(val));
return;
}
2188#endif
void *q_dst = &thread->machine.uNeon->ns_64.v.q[regno];
bcopy(&val, q_dst, sizeof(val));
2191}
2192
2193/*
* Routine: find_user_regs
*
*/
2197struct arm_saved_state *
2198find_user_regs(thread_t thread)
2199{
return thread->machine.upcb;
2201}
2202
2203/*
* Routine: find_kern_regs
*
*/
2207struct arm_saved_state *
2208find_kern_regs(thread_t thread)
2209{
/*
* This works only for an interrupted kernel thread
*/
if (thread != current_thread() || getCpuDatap()->cpu_int_state == NULL) {
return (struct arm_saved_state *) NULL;
} else {
return getCpuDatap()->cpu_int_state;
}
2218}
2219
2220arm_debug_state32_t *
2221find_debug_state32(thread_t thread)
2222{
if (thread && thread->machine.DebugData) {
return &(thread->machine.DebugData->uds.ds32);
} else {
return NULL;
}
2228}
2229
2230arm_debug_state64_t *
2231find_debug_state64(thread_t thread)
2232{
if (thread && thread->machine.DebugData) {
return &(thread->machine.DebugData->uds.ds64);
} else {
return NULL;
}
2238}
2239
2240os_refgrp_decl(static, dbg_refgrp, "arm_debug_state", NULL);
2241
2242/**
*  Finds the debug state for the given 64 bit thread, allocating one if it
*  does not exist.
*
*  @param thread 64 bit thread to find or allocate debug state for
*
*  @returns A pointer to the given thread's 64 bit debug state or a null
*           pointer if the given thread is null or the allocation of a new
*           debug state fails.
*/
2252arm_debug_state64_t *
2253find_or_allocate_debug_state64(thread_t thread)
2254{
arm_debug_state64_t *thread_state = find_debug_state64(thread);
if (thread != NULL && thread_state == NULL) {
thread->machine.DebugData = zalloc_flags(ads_zone,
    Z_WAITOK | Z_NOFAIL);
bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData));
thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE64;
thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE64_COUNT;
os_ref_init(&thread->machine.DebugData->ref, &dbg_refgrp);
thread_state = find_debug_state64(thread);
}
return thread_state;
2266}
2267
2268/**
*  Finds the debug state for the given 32 bit thread, allocating one if it
*  does not exist.
*
*  @param thread 32 bit thread to find or allocate debug state for
*
*  @returns A pointer to the given thread's 32 bit debug state or a null
*           pointer if the given thread is null or the allocation of a new
*           debug state fails.
*/
2278arm_debug_state32_t *
2279find_or_allocate_debug_state32(thread_t thread)
2280{
arm_debug_state32_t *thread_state = find_debug_state32(thread);
if (thread != NULL && thread_state == NULL) {
thread->machine.DebugData = zalloc_flags(ads_zone,
    Z_WAITOK | Z_NOFAIL);
bzero(thread->machine.DebugData, sizeof *(thread->machine.DebugData));
thread->machine.DebugData->dsh.flavor = ARM_DEBUG_STATE32;
thread->machine.DebugData->dsh.count = ARM_DEBUG_STATE32_COUNT;
os_ref_init(&thread->machine.DebugData->ref, &dbg_refgrp);
thread_state = find_debug_state32(thread);
}
return thread_state;
2292}
2293
2294/**
*	Frees a thread's debug state if allocated. Otherwise does nothing.
*
*  @param thread thread to free the debug state of
*/
2299static inline void
2300free_debug_state(thread_t thread)
2301{
if (thread != NULL && thread->machine.DebugData != NULL) {
arm_debug_state_t *pTmp = thread->machine.DebugData;
thread->machine.DebugData = NULL;
2305
if (os_ref_release(&pTmp->ref) == 0) {
	zfree(ads_zone, pTmp);
}
}
2310}
2311
2312/*
* Routine: thread_userstack
*
*/
2316kern_return_t
2317thread_userstack(__unused thread_t  thread,
  int                flavor,
  thread_state_t     tstate,
  unsigned int       count,
  mach_vm_offset_t * user_stack,
  int *              customstack,
  boolean_t          is_64bit_data
  )
2325{
register_t sp;
2327
switch (flavor) {
case ARM_THREAD_STATE:
if (count == ARM_UNIFIED_THREAD_STATE_COUNT) {
2331#if __arm64__
	if (is_64bit_data) {
		sp = ((arm_unified_thread_state_t *)tstate)->ts_64.sp;
	} else
2335#endif
	{
		sp = ((arm_unified_thread_state_t *)tstate)->ts_32.sp;
	}
2339
	break;
}
2342
/* INTENTIONAL FALL THROUGH (see machine_thread_set_state) */
OS_FALLTHROUGH;
case ARM_THREAD_STATE32:
if (count != ARM_THREAD_STATE32_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (is_64bit_data) {
	return KERN_INVALID_ARGUMENT;
}
2352
sp = ((arm_thread_state32_t *)tstate)->sp;
break;
2355#if __arm64__
case ARM_THREAD_STATE64:
if (count != ARM_THREAD_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
if (!is_64bit_data) {
	return KERN_INVALID_ARGUMENT;
}
2363
sp = ((arm_thread_state32_t *)tstate)->sp;
break;
2366#endif
default:
return KERN_INVALID_ARGUMENT;
}
2370
if (sp) {
*user_stack = CAST_USER_ADDR_T(sp);
if (customstack) {
	*customstack = 1;
}
} else {
*user_stack = CAST_USER_ADDR_T(USRSTACK64);
if (customstack) {
	*customstack = 0;
}
}
2382
return KERN_SUCCESS;
2384}
2385
2386/*
* thread_userstackdefault:
*
* Return the default stack location for the
* thread, if otherwise unknown.
*/
2392kern_return_t
2393thread_userstackdefault(mach_vm_offset_t * default_user_stack,
  boolean_t          is64bit)
2395{
if (is64bit) {
*default_user_stack = USRSTACK64;
} else {
*default_user_stack = USRSTACK;
}
2401
return KERN_SUCCESS;
2403}
2404
2405/*
* Routine: thread_setuserstack
*
*/
2409void
2410thread_setuserstack(thread_t          thread,
  mach_vm_address_t user_stack)
2412{
struct arm_saved_state *sv;
2414
sv = get_user_regs(thread);
2416
set_saved_state_sp(sv, user_stack);
2418
return;
2420}
2421
2422/*
* Routine: thread_adjuserstack
*
*/
2426user_addr_t
2427thread_adjuserstack(thread_t thread,
  int      adjust)
2429{
struct arm_saved_state *sv;
uint64_t sp;
2432
sv = get_user_regs(thread);
2434
sp = get_saved_state_sp(sv);
sp += adjust;
set_saved_state_sp(sv, sp);
2438
return sp;
2440}
2441
2442
2443/*
* Routine: thread_setentrypoint
*
*/
2447void
2448thread_setentrypoint(thread_t         thread,
  mach_vm_offset_t entry)
2450{
struct arm_saved_state *sv;
2452
2453#if HAS_APPLE_PAC
uint64_t intr = ml_pac_safe_interrupts_disable();
2455#endif
2456
sv = get_user_regs(thread);
2458
set_user_saved_state_pc(sv, entry);
2460
2461#if HAS_APPLE_PAC
ml_pac_safe_interrupts_restore(intr);
2463#endif
2464
return;
2466}
2467
2468/*
* Routine: thread_entrypoint
*
*/
2472kern_return_t
2473thread_entrypoint(__unused thread_t  thread,
  int                flavor,
  thread_state_t     tstate,
  unsigned int       count,
  mach_vm_offset_t * entry_point
  )
2479{
switch (flavor) {
case ARM_THREAD_STATE:
{
struct arm_thread_state *state;
2484
if (count != ARM_THREAD_STATE_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
2488
state = (struct arm_thread_state *) tstate;
2490
/*
 * If a valid entry point is specified, use it.
 */
if (state->pc) {
	*entry_point = CAST_USER_ADDR_T(state->pc);
} else {
	*entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS);
}
}
break;
2501
case ARM_THREAD_STATE64:
{
struct arm_thread_state64 *state;
2505
if (count != ARM_THREAD_STATE64_COUNT) {
	return KERN_INVALID_ARGUMENT;
}
2509
state = (struct arm_thread_state64*) tstate;
2511
/*
 * If a valid entry point is specified, use it.
 */
if (state->pc) {
	*entry_point = CAST_USER_ADDR_T(state->pc);
} else {
	*entry_point = CAST_USER_ADDR_T(VM_MIN_ADDRESS);
}
2520
break;
}
default:
return KERN_INVALID_ARGUMENT;
}
2526
return KERN_SUCCESS;
2528}
2529
2530
2531/*
* Routine: thread_set_child
*
*/
2535void
2536thread_set_child(thread_t child,
  int      pid)
2538{
struct arm_saved_state *child_state;
2540
child_state = get_user_regs(child);
2542
set_user_saved_state_reg(child_state, 0, pid);
set_user_saved_state_reg(child_state, 1, 1ULL);
2545}
2546
2547
2548struct arm_act_context {
struct arm_unified_thread_state ss;
2550#if __ARM_VFP__
struct arm_neon_saved_state ns;
2552#endif
2553};
2554
2555/*
* Routine: act_thread_csave
*
*/
2559void *
2560act_thread_csave(void)
2561{
struct arm_act_context *ic;
kern_return_t   kret;
unsigned int    val;
thread_t thread = current_thread();
2566
ic = kalloc_type(struct arm_act_context, Z_WAITOK);
if (ic == (struct arm_act_context *) NULL) {
return (void *) 0;
}
2571
val = ARM_UNIFIED_THREAD_STATE_COUNT;
kret = machine_thread_get_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, &val);
if (kret != KERN_SUCCESS) {
kfree_type(struct arm_act_context, ic);
return (void *) 0;
}
2578
2579#if __ARM_VFP__
if (thread_is_64bit_data(thread)) {
val = ARM_NEON_STATE64_COUNT;
kret = machine_thread_get_state(thread,
    ARM_NEON_STATE64,
    (thread_state_t)&ic->ns,
    &val);
} else {
val = ARM_NEON_STATE_COUNT;
kret = machine_thread_get_state(thread,
    ARM_NEON_STATE,
    (thread_state_t)&ic->ns,
    &val);
}
if (kret != KERN_SUCCESS) {
kfree_type(struct arm_act_context, ic);
return (void *) 0;
}
2597#endif
return ic;
2599}
2600
2601/*
* Routine: act_thread_catt
*
*/
2605void
2606act_thread_catt(void * ctx)
2607{
struct arm_act_context *ic;
kern_return_t   kret;
thread_t thread = current_thread();
2611
ic = (struct arm_act_context *) ctx;
if (ic == (struct arm_act_context *) NULL) {
return;
}
2616
kret = machine_thread_set_state(thread, ARM_THREAD_STATE, (thread_state_t)&ic->ss, ARM_UNIFIED_THREAD_STATE_COUNT);
if (kret != KERN_SUCCESS) {
goto out;
}
2621
2622#if __ARM_VFP__
if (thread_is_64bit_data(thread)) {
kret = machine_thread_set_state(thread,
    ARM_NEON_STATE64,
    (thread_state_t)&ic->ns,
    ARM_NEON_STATE64_COUNT);
} else {
kret = machine_thread_set_state(thread,
    ARM_NEON_STATE,
    (thread_state_t)&ic->ns,
    ARM_NEON_STATE_COUNT);
}
if (kret != KERN_SUCCESS) {
goto out;
}
2637#endif
2638out:
kfree_type(struct arm_act_context, ic);
2640}
2641
2642/*
* Routine: act_thread_catt
*
*/
2646void
2647act_thread_cfree(void *ctx)
2648{
kfree_type(struct arm_act_context, ctx);
2650}
2651
2652kern_return_t
2653thread_set_wq_state32(thread_t       thread,
  thread_state_t tstate)
2655{
arm_thread_state_t *state;
struct arm_saved_state *saved_state;
struct arm_saved_state32 *saved_state_32;
thread_t curth = current_thread();
spl_t s = 0;
2661
assert(!thread_is_64bit_data(thread));
2663
saved_state = thread->machine.upcb;
saved_state_32 = saved_state32(saved_state);
2666
state = (arm_thread_state_t *)tstate;
2668
if (curth != thread) {
s = splsched();
thread_lock(thread);
}
2673
/*
* do not zero saved_state, it can be concurrently accessed
* and zero is not a valid state for some of the registers,
* like sp.
*/
thread_state32_to_saved_state(state, saved_state);
saved_state_32->cpsr = PSR64_USER32_DEFAULT;
2681
if (curth != thread) {
thread_unlock(thread);
splx(s);
}
2686
return KERN_SUCCESS;
2688}
2689
2690kern_return_t
2691thread_set_wq_state64(thread_t       thread,
  thread_state_t tstate)
2693{
arm_thread_state64_t *state;
struct arm_saved_state *saved_state;
struct arm_saved_state64 *saved_state_64;
thread_t curth = current_thread();
spl_t s = 0;
2699
assert(thread_is_64bit_data(thread));
2701
saved_state = thread->machine.upcb;
saved_state_64 = saved_state64(saved_state);
state = (arm_thread_state64_t *)tstate;
2705
if (curth != thread) {
s = splsched();
thread_lock(thread);
}
2710
/*
* do not zero saved_state, it can be concurrently accessed
* and zero is not a valid state for some of the registers,
* like sp.
*/
thread_state64_to_saved_state(state, saved_state);
set_user_saved_state_cpsr(saved_state, PSR64_USER64_DEFAULT);
2718
if (curth != thread) {
thread_unlock(thread);
splx(s);
}
2723
return KERN_SUCCESS;
2725}