osfmk/mach/arm/thread_status.h
1/*
2 * Copyright (c) 2007 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * FILE_ID: thread_status.h
30 */
31
32
33#ifndef _ARM_THREAD_STATUS_H_
34#define _ARM_THREAD_STATUS_H_
35
36#if defined (__arm__) || defined (__arm64__)
37
38#include <mach/machine/_structs.h>
39#include <mach/machine/thread_state.h>
40#include <mach/message.h>
41#include <mach/vm_types.h>
42
43#ifdef XNU_KERNEL_PRIVATE
44#include <os/refcnt.h>
45#endif
46
47/*
48 * Support for determining the state of a thread
49 */
50
51
52/*
53 * Flavors
54 */
55
56#define ARM_THREAD_STATE 1
57#define ARM_UNIFIED_THREAD_STATE ARM_THREAD_STATE
58#define ARM_VFP_STATE 2
59#define ARM_EXCEPTION_STATE 3
60#define ARM_DEBUG_STATE 4 /* pre-armv8 */
61#define THREAD_STATE_NONE 5
62#define ARM_THREAD_STATE64 6
63#define ARM_EXCEPTION_STATE64 7
64// ARM_THREAD_STATE_LAST 8 /* legacy */
65#define ARM_THREAD_STATE32 9
66#define ARM_EXCEPTION_STATE64_V2 10
67
68#ifdef XNU_KERNEL_PRIVATE
69#define X86_THREAD_STATE_NONE 13 /* i386/thread_status.h THREAD_STATE_NONE */
70#endif /* XNU_KERNEL_PRIVATE */
71
72/* API */
73#define ARM_DEBUG_STATE32 14
74#define ARM_DEBUG_STATE64 15
75#define ARM_NEON_STATE 16
76#define ARM_NEON_STATE64 17
77#define ARM_CPMU_STATE64 18
78
79#ifdef XNU_KERNEL_PRIVATE
80/* For kernel use */
81#define ARM_SAVED_STATE32 20
82#define ARM_SAVED_STATE64 21
83#define ARM_NEON_SAVED_STATE32 22
84#define ARM_NEON_SAVED_STATE64 23
85#endif /* XNU_KERNEL_PRIVATE */
86
87#define ARM_PAGEIN_STATE 27
88
89/* API */
90#define ARM_SME_STATE 28
91#define ARM_SVE_Z_STATE1 29
92#define ARM_SVE_Z_STATE2 30
93#define ARM_SVE_P_STATE 31
94#define ARM_SME_ZA_STATE1 32
95#define ARM_SME_ZA_STATE2 33
96#define ARM_SME_ZA_STATE3 34
97#define ARM_SME_ZA_STATE4 35
98#define ARM_SME_ZA_STATE5 36
99#define ARM_SME_ZA_STATE6 37
100#define ARM_SME_ZA_STATE7 38
101#define ARM_SME_ZA_STATE8 39
102#define ARM_SME_ZA_STATE9 40
103#define ARM_SME_ZA_STATE10 41
104#define ARM_SME_ZA_STATE11 42
105#define ARM_SME_ZA_STATE12 42
106#define ARM_SME_ZA_STATE13 44
107#define ARM_SME_ZA_STATE14 45
108#define ARM_SME_ZA_STATE15 46
109#define ARM_SME_ZA_STATE16 47
110#define ARM_SME2_STATE 48
111#if XNU_KERNEL_PRIVATE
112/* For kernel use */
113#define ARM_SME_SAVED_STATE 49
114#endif /* XNU_KERNEL_PRIVATE */
115
116#define THREAD_STATE_FLAVORS 50 /* This must be updated to 1 more than the highest numerical state flavor */
117
118#ifndef ARM_STATE_FLAVOR_IS_OTHER_VALID
119#define ARM_STATE_FLAVOR_IS_OTHER_VALID(_flavor_) 0
120#endif
121
122#define FLAVOR_MODIFIES_CORE_CPU_REGISTERS(x) \
123((x == ARM_THREAD_STATE) || \
124 (x == ARM_THREAD_STATE32) || \
125 (x == ARM_THREAD_STATE64))
126
127#define VALID_THREAD_STATE_FLAVOR(x) \
128 ((x == ARM_THREAD_STATE) || \
129 (x == ARM_VFP_STATE) || \
130 (x == ARM_EXCEPTION_STATE) || \
131 (x == ARM_DEBUG_STATE) || \
132 (x == THREAD_STATE_NONE) || \
133 (x == ARM_THREAD_STATE32) || \
134 (x == ARM_THREAD_STATE64) || \
135 (x == ARM_EXCEPTION_STATE64) || \
136 (x == ARM_EXCEPTION_STATE64_V2) || \
137 (x == ARM_NEON_STATE) || \
138 (x == ARM_NEON_STATE64) || \
139 (x == ARM_DEBUG_STATE32) || \
140 (x == ARM_DEBUG_STATE64) || \
141 (x == ARM_PAGEIN_STATE) || \
142 (ARM_STATE_FLAVOR_IS_OTHER_VALID(x)))
143/*
144 * VALID_THREAD_STATE_FLAVOR() intentionally excludes ARM_SME_STATE through
145 * ARM_SME2_STATE, since these are not currently supported inside Mach exception
146 * ports.
147 */
148
149struct arm_state_hdr {
150 uint32_t flavor;
151 uint32_t count;
152};
153typedef struct arm_state_hdr arm_state_hdr_t;
154
155typedef _STRUCT_ARM_THREAD_STATE arm_thread_state_t;
156typedef _STRUCT_ARM_THREAD_STATE arm_thread_state32_t;
157typedef _STRUCT_ARM_THREAD_STATE64 arm_thread_state64_t;
158
159#if !defined(KERNEL)
160#if __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__)
161
162/* Accessor macros for arm_thread_state64_t pointer fields */
163
164/* Return pc field of arm_thread_state64_t as a data pointer value */
165#define arm_thread_state64_get_pc(ts) \
166 __darwin_arm_thread_state64_get_pc(ts)
167/* Return pc field of arm_thread_state64_t as a function pointer. May return
168 * NULL if a valid function pointer cannot be constructed, the caller should
169 * fall back to the arm_thread_state64_get_pc() macro in that case. */
170#define arm_thread_state64_get_pc_fptr(ts) \
171 __darwin_arm_thread_state64_get_pc_fptr(ts)
172/* Set pc field of arm_thread_state64_t to a function pointer */
173#define arm_thread_state64_set_pc_fptr(ts, fptr) \
174 __darwin_arm_thread_state64_set_pc_fptr(ts, fptr)
175/* Set pc field of arm_thread_state64_t to an already signed function pointer */
176#define arm_thread_state64_set_pc_presigned_fptr(ts, fptr) \
177 __darwin_arm_thread_state64_set_pc_presigned_fptr(ts, fptr)
178/* Return lr field of arm_thread_state64_t as a data pointer value */
179#define arm_thread_state64_get_lr(ts) \
180 __darwin_arm_thread_state64_get_lr(ts)
181/* Return lr field of arm_thread_state64_t as a function pointer. May return
182 * NULL if a valid function pointer cannot be constructed, the caller should
183 * fall back to the arm_thread_state64_get_lr() macro in that case. */
184#define arm_thread_state64_get_lr_fptr(ts) \
185 __darwin_arm_thread_state64_get_lr_fptr(ts)
186/* Set lr field of arm_thread_state64_t to a function pointer */
187#define arm_thread_state64_set_lr_fptr(ts, fptr) \
188 __darwin_arm_thread_state64_set_lr_fptr(ts, fptr)
189/* Set lr field of arm_thread_state64_t to an already signed function pointer */
190#define arm_thread_state64_set_lr_presigned_fptr(ts, fptr) \
191 __darwin_arm_thread_state64_set_lr_presigned_fptr(ts, fptr)
192/* Return sp field of arm_thread_state64_t as a data pointer value */
193#define arm_thread_state64_get_sp(ts) \
194 __darwin_arm_thread_state64_get_sp(ts)
195/* Set sp field of arm_thread_state64_t to a data pointer value */
196#define arm_thread_state64_set_sp(ts, ptr) \
197 __darwin_arm_thread_state64_set_sp(ts, ptr)
198/* Return fp field of arm_thread_state64_t as a data pointer value */
199#define arm_thread_state64_get_fp(ts) \
200 __darwin_arm_thread_state64_get_fp(ts)
201/* Set fp field of arm_thread_state64_t to a data pointer value */
202#define arm_thread_state64_set_fp(ts, ptr) \
203 __darwin_arm_thread_state64_set_fp(ts, ptr)
204/* Strip ptr auth bits from pc, lr, sp and fp field of arm_thread_state64_t */
205#define arm_thread_state64_ptrauth_strip(ts) \
206 __darwin_arm_thread_state64_ptrauth_strip(ts)
207
208#endif /* __DARWIN_C_LEVEL >= __DARWIN_C_FULL && defined(__arm64__) */
209#endif /* !defined(KERNEL) */
210
211struct arm_unified_thread_state {
212 arm_state_hdr_t ash;
213 union {
214 arm_thread_state32_t ts_32;
215 arm_thread_state64_t ts_64;
216 } uts;
217};
218#define ts_32 uts.ts_32
219#define ts_64 uts.ts_64
220typedef struct arm_unified_thread_state arm_unified_thread_state_t;
221
222#define ARM_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
223 (sizeof (arm_thread_state_t)/sizeof(uint32_t)))
224#define ARM_THREAD_STATE32_COUNT ((mach_msg_type_number_t) \
225 (sizeof (arm_thread_state32_t)/sizeof(uint32_t)))
226#define ARM_THREAD_STATE64_COUNT ((mach_msg_type_number_t) \
227 (sizeof (arm_thread_state64_t)/sizeof(uint32_t)))
228#define ARM_UNIFIED_THREAD_STATE_COUNT ((mach_msg_type_number_t) \
229 (sizeof (arm_unified_thread_state_t)/sizeof(uint32_t)))
230
231
232typedef _STRUCT_ARM_VFP_STATE arm_vfp_state_t;
233typedef _STRUCT_ARM_NEON_STATE arm_neon_state_t;
234typedef _STRUCT_ARM_NEON_STATE arm_neon_state32_t;
235typedef _STRUCT_ARM_NEON_STATE64 arm_neon_state64_t;
236
237
238typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state_t;
239typedef _STRUCT_ARM_EXCEPTION_STATE arm_exception_state32_t;
240typedef _STRUCT_ARM_EXCEPTION_STATE64 arm_exception_state64_t;
241typedef _STRUCT_ARM_EXCEPTION_STATE64_V2 arm_exception_state64_v2_t;
242
243typedef _STRUCT_ARM_DEBUG_STATE32 arm_debug_state32_t;
244typedef _STRUCT_ARM_DEBUG_STATE64 arm_debug_state64_t;
245
246typedef _STRUCT_ARM_PAGEIN_STATE arm_pagein_state_t;
247
248typedef _STRUCT_ARM_SME_STATE arm_sme_state_t;
249typedef _STRUCT_ARM_SVE_Z_STATE arm_sve_z_state_t;
250typedef _STRUCT_ARM_SVE_P_STATE arm_sve_p_state_t;
251typedef _STRUCT_ARM_SME_ZA_STATE arm_sme_za_state_t;
252typedef _STRUCT_ARM_SME2_STATE arm_sme2_state_t;
253
254#if defined(XNU_KERNEL_PRIVATE) && defined(__arm64__)
255/* See below for ARM64 kernel structure definition for arm_debug_state. */
256#else /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
257/*
258 * Otherwise not ARM64 kernel and we must preserve legacy ARM definitions of
259 * arm_debug_state for binary compatability of userland consumers of this file.
260 */
261#if defined(__arm__)
262typedef _STRUCT_ARM_DEBUG_STATE arm_debug_state_t;
263#elif defined(__arm64__)
264typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_debug_state_t;
265#else /* defined(__arm__) */
266#error Undefined architecture
267#endif /* defined(__arm__) */
268#endif /* defined(XNU_KERNEL_PRIVATE) && defined(__arm64__) */
269
270#define ARM_VFP_STATE_COUNT ((mach_msg_type_number_t) \
271 (sizeof (arm_vfp_state_t)/sizeof(uint32_t)))
272
273#define ARM_EXCEPTION_STATE_COUNT ((mach_msg_type_number_t) \
274 (sizeof (arm_exception_state_t)/sizeof(uint32_t)))
275
276#define ARM_EXCEPTION_STATE64_COUNT ((mach_msg_type_number_t) \
277 (sizeof (arm_exception_state64_t)/sizeof(uint32_t)))
278
279#define ARM_EXCEPTION_STATE64_V2_COUNT ((mach_msg_type_number_t) \
280 (sizeof (arm_exception_state64_v2_t)/sizeof(uint32_t)))
281
282#define ARM_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
283 (sizeof (arm_debug_state_t)/sizeof(uint32_t)))
284
285#define ARM_DEBUG_STATE32_COUNT ((mach_msg_type_number_t) \
286 (sizeof (arm_debug_state32_t)/sizeof(uint32_t)))
287
288#define ARM_PAGEIN_STATE_COUNT ((mach_msg_type_number_t) \
289 (sizeof (arm_pagein_state_t)/sizeof(uint32_t)))
290
291#define ARM_DEBUG_STATE64_COUNT ((mach_msg_type_number_t) \
292 (sizeof (arm_debug_state64_t)/sizeof(uint32_t)))
293
294#define ARM_NEON_STATE_COUNT ((mach_msg_type_number_t) \
295 (sizeof (arm_neon_state_t)/sizeof(uint32_t)))
296
297#define ARM_NEON_STATE64_COUNT ((mach_msg_type_number_t) \
298 (sizeof (arm_neon_state64_t)/sizeof(uint32_t)))
299
300#define ARM_SME_STATE_COUNT ((mach_msg_type_number_t) \
301 (sizeof (arm_sme_state_t)/sizeof(uint32_t)))
302
303#define ARM_SVE_Z_STATE_COUNT ((mach_msg_type_number_t) \
304 (sizeof (arm_sve_z_state_t)/sizeof(uint32_t)))
305
306#define ARM_SVE_P_STATE_COUNT ((mach_msg_type_number_t) \
307 (sizeof (arm_sve_p_state_t)/sizeof(uint32_t)))
308
309#define ARM_SME_ZA_STATE_COUNT ((mach_msg_type_number_t) \
310 (sizeof (arm_sme_za_state_t)/sizeof(uint32_t)))
311
312#define ARM_SME2_STATE_COUNT ((mach_msg_type_number_t) \
313 (sizeof (arm_sme2_state_t)/sizeof(uint32_t)))
314
315#define MACHINE_THREAD_STATE ARM_THREAD_STATE
316#define MACHINE_THREAD_STATE_COUNT ARM_UNIFIED_THREAD_STATE_COUNT
317
318
319/*
320 * Largest state on this machine:
321 */
322#define THREAD_MACHINE_STATE_MAX THREAD_STATE_MAX
323
324#ifdef XNU_KERNEL_PRIVATE
325
326#if CONFIG_DTRACE
327#define HAS_ADD_SAVED_STATE_PC 1
328#define HAS_SET_SAVED_STATE_PC 1
329#define HAS_SET_SAVED_STATE_LR 1
330#define HAS_SET_SAVED_STATE_REG 1
331#define HAS_MASK_SAVED_STATE_CPSR 1
332#endif /* CONFIG_DTRACE */
333
334#if CONFIG_KDP_INTERACTIVE_DEBUGGING
335#define HAS_SET_SAVED_STATE_CPSR 1
336#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
337
338#if CONFIG_XNUPOST
339#define HAS_ADD_SAVED_STATE_PC 1
340#define HAS_SET_SAVED_STATE_PC 1
341#define HAS_SET_SAVED_STATE_CPSR 1
342#endif /* CONFIG_DTRACE */
343
344#if DEBUG || DEVELOPMENT
345#define HAS_ADD_SAVED_STATE_PC 1
346#endif
347
348
349static inline boolean_t
350is_thread_state32(const arm_unified_thread_state_t *its)
351{
352 return its->ash.flavor == ARM_THREAD_STATE32;
353}
354
355static inline boolean_t
356is_thread_state64(const arm_unified_thread_state_t *its)
357{
358 return its->ash.flavor == ARM_THREAD_STATE64;
359}
360
361static inline arm_thread_state32_t*
362thread_state32(arm_unified_thread_state_t *its)
363{
364 return &its->ts_32;
365}
366
367static inline arm_thread_state64_t*
368thread_state64(arm_unified_thread_state_t *its)
369{
370 return &its->ts_64;
371}
372
373static inline const arm_thread_state32_t*
374const_thread_state32(const arm_unified_thread_state_t *its)
375{
376 return &its->ts_32;
377}
378
379static inline const arm_thread_state64_t*
380const_thread_state64(const arm_unified_thread_state_t *its)
381{
382 return &its->ts_64;
383}
384
385#if defined(__arm64__)
386
387#include <kern/assert.h>
388#include <arm64/proc_reg.h>
389#define CAST_ASSERT_SAFE(type, val) (assert((val) == ((type)(val))), (type)(val))
390
391/*
392 * GPR context
393 */
394
395struct arm_saved_state32 {
396 uint32_t r[13]; /* General purpose register r0-r12 */
397 uint32_t sp; /* Stack pointer r13 */
398 uint32_t lr; /* Link register r14 */
399 uint32_t pc; /* Program counter r15 */
400 uint32_t cpsr; /* Current program status register */
401 uint32_t far; /* Virtual fault address */
402 uint32_t esr; /* Exception syndrome register */
403 uint32_t exception; /* Exception number */
404};
405typedef struct arm_saved_state32 arm_saved_state32_t;
406
407struct arm_saved_state32_tagged {
408 uint32_t tag;
409 struct arm_saved_state32 state;
410};
411typedef struct arm_saved_state32_tagged arm_saved_state32_tagged_t;
412
413#define ARM_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
414 (sizeof(arm_saved_state32_t)/sizeof(unsigned int)))
415
416struct arm_saved_state64 {
417 uint64_t x[29]; /* General purpose registers x0-x28 */
418 uint64_t fp; /* Frame pointer x29 */
419 uint64_t lr; /* Link register x30 */
420 uint64_t sp; /* Stack pointer x31 */
421 uint64_t pc; /* Program counter */
422 uint32_t cpsr; /* Current program status register */
423 uint32_t reserved; /* Reserved padding */
424 uint64_t far; /* Virtual fault address */
425 uint64_t esr; /* Exception syndrome register */
426#if HAS_APPLE_PAC
427 uint64_t jophash;
428#endif /* HAS_APPLE_PAC */
429};
430typedef struct arm_saved_state64 arm_saved_state64_t;
431
432#define ARM_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
433 (sizeof(arm_saved_state64_t)/sizeof(unsigned int)))
434
435struct arm_saved_state64_tagged {
436 uint32_t tag;
437 struct arm_saved_state64 state;
438};
439typedef struct arm_saved_state64_tagged arm_saved_state64_tagged_t;
440
441struct arm_saved_state {
442 arm_state_hdr_t ash;
443 union {
444 struct arm_saved_state32 ss_32;
445 struct arm_saved_state64 ss_64;
446 } uss;
447} __attribute__((aligned(16)));
448#define ss_32 uss.ss_32
449#define ss_64 uss.ss_64
450
451typedef struct arm_saved_state arm_saved_state_t;
452
453struct arm_kernel_saved_state {
454 uint64_t x[10]; /* General purpose registers x19-x28 */
455 uint64_t fp; /* Frame pointer x29 */
456 uint64_t lr; /* Link register x30 */
457 uint64_t sp; /* Stack pointer x31 */
458 /* Some things here we DO need to preserve */
459 uint8_t pc_was_in_userspace;
460 uint8_t ssbs;
461 uint8_t dit;
462 uint8_t uao;
463} __attribute__((aligned(16)));
464
465typedef struct arm_kernel_saved_state arm_kernel_saved_state_t;
466
467extern void ml_panic_on_invalid_old_cpsr(const arm_saved_state_t *) __attribute__((noreturn));
468
469extern void ml_panic_on_invalid_new_cpsr(const arm_saved_state_t *, uint32_t) __attribute__((noreturn));
470
471#if HAS_APPLE_PAC
472
473#include <sys/cdefs.h>
474
475/*
476 * Used by MANIPULATE_SIGNED_THREAD_STATE(), potentially from C++ (IOKit) code.
477 * Open-coded to prevent a circular dependency between mach/arm/thread_status.h
478 * and osfmk/arm/machine_routines.h.
479 */
480__BEGIN_DECLS
481extern uint64_t ml_pac_safe_interrupts_disable(void);
482extern void ml_pac_safe_interrupts_restore(uint64_t);
483__END_DECLS
484
485/*
486 * Methods used to sign and check thread state to detect corruptions of saved
487 * thread state across exceptions and context switches.
488 */
489extern void ml_sign_thread_state(arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
490
491extern void ml_check_signed_state(const arm_saved_state_t *, uint64_t, uint32_t, uint64_t, uint64_t, uint64_t);
492
493/* XXX: including stddef.f here breaks ctfmerge on some builds, so use __builtin_offsetof() instead of offsetof() */
494#define ss64_offsetof(x) __builtin_offsetof(struct arm_saved_state, ss_64.x)
495
496/**
497 * Verify the signed thread state in _iss, execute the assembly instructions
498 * _instr, and re-sign the modified thread state. Varargs specify additional
499 * inputs.
500 *
501 * _instr may read or modify the thread state in the following registers:
502 *
503 * x0: _iss
504 * x1: authed _iss->ss_64.pc
505 * w2: authed _iss->ss_64.cpsr
506 * x3: authed _iss->ss_64.lr
507 * x4: authed _iss->ss_64.x16
508 * x5: authed _iss->ss_64.x17
509 * x6: scratch register
510 * x7: scratch register
511 * x8: scratch register
512 *
513 * If _instr makes no changes to the thread state, it may skip re-signing by
514 * branching to label 0.
515 */
516#define MANIPULATE_SIGNED_THREAD_STATE(_iss, _instr, ...) \
517 do { \
518 uint64_t _intr = ml_pac_safe_interrupts_disable(); \
519 asm volatile ( \
520 "mov x9, lr" "\n" \
521 "mov x0, %[iss]" "\n" \
522 "msr SPSel, #1" "\n" \
523 "ldp x4, x5, [x0, %[SS64_X16]]" "\n" \
524 "ldr x7, [x0, %[SS64_PC]]" "\n" \
525 "ldr w8, [x0, %[SS64_CPSR]]" "\n" \
526 "ldr x3, [x0, %[SS64_LR]]" "\n" \
527 "mov x1, x7" "\n" \
528 "mov w2, w8" "\n" \
529 "bl _ml_check_signed_state" "\n" \
530 "mov x1, x7" "\n" \
531 "mov w2, w8" "\n" \
532 _instr "\n" \
533 "bl _ml_sign_thread_state" "\n" \
534 "0:" "\n" \
535 "msr SPSel, #0" "\n" \
536 "mov lr, x9" "\n" \
537 : \
538 : [iss] "r"(_iss), \
539 [SS64_X16] "i"(ss64_offsetof(x[16])), \
540 [SS64_PC] "i"(ss64_offsetof(pc)), \
541 [SS64_CPSR] "i"(ss64_offsetof(cpsr)), \
542 [SS64_LR] "i"(ss64_offsetof(lr)),##__VA_ARGS__ \
543 : "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", \
544 "x9", "x16", "x17" \
545 ); \
546 ml_pac_safe_interrupts_restore(_intr); \
547 } while (0)
548
549#define VERIFY_USER_THREAD_STATE_INSTR \
550 "and w6, w2, %[CPSR_EL_MASK]" "\n" \
551 "cmp w6, %[CPSR_EL0]" "\n" \
552 "b.eq 1f" "\n" \
553 "bl _ml_panic_on_invalid_old_cpsr" "\n" \
554 "brk #0" "\n" \
555 "1:" "\n"
556
557#define VERIFY_USER_THREAD_STATE_INPUTS \
558 [CPSR_EL_MASK] "i"(PSR64_MODE_EL_MASK), \
559 [CPSR_EL0] "i"(PSR64_MODE_EL0)
560
561#define MANIPULATE_SIGNED_USER_THREAD_STATE(_iss, _instr, ...) \
562 MANIPULATE_SIGNED_THREAD_STATE(_iss, \
563 VERIFY_USER_THREAD_STATE_INSTR \
564 _instr, \
565 VERIFY_USER_THREAD_STATE_INPUTS, ##__VA_ARGS__)
566
567static inline void
568check_and_sign_copied_user_thread_state(arm_saved_state_t *dst, const arm_saved_state_t *src)
569{
570 MANIPULATE_SIGNED_USER_THREAD_STATE(src,
571 "mov x0, %[dst]",
572 [dst] "r"(dst)
573 );
574}
575#endif /* HAS_APPLE_PAC */
576
577static inline boolean_t
578is_saved_state32(const arm_saved_state_t *iss)
579{
580 return iss->ash.flavor == ARM_SAVED_STATE32;
581}
582
583static inline boolean_t
584is_saved_state64(const arm_saved_state_t *iss)
585{
586 return iss->ash.flavor == ARM_SAVED_STATE64;
587}
588
589static inline arm_saved_state32_t*
590saved_state32(arm_saved_state_t *iss)
591{
592 return &iss->ss_32;
593}
594
595static inline const arm_saved_state32_t*
596const_saved_state32(const arm_saved_state_t *iss)
597{
598 return &iss->ss_32;
599}
600
601static inline arm_saved_state64_t*
602saved_state64(arm_saved_state_t *iss)
603{
604 return &iss->ss_64;
605}
606
607static inline const arm_saved_state64_t*
608const_saved_state64(const arm_saved_state_t *iss)
609{
610 return &iss->ss_64;
611}
612
613static inline register_t
614get_saved_state_pc(const arm_saved_state_t *iss)
615{
616 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->pc : const_saved_state64(iss)->pc);
617}
618
619#if HAS_ADD_SAVED_STATE_PC
620static inline void
621add_saved_state_pc(arm_saved_state_t *iss, int diff)
622{
623 if (is_saved_state32(iss)) {
624 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
625 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
626 } else {
627#if HAS_APPLE_PAC
628 MANIPULATE_SIGNED_THREAD_STATE(iss,
629 "mov w6, %w[diff] \n"
630 "add x1, x1, w6, sxtw \n"
631 "str x1, [x0, %[SS64_PC]] \n",
632 [diff] "r"(diff)
633 );
634#else
635 saved_state64(iss)->pc += (unsigned long)diff;
636#endif /* HAS_APPLE_PAC */
637 }
638}
639#endif /* HAS_ADD_SAVED_STATE_PC */
640
641static inline void
642add_user_saved_state_pc(arm_saved_state_t *iss, int diff)
643{
644 if (is_saved_state32(iss)) {
645 uint64_t pc = saved_state32(iss)->pc + (uint32_t)diff;
646 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
647 } else {
648#if HAS_APPLE_PAC
649 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
650 "mov w6, %w[diff] \n"
651 "add x1, x1, w6, sxtw \n"
652 "str x1, [x0, %[SS64_PC]] \n",
653 [diff] "r"(diff)
654 );
655#else
656 saved_state64(iss)->pc += (unsigned long)diff;
657#endif /* HAS_APPLE_PAC */
658 }
659}
660
661#if HAS_SET_SAVED_STATE_PC
662static inline void
663set_saved_state_pc(arm_saved_state_t *iss, register_t pc)
664{
665 if (is_saved_state32(iss)) {
666 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
667 } else {
668#if HAS_APPLE_PAC
669 MANIPULATE_SIGNED_THREAD_STATE(iss,
670 "mov x1, %[pc] \n"
671 "str x1, [x0, %[SS64_PC]] \n",
672 [pc] "r"(pc)
673 );
674#else
675 saved_state64(iss)->pc = (unsigned long)pc;
676#endif /* HAS_APPLE_PAC */
677 }
678}
679#endif /* HAS_SET_SAVED_STATE_PC */
680
681static inline void
682set_user_saved_state_pc(arm_saved_state_t *iss, register_t pc)
683{
684 if (is_saved_state32(iss)) {
685 saved_state32(iss)->pc = CAST_ASSERT_SAFE(uint32_t, pc);
686 } else {
687#if HAS_APPLE_PAC
688 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
689 "mov x1, %[pc] \n"
690 "str x1, [x0, %[SS64_PC]] \n",
691 [pc] "r"(pc)
692 );
693#else
694 saved_state64(iss)->pc = (unsigned long)pc;
695#endif /* HAS_APPLE_PAC */
696 }
697}
698
699static inline register_t
700get_saved_state_sp(const arm_saved_state_t *iss)
701{
702 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->sp : const_saved_state64(iss)->sp);
703}
704
705static inline void
706set_saved_state_sp(arm_saved_state_t *iss, register_t sp)
707{
708 if (is_saved_state32(iss)) {
709 saved_state32(iss)->sp = CAST_ASSERT_SAFE(uint32_t, sp);
710 } else {
711 saved_state64(iss)->sp = (uint64_t)sp;
712 }
713}
714
715static inline register_t
716get_saved_state_lr(const arm_saved_state_t *iss)
717{
718 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->lr : const_saved_state64(iss)->lr);
719}
720
721#if HAS_SET_SAVED_STATE_LR
722static inline void
723set_saved_state_lr(arm_saved_state_t *iss, register_t lr)
724{
725 if (is_saved_state32(iss)) {
726 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
727 } else {
728#if HAS_APPLE_PAC
729 MANIPULATE_SIGNED_THREAD_STATE(iss,
730 "mov x3, %[lr] \n"
731 "str x3, [x0, %[SS64_LR]] \n",
732 [lr] "r"(lr)
733 );
734#else
735 saved_state64(iss)->lr = (unsigned long)lr;
736#endif /* HAS_APPLE_PAC */
737 }
738}
739#endif /* HAS_SET_SAVED_STATE_PC */
740
741static inline void
742set_user_saved_state_lr(arm_saved_state_t *iss, register_t lr)
743{
744 if (is_saved_state32(iss)) {
745 saved_state32(iss)->lr = CAST_ASSERT_SAFE(uint32_t, lr);
746 } else {
747#if HAS_APPLE_PAC
748 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
749 "mov x3, %[lr] \n"
750 "str x3, [x0, %[SS64_LR]] \n",
751 [lr] "r"(lr)
752 );
753#else
754 saved_state64(iss)->lr = (unsigned long)lr;
755#endif /* HAS_APPLE_PAC */
756 }
757}
758
759static inline register_t
760get_saved_state_fp(const arm_saved_state_t *iss)
761{
762 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->r[7] : const_saved_state64(iss)->fp);
763}
764
765static inline void
766set_saved_state_fp(arm_saved_state_t *iss, register_t fp)
767{
768 if (is_saved_state32(iss)) {
769 saved_state32(iss)->r[7] = CAST_ASSERT_SAFE(uint32_t, fp);
770 } else {
771 saved_state64(iss)->fp = (uint64_t)fp;
772 }
773}
774
775static inline int
776check_saved_state_reglimit(const arm_saved_state_t *iss, unsigned reg)
777{
778 return is_saved_state32(iss) ? (reg < ARM_SAVED_STATE32_COUNT) : (reg < ARM_SAVED_STATE64_COUNT);
779}
780
781static inline register_t
782get_saved_state_reg(const arm_saved_state_t *iss, unsigned reg)
783{
784 if (!check_saved_state_reglimit(iss, reg)) {
785 return 0;
786 }
787
788 return (register_t)(is_saved_state32(iss) ? (const_saved_state32(iss)->r[reg]) : (const_saved_state64(iss)->x[reg]));
789}
790
791#if HAS_SET_SAVED_STATE_REG
792static inline void
793set_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
794{
795 if (!check_saved_state_reglimit(iss, reg)) {
796 return;
797 }
798
799 if (is_saved_state32(iss)) {
800 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
801 } else {
802#if HAS_APPLE_PAC
803 /* x16 and x17 are part of the jophash */
804 if (reg == 16) {
805 MANIPULATE_SIGNED_THREAD_STATE(iss,
806 "mov x4, %[value] \n"
807 "str x4, [x0, %[SS64_X16]] \n",
808 [value] "r"(value)
809 );
810 return;
811 } else if (reg == 17) {
812 MANIPULATE_SIGNED_THREAD_STATE(iss,
813 "mov x5, %[value] \n"
814 "str x5, [x0, %[SS64_X17]] \n",
815 [value] "r"(value),
816 [SS64_X17] "i"(ss64_offsetof(x[17]))
817 );
818 return;
819 }
820#endif
821 saved_state64(iss)->x[reg] = (uint64_t)value;
822 }
823}
824#endif /* HAS_SET_SAVED_STATE_REG */
825
826static inline void
827set_user_saved_state_reg(arm_saved_state_t *iss, unsigned reg, register_t value)
828{
829 if (!check_saved_state_reglimit(iss, reg)) {
830 return;
831 }
832
833 if (is_saved_state32(iss)) {
834 saved_state32(iss)->r[reg] = CAST_ASSERT_SAFE(uint32_t, value);
835 } else {
836#if HAS_APPLE_PAC
837 /* x16 and x17 are part of the jophash */
838 if (reg == 16) {
839 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
840 "mov x4, %[value] \n"
841 "str x4, [x0, %[SS64_X16]] \n",
842 [value] "r"(value)
843 );
844 return;
845 } else if (reg == 17) {
846 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
847 "mov x5, %[value] \n"
848 "str x5, [x0, %[SS64_X17]] \n",
849 [value] "r"(value),
850 [SS64_X17] "i"(ss64_offsetof(x[17]))
851 );
852 return;
853 }
854#endif
855 saved_state64(iss)->x[reg] = (uint64_t)value;
856 }
857}
858
859
860static inline uint32_t
861get_saved_state_cpsr(const arm_saved_state_t *iss)
862{
863 return is_saved_state32(iss) ? const_saved_state32(iss)->cpsr : const_saved_state64(iss)->cpsr;
864}
865
866#if HAS_MASK_SAVED_STATE_CPSR
867static inline void
868mask_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
869{
870 if (is_saved_state32(iss)) {
871 saved_state32(iss)->cpsr |= set_bits;
872 saved_state32(iss)->cpsr &= ~clear_bits;
873 } else {
874#if HAS_APPLE_PAC
875 MANIPULATE_SIGNED_THREAD_STATE(iss,
876 "mov w6, %w[set_bits] \n"
877 "orr w2, w2, w6, lsl #0 \n"
878 "mov w6, %w[clear_bits] \n"
879 "bic w2, w2, w6, lsl #0 \n"
880 "str w2, [x0, %[SS64_CPSR]] \n",
881 [set_bits] "r"(set_bits),
882 [clear_bits] "r"(clear_bits)
883 );
884#else
885 saved_state64(iss)->cpsr |= set_bits;
886 saved_state64(iss)->cpsr &= ~clear_bits;
887#endif /* HAS_APPLE_PAC */
888 }
889}
890#endif /* HAS_MASK_SAVED_STATE_CPSR */
891
892static inline void
893mask_user_saved_state_cpsr(arm_saved_state_t *iss, uint32_t set_bits, uint32_t clear_bits)
894{
895 if (is_saved_state32(iss)) {
896 uint32_t new_cpsr = saved_state32(iss)->cpsr;
897 new_cpsr |= set_bits;
898 new_cpsr &= ~clear_bits;
899 if (!PSR_IS_USER(new_cpsr)) {
900 ml_panic_on_invalid_new_cpsr(iss, new_cpsr);
901 }
902 saved_state32(iss)->cpsr = new_cpsr;
903 } else {
904#if HAS_APPLE_PAC
905 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
906 "mov w6, %w[set_bits] \n"
907 "orr w2, w2, w6, lsl #0 \n"
908 "mov w6, %w[clear_bits] \n"
909 "bic w2, w2, w6, lsl #0 \n"
910 "and w6, w2, %[CPSR_EL_MASK] \n"
911 "cmp w6, %[CPSR_EL0] \n"
912 "b.eq 1f \n"
913 "mov w1, w2 \n"
914 "bl _ml_panic_on_invalid_new_cpsr \n"
915 "brk #0 \n"
916 "1: \n"
917 "str w2, [x0, %[SS64_CPSR]] \n",
918 [set_bits] "r"(set_bits),
919 [clear_bits] "r"(clear_bits)
920 );
921#else
922 uint32_t new_cpsr = saved_state64(iss)->cpsr;
923 new_cpsr |= set_bits;
924 new_cpsr &= ~clear_bits;
925 if (!PSR64_IS_USER(new_cpsr)) {
926 ml_panic_on_invalid_new_cpsr(iss, new_cpsr);
927 }
928 saved_state64(iss)->cpsr = new_cpsr;
929#endif /* HAS_APPLE_PAC */
930 }
931}
932
933#if HAS_SET_SAVED_STATE_CPSR
934static inline void
935set_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
936{
937 if (is_saved_state32(iss)) {
938 saved_state32(iss)->cpsr = cpsr;
939 } else {
940#if HAS_APPLE_PAC
941 MANIPULATE_SIGNED_THREAD_STATE(iss,
942 "mov w2, %w[cpsr] \n"
943 "str w2, [x0, %[SS64_CPSR]] \n",
944 [cpsr] "r"(cpsr)
945 );
946#else
947 saved_state64(iss)->cpsr = cpsr;
948#endif /* HAS_APPLE_PAC */
949 }
950}
951#endif /* HAS_SET_SAVED_STATE_CPSR */
952
953static inline void
954set_user_saved_state_cpsr(arm_saved_state_t *iss, uint32_t cpsr)
955{
956 if (is_saved_state32(iss)) {
957 if (!PSR_IS_USER(cpsr)) {
958 ml_panic_on_invalid_new_cpsr(iss, cpsr);
959 }
960 saved_state32(iss)->cpsr = cpsr;
961 } else {
962#if HAS_APPLE_PAC
963 MANIPULATE_SIGNED_USER_THREAD_STATE(iss,
964 "mov w2, %w[cpsr] \n"
965 "and w6, w2, %[CPSR_EL_MASK] \n"
966 "cmp w6, %[CPSR_EL0] \n"
967 "b.eq 1f \n"
968 "mov w1, w2 \n"
969 "bl _ml_panic_on_invalid_new_cpsr \n"
970 "brk #0 \n"
971 "1: \n"
972 "str w2, [x0, %[SS64_CPSR]] \n",
973 [cpsr] "r"(cpsr)
974 );
975#else
976 if (!PSR64_IS_USER(cpsr)) {
977 ml_panic_on_invalid_new_cpsr(iss, cpsr);
978 }
979 saved_state64(iss)->cpsr = cpsr;
980#endif /* HAS_APPLE_PAC */
981 }
982}
983
984static inline register_t
985get_saved_state_far(const arm_saved_state_t *iss)
986{
987 return (register_t)(is_saved_state32(iss) ? const_saved_state32(iss)->far : const_saved_state64(iss)->far);
988}
989
990static inline void
991set_saved_state_far(arm_saved_state_t *iss, register_t far)
992{
993 if (is_saved_state32(iss)) {
994 saved_state32(iss)->far = CAST_ASSERT_SAFE(uint32_t, far);
995 } else {
996 saved_state64(iss)->far = (uint64_t)far;
997 }
998}
999
1000static inline uint64_t
1001get_saved_state_esr(const arm_saved_state_t *iss)
1002{
1003 return is_saved_state32(iss) ? const_saved_state32(iss)->esr : const_saved_state64(iss)->esr;
1004}
1005
1006static inline void
1007set_saved_state_esr(arm_saved_state_t *iss, uint64_t esr)
1008{
1009 if (is_saved_state32(iss)) {
1010 assert(esr < (uint64_t) (uint32_t) -1);
1011 saved_state32(iss)->esr = (uint32_t) esr;
1012 } else {
1013 saved_state64(iss)->esr = esr;
1014 }
1015}
1016
1017extern void panic_unimplemented(void);
1018
1019/**
1020 * Extracts the SVC (Supervisor Call) number from the appropriate GPR (General
1021 * Purpose Register).
1022 *
1023 * @param[in] iss the 32-bit or 64-bit ARM saved state (i.e. trap frame).
1024 *
1025 * @return The SVC number.
1026 */
1027static inline int
1028get_saved_state_svc_number(const arm_saved_state_t *iss)
1029{
1030 return is_saved_state32(iss) ? (int)const_saved_state32(iss)->r[12] : (int)const_saved_state64(iss)->x[ARM64_SYSCALL_CODE_REG_NUM]; /* Only first word counts here */
1031}
1032
1033typedef _STRUCT_ARM_LEGACY_DEBUG_STATE arm_legacy_debug_state_t;
1034
1035struct arm_debug_aggregate_state {
1036 arm_state_hdr_t dsh;
1037 union {
1038 arm_debug_state32_t ds32;
1039 arm_debug_state64_t ds64;
1040 } uds;
1041 os_refcnt_t ref;
1042} __attribute__((aligned(16)));
1043
1044typedef struct arm_debug_aggregate_state arm_debug_state_t;
1045
1046#define ARM_LEGACY_DEBUG_STATE_COUNT ((mach_msg_type_number_t) \
1047 (sizeof (arm_legacy_debug_state_t)/sizeof(uint32_t)))
1048
1049/*
1050 * NEON context
1051 */
1052typedef __uint128_t uint128_t;
1053typedef uint64_t uint64x2_t __attribute__((ext_vector_type(2)));
1054typedef uint32_t uint32x4_t __attribute__((ext_vector_type(4)));
1055
1056struct arm_neon_saved_state32 {
1057 union {
1058 uint128_t q[16];
1059 uint64_t d[32];
1060 uint32_t s[32];
1061 } v;
1062 uint32_t fpsr;
1063 uint32_t fpcr;
1064};
1065typedef struct arm_neon_saved_state32 arm_neon_saved_state32_t;
1066
1067#define ARM_NEON_SAVED_STATE32_COUNT ((mach_msg_type_number_t) \
1068 (sizeof (arm_neon_saved_state32_t)/sizeof(unsigned int)))
1069
1070struct arm_neon_saved_state64 {
1071 union {
1072 uint128_t q[32];
1073 uint64x2_t d[32];
1074 uint32x4_t s[32];
1075 } v;
1076 uint32_t fpsr;
1077 uint32_t fpcr;
1078};
1079typedef struct arm_neon_saved_state64 arm_neon_saved_state64_t;
1080
1081#define ARM_NEON_SAVED_STATE64_COUNT ((mach_msg_type_number_t) \
1082 (sizeof (arm_neon_saved_state64_t)/sizeof(unsigned int)))
1083
1084struct arm_neon_saved_state {
1085 arm_state_hdr_t nsh;
1086 union {
1087 struct arm_neon_saved_state32 ns_32;
1088 struct arm_neon_saved_state64 ns_64;
1089 } uns;
1090};
1091typedef struct arm_neon_saved_state arm_neon_saved_state_t;
1092#define ns_32 uns.ns_32
1093#define ns_64 uns.ns_64
1094
1095struct arm_kernel_neon_saved_state {
1096 uint64_t d[8];
1097 uint32_t fpcr;
1098};
1099typedef struct arm_kernel_neon_saved_state arm_kernel_neon_saved_state_t;
1100
1101static inline boolean_t
1102is_neon_saved_state32(const arm_neon_saved_state_t *state)
1103{
1104 return state->nsh.flavor == ARM_NEON_SAVED_STATE32;
1105}
1106
1107static inline boolean_t
1108is_neon_saved_state64(const arm_neon_saved_state_t *state)
1109{
1110 return state->nsh.flavor == ARM_NEON_SAVED_STATE64;
1111}
1112
1113static inline arm_neon_saved_state32_t *
1114neon_state32(arm_neon_saved_state_t *state)
1115{
1116 return &state->ns_32;
1117}
1118
1119static inline arm_neon_saved_state64_t *
1120neon_state64(arm_neon_saved_state_t *state)
1121{
1122 return &state->ns_64;
1123}
1124
1125
1126#if HAS_ARM_FEAT_SME
1127
1128
1129struct arm_sme_saved_state;
1130typedef struct arm_sme_saved_state arm_sme_saved_state_t;
1131
1132#if !__has_ptrcheck
1133typedef struct {
1134 uint8_t zt0[64];
1135 uint8_t __z_p_za[];
1136} arm_sme_context_t;
1137
1138struct arm_sme_saved_state {
1139 arm_state_hdr_t hdr;
1140 uint64_t svcr;
1141 uint16_t svl_b;
1142 arm_sme_context_t context;
1143};
1144
1145static inline size_t
1146arm_sme_z_size(uint16_t svl_b)
1147{
1148 return 32 * svl_b;
1149}
1150
1151static inline size_t
1152arm_sme_p_size(uint16_t svl_b)
1153{
1154 return 2 * svl_b;
1155}
1156
1157static inline size_t
1158arm_sme_za_size(uint16_t svl_b)
1159{
1160 return svl_b * svl_b;
1161}
1162
1163static inline mach_msg_type_number_t
1164arm_sme_saved_state_count(uint16_t svl_b)
1165{
1166 assert(svl_b % 16 == 0);
1167 size_t size = sizeof(arm_sme_saved_state_t) +
1168 arm_sme_z_size(svl_b) +
1169 arm_sme_p_size(svl_b) +
1170 arm_sme_za_size(svl_b);
1171 return (mach_msg_type_number_t)(size / sizeof(unsigned int));
1172}
1173
1174static inline uint8_t *
1175arm_sme_z(arm_sme_context_t *ss)
1176{
1177 return ss->__z_p_za;
1178}
1179
1180static inline const uint8_t *
1181const_arm_sme_z(const arm_sme_context_t *ss)
1182{
1183 return ss->__z_p_za;
1184}
1185
1186static inline uint8_t *
1187arm_sme_p(arm_sme_context_t *ss, uint16_t svl_b)
1188{
1189 return ss->__z_p_za + arm_sme_z_size(svl_b);
1190}
1191
1192static inline const uint8_t *
1193const_arm_sme_p(const arm_sme_context_t *ss, uint16_t svl_b)
1194{
1195 return ss->__z_p_za + arm_sme_z_size(svl_b);
1196}
1197
1198static inline uint8_t *
1199arm_sme_za(arm_sme_context_t *ss, uint16_t svl_b)
1200{
1201 return ss->__z_p_za + arm_sme_z_size(svl_b) + arm_sme_p_size(svl_b);
1202}
1203
1204static inline const uint8_t *
1205const_arm_sme_za(const arm_sme_context_t *ss, uint16_t svl_b)
1206{
1207 return ss->__z_p_za + arm_sme_z_size(svl_b) + arm_sme_p_size(svl_b);
1208}
1209
1210#endif /* !__has_ptrcheck */
1211#endif /* HAS_ARM_FEAT_SME */
1212
1213/*
1214 * Aggregated context
1215 */
1216
1217struct arm_context {
1218 struct arm_saved_state ss;
1219 struct arm_neon_saved_state ns;
1220};
1221typedef struct arm_context arm_context_t;
1222
1223struct arm_kernel_context {
1224 struct arm_kernel_saved_state ss;
1225 struct arm_kernel_neon_saved_state ns;
1226};
1227typedef struct arm_kernel_context arm_kernel_context_t;
1228
1229extern void saved_state_to_thread_state64(const arm_saved_state_t*, arm_thread_state64_t*);
1230extern void thread_state64_to_saved_state(const arm_thread_state64_t*, arm_saved_state_t*);
1231
1232#else /* defined(__arm64__) */
1233#error Unknown arch
1234#endif /* defined(__arm64__) */
1235
1236extern void saved_state_to_thread_state32(const arm_saved_state_t*, arm_thread_state32_t*);
1237extern void thread_state32_to_saved_state(const arm_thread_state32_t*, arm_saved_state_t*);
1238
1239#endif /* XNU_KERNEL_PRIVATE */
1240
1241#endif /* defined (__arm__) || defined (__arm64__) */
1242
1243#endif /* _ARM_THREAD_STATUS_H_ */