Implementation sources

osfmk/arm64/sleh.c

xnu source @ f6217f8 2025-10-16 · 2953 lines · view on GitHub
1/* 2 * Copyright (c) 2012-2023 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29#include <arm/caches_internal.h> 30#include <arm/cpu_data.h> 31#include <arm/cpu_data_internal.h> 32#include <arm/misc_protos.h> 33#include <arm/thread.h> 34#include <arm/rtclock.h> 35#include <arm/trap_internal.h> /* for IS_ARM_GDB_TRAP() et al */ 36#include <arm64/proc_reg.h> 37#include <arm64/machine_machdep.h> 38#include <arm64/monotonic.h> 39#include <arm64/instructions.h> 40 41#include <kern/debug.h> 42#include <kern/exc_guard.h> 43#include <kern/restartable.h> 44#include <kern/socd_client.h> 45#include <kern/task.h> 46#include <kern/thread.h> 47#include <kern/zalloc_internal.h> 48#include <mach/exception.h> 49#include <mach/arm/traps.h> 50#include <mach/vm_types.h> 51#include <mach/machine/thread_status.h> 52 53#include <machine/atomic.h> 54#include <machine/limits.h> 55 56#include <pexpert/arm/protos.h> 57#include <pexpert/arm64/apple_arm64_cpu.h> 58#include <pexpert/arm64/apple_arm64_regs.h> 59#include <pexpert/arm64/board_config.h> 60 61#include <vm/vm_page.h> 62#include <vm/pmap.h> 63#include <vm/vm_fault.h> 64#include <vm/vm_kern.h> 65#include <vm/vm_map_xnu.h> 66 67#include <sys/errno.h> 68#include <sys/kdebug.h> 69#include <sys/code_signing.h> 70#include <sys/reason.h> 71#include <kperf/kperf.h> 72 73#include <kern/policy_internal.h> 74#if CONFIG_TELEMETRY 75#include <kern/telemetry.h> 76#include <kern/trap_telemetry.h> 77#endif 78 79#include <prng/entropy.h> 80 81 82 83 84#include <arm64/platform_error_handler.h> 85 86#if KASAN_TBI 87#include <san/kasan.h> 88#endif /* KASAN_TBI */ 89 90#if CONFIG_UBSAN_MINIMAL 91#include <san/ubsan_minimal.h> 92#endif 93 94 95 96#ifndef __arm64__ 97#error Should only be compiling for arm64. 98#endif 99 100#if DEBUG || DEVELOPMENT 101#define HAS_TELEMETRY_KERNEL_BRK 1 102#endif 103 104 105#define TEST_CONTEXT32_SANITY(context) \ 106 (context->ss.ash.flavor == ARM_SAVED_STATE32 && context->ss.ash.count == ARM_SAVED_STATE32_COUNT && \ 107 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE32 && context->ns.nsh.count == ARM_NEON_SAVED_STATE32_COUNT) 108 109#define TEST_CONTEXT64_SANITY(context) \ 110 (context->ss.ash.flavor == ARM_SAVED_STATE64 && context->ss.ash.count == ARM_SAVED_STATE64_COUNT && \ 111 context->ns.nsh.flavor == ARM_NEON_SAVED_STATE64 && context->ns.nsh.count == ARM_NEON_SAVED_STATE64_COUNT) 112 113#define ASSERT_CONTEXT_SANITY(context) \ 114 assert(TEST_CONTEXT32_SANITY(context) || TEST_CONTEXT64_SANITY(context)) 115 116 117#define COPYIN(src, dst, size) \ 118 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ 119 copyin_kern(src, dst, size) : \ 120 copyin(src, dst, size) 121 122#define COPYOUT(src, dst, size) \ 123 (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) ? \ 124 copyout_kern(src, dst, size) : \ 125 copyout(src, dst, size) 126 127// Below is for concatenating a string param to a string literal 128#define STR1(x) #x 129#define STR(x) STR1(x) 130 131#define ARM64_KDBG_CODE_KERNEL (0 << 8) 132#define ARM64_KDBG_CODE_USER (1 << 8) 133#define ARM64_KDBG_CODE_GUEST (2 << 8) 134 135_Static_assert(ARM64_KDBG_CODE_GUEST <= KDBG_CODE_MAX, "arm64 KDBG trace codes out of range"); 136_Static_assert(ARM64_KDBG_CODE_GUEST <= UINT16_MAX, "arm64 KDBG trace codes out of range"); 137 138void panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) __abortlike; 139 140void sleh_synchronous_sp1(arm_context_t *, uint64_t, vm_offset_t) __abortlike; 141void sleh_synchronous(arm_context_t *, uint64_t, vm_offset_t, bool); 142 143 144 145void sleh_irq(arm_saved_state_t *); 146void sleh_fiq(arm_saved_state_t *); 147void sleh_serror(arm_context_t *context, uint64_t esr, vm_offset_t far); 148void sleh_invalid_stack(arm_context_t *context, uint64_t esr, vm_offset_t far) __dead2; 149 150static void sleh_interrupt_handler_prologue(arm_saved_state_t *, unsigned int type); 151static void sleh_interrupt_handler_epilogue(void); 152 153static void handle_svc(arm_saved_state_t *); 154static void handle_mach_absolute_time_trap(arm_saved_state_t *); 155static void handle_mach_continuous_time_trap(arm_saved_state_t *); 156 157static void handle_msr_trap(arm_saved_state_t *state, uint64_t esr); 158#if __has_feature(ptrauth_calls) 159static void handle_pac_fail(arm_saved_state_t *state, uint64_t esr) __dead2; 160static inline uint64_t fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to); 161#endif 162static void handle_bti_fail(arm_saved_state_t *state, uint64_t esr); 163extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool); 164 165static void handle_uncategorized(arm_saved_state_t *); 166 167static void handle_kernel_breakpoint(arm_saved_state_t *, uint64_t); 168 169static void handle_user_breakpoint(arm_saved_state_t *, uint64_t) __dead2; 170 171typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *); 172static void inspect_instruction_abort(uint32_t, fault_status_t *, vm_prot_t *); 173static void inspect_data_abort(uint32_t, fault_status_t *, vm_prot_t *); 174 175static int is_vm_fault(fault_status_t); 176static int is_translation_fault(fault_status_t); 177static int is_alignment_fault(fault_status_t); 178 179typedef void (*abort_handler_t)(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); 180static void handle_user_abort(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); 181static void handle_kernel_abort(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t); 182 183static void handle_pc_align(arm_saved_state_t *ss) __dead2; 184static void handle_sp_align(arm_saved_state_t *ss) __dead2; 185static void handle_sw_step_debug(arm_saved_state_t *ss) __dead2; 186static void handle_wf_trap(arm_saved_state_t *ss) __dead2; 187static void handle_fp_trap(arm_saved_state_t *ss, uint64_t esr) __dead2; 188#if HAS_ARM_FEAT_SME 189static void handle_sme_trap(arm_saved_state_t *state, uint64_t esr); 190#endif /* HAS_ARM_FEAT_SME */ 191 192static void handle_watchpoint(vm_offset_t fault_addr) __dead2; 193 194static void handle_abort(arm_saved_state_t *, uint64_t, vm_offset_t, abort_inspector_t, abort_handler_t, expected_fault_handler_t); 195 196static void handle_user_trapped_instruction32(arm_saved_state_t *, uint64_t esr) __dead2; 197 198static void handle_simd_trap(arm_saved_state_t *, uint64_t esr) __dead2; 199 200extern void current_cached_proc_cred_update(void); 201void mach_syscall_trace_exit(unsigned int retval, unsigned int call_number); 202 203struct proc; 204 205typedef uint32_t arm64_instr_t; 206 207extern void 208unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc); 209 210extern void 211mach_syscall(struct arm_saved_state*); 212 213#if CONFIG_SPTM 214bool sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr, uint64_t elr, uint64_t far, uint64_t spsr); 215#endif /* CONFIG_SPTM */ 216 217#if CONFIG_DTRACE 218extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs); 219extern boolean_t dtrace_tally_fault(user_addr_t); 220 221/* 222 * Traps for userland processing. Can't include bsd/sys/fasttrap_isa.h, so copy 223 * and paste the trap instructions 224 * over from that file. Need to keep these in sync! 225 */ 226#define FASTTRAP_ARM32_INSTR 0xe7ffdefc 227#define FASTTRAP_THUMB32_INSTR 0xdefc 228#define FASTTRAP_ARM64_INSTR 0xe7eeee7e 229 230#define FASTTRAP_ARM32_RET_INSTR 0xe7ffdefb 231#define FASTTRAP_THUMB32_RET_INSTR 0xdefb 232#define FASTTRAP_ARM64_RET_INSTR 0xe7eeee7d 233 234/* See <rdar://problem/4613924> */ 235perfCallback tempDTraceTrapHook = NULL; /* Pointer to DTrace fbt trap hook routine */ 236#endif 237 238 239 240extern void arm64_thread_exception_return(void) __dead2; 241 242#if defined(APPLETYPHOON) 243#define CPU_NAME "Typhoon" 244#elif defined(APPLETWISTER) 245#define CPU_NAME "Twister" 246#elif defined(APPLEHURRICANE) 247#define CPU_NAME "Hurricane" 248#elif defined(APPLELIGHTNING) 249#define CPU_NAME "Lightning" 250#elif defined(APPLEEVEREST) 251#define CPU_NAME "Everest" 252#elif defined(APPLEH16) 253#define CPU_NAME "AppleH16" 254#else 255#define CPU_NAME "Unknown" 256#endif 257 258#if (CONFIG_KERNEL_INTEGRITY && defined(KERNEL_INTEGRITY_WT)) 259#define ESR_WT_SERROR(esr) (((esr) & 0xffffff00) == 0xbf575400) 260#define ESR_WT_REASON(esr) ((esr) & 0xff) 261 262#define WT_REASON_NONE 0 263#define WT_REASON_INTEGRITY_FAIL 1 264#define WT_REASON_BAD_SYSCALL 2 265#define WT_REASON_NOT_LOCKED 3 266#define WT_REASON_ALREADY_LOCKED 4 267#define WT_REASON_SW_REQ 5 268#define WT_REASON_PT_INVALID 6 269#define WT_REASON_PT_VIOLATION 7 270#define WT_REASON_REG_VIOLATION 8 271#endif 272 273#if defined(HAS_IPI) 274void cpu_signal_handler(void); 275extern unsigned int gFastIPI; 276#endif /* defined(HAS_IPI) */ 277 278static arm_saved_state64_t *original_faulting_state = NULL; 279 280/* 281 * A self-restrict mode describes which (if any, or several) special permissive 282 * modes are active at the time of a fault. This, in part, determines how the 283 * fault will be handled. 284 */ 285__options_closed_decl(self_restrict_mode_t, unsigned int, { 286 /* None of the special modes are active. */ 287 SELF_RESTRICT_NONE = 0U, 288 289 /* 290 * Any of the other more specific modes, this should be active if any other 291 * mode is active. 292 */ 293 SELF_RESTRICT_ANY = (1U << 0), 294 295 /* Reserved */ 296 297 /* Reserved */ 298}); 299 300 301TUNABLE(bool, fp_exceptions_enabled, "-fp_exceptions", false); 302 303extern const vm_map_address_t physmap_base; 304extern const vm_map_address_t physmap_end; 305extern vm_offset_t static_memory_end; 306 307/* 308 * Fault copyio_recovery_entry in copyin/copyout routines. 309 * 310 * Offets are expressed in bytes from &copy_recovery_table 311 */ 312struct copyio_recovery_entry { 313 ptrdiff_t cre_start; 314 ptrdiff_t cre_end; 315 ptrdiff_t cre_recovery; 316}; 317 318extern struct copyio_recovery_entry copyio_recover_table[]; 319extern struct copyio_recovery_entry copyio_recover_table_end[]; 320 321static inline ptrdiff_t 322copyio_recovery_offset(uintptr_t addr) 323{ 324 return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table); 325} 326 327#if !HAS_APPLE_PAC 328static inline uintptr_t 329copyio_recovery_addr(ptrdiff_t offset) 330{ 331 return (uintptr_t)copyio_recover_table + (uintptr_t)offset; 332} 333#endif 334 335static inline struct copyio_recovery_entry * 336find_copyio_recovery_entry(uint64_t pc) 337{ 338 ptrdiff_t offset = copyio_recovery_offset(pc); 339 struct copyio_recovery_entry *e; 340 341 for (e = copyio_recover_table; e < copyio_recover_table_end; e++) { 342 if (offset >= e->cre_start && offset < e->cre_end) { 343 return e; 344 } 345 } 346 347 return NULL; 348} 349 350static inline int 351is_vm_fault(fault_status_t status) 352{ 353 switch (status) { 354 case FSC_TRANSLATION_FAULT_L0: 355 case FSC_TRANSLATION_FAULT_L1: 356 case FSC_TRANSLATION_FAULT_L2: 357 case FSC_TRANSLATION_FAULT_L3: 358 case FSC_ACCESS_FLAG_FAULT_L1: 359 case FSC_ACCESS_FLAG_FAULT_L2: 360 case FSC_ACCESS_FLAG_FAULT_L3: 361 case FSC_PERMISSION_FAULT_L1: 362 case FSC_PERMISSION_FAULT_L2: 363 case FSC_PERMISSION_FAULT_L3: 364 return TRUE; 365 default: 366 return FALSE; 367 } 368} 369 370static inline int 371is_translation_fault(fault_status_t status) 372{ 373 switch (status) { 374 case FSC_TRANSLATION_FAULT_L0: 375 case FSC_TRANSLATION_FAULT_L1: 376 case FSC_TRANSLATION_FAULT_L2: 377 case FSC_TRANSLATION_FAULT_L3: 378 return TRUE; 379 default: 380 return FALSE; 381 } 382} 383 384static inline int 385is_permission_fault(fault_status_t status) 386{ 387 switch (status) { 388 case FSC_PERMISSION_FAULT_L1: 389 case FSC_PERMISSION_FAULT_L2: 390 case FSC_PERMISSION_FAULT_L3: 391 return TRUE; 392 default: 393 return FALSE; 394 } 395} 396 397static inline int 398is_alignment_fault(fault_status_t status) 399{ 400 return status == FSC_ALIGNMENT_FAULT; 401} 402 403static inline int 404is_parity_error(fault_status_t status) 405{ 406 switch (status) { 407#if defined(ARM64_BOARD_CONFIG_T6020) 408 /* 409 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented, 410 * FSC_SYNC_PARITY_X can be reported for data and instruction aborts 411 * and should be interpreted as FSC_SYNC_EXT_ABORT_x 412 */ 413#else 414 /* 415 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is 416 * only applicable to AArch32 HSR register. Can this be removed? 417 */ 418 case FSC_ASYNC_PARITY: 419 case FSC_SYNC_PARITY: 420 case FSC_SYNC_PARITY_TT_L1: 421 case FSC_SYNC_PARITY_TT_L2: 422 case FSC_SYNC_PARITY_TT_L3: 423 return TRUE; 424#endif 425 default: 426 return FALSE; 427 } 428} 429 430static inline int 431is_sync_external_abort(fault_status_t status) 432{ 433 switch (status) { 434#if defined(ARM64_BOARD_CONFIG_T6020) 435 /* 436 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented, 437 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts 438 * and should be interpreted as FSC_SYNC_EXT_ABORT_x 439 */ 440 case FSC_SYNC_PARITY: 441#endif /* defined(ARM64_BOARD_CONFIG_T6020) */ 442 case FSC_SYNC_EXT_ABORT: 443 return TRUE; 444 default: 445 return FALSE; 446 } 447} 448 449static inline int 450is_table_walk_error(fault_status_t status) 451{ 452 switch (status) { 453 case FSC_SYNC_EXT_ABORT_TT_L1: 454 case FSC_SYNC_EXT_ABORT_TT_L2: 455 case FSC_SYNC_EXT_ABORT_TT_L3: 456#if defined(ARM64_BOARD_CONFIG_T6020) 457 /* 458 * H14 Erratum(rdar://61553243): Despite having FEAT_RAS implemented, 459 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts 460 * and should be interpreted as FSC_SYNC_EXT_ABORT_x 461 */ 462 case FSC_SYNC_PARITY_TT_L1: 463 case FSC_SYNC_PARITY_TT_L2: 464 case FSC_SYNC_PARITY_TT_L3: 465#endif /* defined(ARM64_BOARD_CONFIG_T6020) */ 466 return TRUE; 467 default: 468 return FALSE; 469 } 470} 471 472 473 474 475static inline int 476is_servicible_fault(fault_status_t status, uint64_t esr) 477{ 478#pragma unused(esr) 479 return is_vm_fault(status); 480} 481 482__dead2 __unused 483static void 484arm64_implementation_specific_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far) 485{ 486#pragma unused (state, esr, far) 487 panic_plain("Unhandled implementation specific error\n"); 488} 489 490#if CONFIG_KERNEL_INTEGRITY 491#pragma clang diagnostic push 492#pragma clang diagnostic ignored "-Wunused-parameter" 493static void 494kernel_integrity_error_handler(uint64_t esr, vm_offset_t far) 495{ 496#if defined(KERNEL_INTEGRITY_WT) 497#if (DEVELOPMENT || DEBUG) 498 if (ESR_WT_SERROR(esr)) { 499 switch (ESR_WT_REASON(esr)) { 500 case WT_REASON_INTEGRITY_FAIL: 501 panic_plain("Kernel integrity, violation in frame 0x%016lx.", far); 502 case WT_REASON_BAD_SYSCALL: 503 panic_plain("Kernel integrity, bad syscall."); 504 case WT_REASON_NOT_LOCKED: 505 panic_plain("Kernel integrity, not locked."); 506 case WT_REASON_ALREADY_LOCKED: 507 panic_plain("Kernel integrity, already locked."); 508 case WT_REASON_SW_REQ: 509 panic_plain("Kernel integrity, software request."); 510 case WT_REASON_PT_INVALID: 511 panic_plain("Kernel integrity, encountered invalid TTE/PTE while " 512 "walking 0x%016lx.", far); 513 case WT_REASON_PT_VIOLATION: 514 panic_plain("Kernel integrity, violation in mapping 0x%016lx.", 515 far); 516 case WT_REASON_REG_VIOLATION: 517 panic_plain("Kernel integrity, violation in system register %d.", 518 (unsigned) far); 519 default: 520 panic_plain("Kernel integrity, unknown (esr=0x%08llx).", esr); 521 } 522 } 523#else 524 if (ESR_WT_SERROR(esr)) { 525 panic_plain("SError esr: 0x%08llx far: 0x%016lx.", esr, far); 526 } 527#endif 528#endif 529} 530#pragma clang diagnostic pop 531#endif 532 533static void 534arm64_platform_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far, platform_error_source_t source) 535{ 536#if CONFIG_KERNEL_INTEGRITY 537 kernel_integrity_error_handler(esr, far); 538#endif 539 540 (void)source; 541 cpu_data_t *cdp = getCpuDatap(); 542 543 if (PE_handle_platform_error(far)) { 544 return; 545 } else if (cdp->platform_error_handler != NULL) { 546 cdp->platform_error_handler(cdp->cpu_id, far); 547 } else { 548 arm64_implementation_specific_error(state, esr, far); 549 } 550} 551 552 553void 554panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss) 555{ 556 boolean_t ss_valid; 557 558 ss_valid = is_saved_state64(ss); 559 arm_saved_state64_t *state = saved_state64(ss); 560 561 os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst); 562 563 // rdar://80659177 564 // Read SoCD tracepoints up to twice — once the first time we call panic and 565 // another time if we encounter a nested panic after that. 566 static int twice = 2; 567 if (twice > 0) { 568 twice--; 569 SOCD_TRACE_XNU(KERNEL_STATE_PANIC, 570 SOCD_TRACE_MODE_STICKY_TRACEPOINT, 571 ADDR(state->pc), 572 PACK_LSB(VALUE(state->lr), VALUE(ss_valid)), 573 PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)), 574 VALUE(state->far)); 575 } 576 577 578 579 panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n" 580 "\t x0: 0x%016llx x1: 0x%016llx x2: 0x%016llx x3: 0x%016llx\n" 581 "\t x4: 0x%016llx x5: 0x%016llx x6: 0x%016llx x7: 0x%016llx\n" 582 "\t x8: 0x%016llx x9: 0x%016llx x10: 0x%016llx x11: 0x%016llx\n" 583 "\t x12: 0x%016llx x13: 0x%016llx x14: 0x%016llx x15: 0x%016llx\n" 584 "\t x16: 0x%016llx x17: 0x%016llx x18: 0x%016llx x19: 0x%016llx\n" 585 "\t x20: 0x%016llx x21: 0x%016llx x22: 0x%016llx x23: 0x%016llx\n" 586 "\t x24: 0x%016llx x25: 0x%016llx x26: 0x%016llx x27: 0x%016llx\n" 587 "\t x28: 0x%016llx fp: 0x%016llx lr: 0x%016llx sp: 0x%016llx\n" 588 "\t pc: 0x%016llx cpsr: 0x%08x esr: 0x%016llx far: 0x%016llx\n", 589 msg, state->pc, state->lr, ss, (ss_valid ? "" : " INVALID"), 590 state->x[0], state->x[1], state->x[2], state->x[3], 591 state->x[4], state->x[5], state->x[6], state->x[7], 592 state->x[8], state->x[9], state->x[10], state->x[11], 593 state->x[12], state->x[13], state->x[14], state->x[15], 594 state->x[16], state->x[17], state->x[18], state->x[19], 595 state->x[20], state->x[21], state->x[22], state->x[23], 596 state->x[24], state->x[25], state->x[26], state->x[27], 597 state->x[28], state->fp, state->lr, state->sp, 598 state->pc, state->cpsr, state->esr, state->far); 599} 600 601void 602sleh_synchronous_sp1(arm_context_t *context, uint64_t esr, vm_offset_t far __unused) 603{ 604 esr_exception_class_t class = ESR_EC(esr); 605 arm_saved_state_t * state = &context->ss; 606 607 switch (class) { 608 case ESR_EC_UNCATEGORIZED: 609 { 610#if (DEVELOPMENT || DEBUG) 611 uint32_t instr = *((uint32_t*)get_saved_state_pc(state)); 612 if (IS_ARM_GDB_TRAP(instr)) { 613 DebuggerCall(EXC_BREAKPOINT, state); 614 } 615 OS_FALLTHROUGH; // panic if we return from the debugger 616#else 617 panic_with_thread_kernel_state("Unexpected debugger trap while SP1 selected", state); 618#endif /* (DEVELOPMENT || DEBUG) */ 619 } 620 default: 621 panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state); 622 } 623} 624 625 626__attribute__((noreturn)) 627void 628thread_exception_return() 629{ 630 thread_t thread = current_thread(); 631 if (thread->machine.exception_trace_code != 0) { 632 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 633 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0); 634 thread->machine.exception_trace_code = 0; 635 } 636 637 638#if KASAN_TBI 639 kasan_unpoison_curstack(true); 640#endif /* KASAN_TBI */ 641 arm64_thread_exception_return(); 642 __builtin_unreachable(); 643} 644 645/* 646 * check whether task vtimers are running and set thread and CPU BSD AST 647 * 648 * must be called with interrupts masked so updates of fields are atomic 649 * must be emitted inline to avoid generating an FBT probe on the exception path 650 * 651 */ 652__attribute__((__always_inline__)) 653static inline void 654task_vtimer_check(thread_t thread) 655{ 656 task_t task = get_threadtask_early(thread); 657 658 if (__improbable(task != NULL && task->vtimers)) { 659 thread_ast_set(thread, AST_BSD); 660 thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD; 661 } 662} 663 664#if MACH_ASSERT 665/** 666 * A version of get_preemption_level() that works in early boot. 667 * 668 * If an exception is raised in early boot before the initial thread has been 669 * set up, then calling get_preemption_level() in the SLEH will trigger an 670 * infinitely-recursing exception. This function handles this edge case. 671 */ 672static inline int 673sleh_get_preemption_level(void) 674{ 675 if (__improbable(current_thread() == NULL)) { 676 return 0; 677 } 678 return get_preemption_level(); 679} 680#endif // MACH_ASSERT 681 682static inline bool 683is_platform_error(uint64_t esr) 684{ 685 esr_exception_class_t class = ESR_EC(esr); 686 uint32_t iss = ESR_ISS(esr); 687 fault_status_t fault_code; 688 689 if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) { 690 fault_code = ISS_DA_FSC(iss); 691 } else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) { 692 fault_code = ISS_IA_FSC(iss); 693 } else { 694 return false; 695 } 696 697 return is_parity_error(fault_code) || is_sync_external_abort(fault_code) || 698 is_table_walk_error(fault_code); 699} 700 701void 702sleh_synchronous(arm_context_t *context, uint64_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown) 703{ 704 esr_exception_class_t class = ESR_EC(esr); 705 arm_saved_state_t * state = &context->ss; 706 thread_t thread = current_thread(); 707#if MACH_ASSERT 708 int preemption_level = sleh_get_preemption_level(); 709#endif 710 expected_fault_handler_t expected_fault_handler = NULL; 711#ifdef CONFIG_XNUPOST 712 expected_fault_handler_t saved_expected_fault_handler = NULL; 713 uintptr_t saved_expected_fault_addr = 0; 714 uintptr_t saved_expected_fault_pc = 0; 715#endif /* CONFIG_XNUPOST */ 716 717 ASSERT_CONTEXT_SANITY(context); 718 719 task_vtimer_check(thread); 720 721#if CONFIG_DTRACE 722 /* 723 * Handle kernel DTrace probes as early as possible to minimize the likelihood 724 * that this path will itself trigger a DTrace probe, which would lead to infinite 725 * probe recursion. 726 */ 727 if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook && 728 (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) { 729#if CONFIG_SPTM 730 if (__improbable(did_initiate_panic_lockdown)) { 731 panic("Unexpectedly initiated lockdown for DTrace probe?"); 732 } 733#endif 734 return; 735 } 736#endif 737 bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state)); 738 739#if CONFIG_SPTM 740 // Lockdown should only be initiated for kernel exceptions 741 assert(!(is_user && did_initiate_panic_lockdown)); 742#endif /* CONFIG_SPTM */ 743 744 /* 745 * Use KERNEL_DEBUG_CONSTANT_IST here to avoid producing tracepoints 746 * that would disclose the behavior of PT_DENY_ATTACH processes. 747 */ 748 if (is_user) { 749 /* Sanitize FAR (but only if the exception was taken from userspace) */ 750 switch (class) { 751 case ESR_EC_IABORT_EL1: 752 case ESR_EC_IABORT_EL0: 753 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */ 754 if (ISS_IA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) { 755 saved_state64(state)->far = 0; 756 } 757 break; 758 case ESR_EC_DABORT_EL1: 759 case ESR_EC_DABORT_EL0: 760 /* If this is a SEA, since we can't trust FnV, just clear FAR from the save area. */ 761 if (ISS_DA_FSC(ESR_ISS(esr)) == FSC_SYNC_EXT_ABORT) { 762 saved_state64(state)->far = 0; 763 } 764 break; 765 case ESR_EC_WATCHPT_MATCH_EL1: 766 case ESR_EC_WATCHPT_MATCH_EL0: 767 case ESR_EC_PC_ALIGN: 768 break; /* FAR_ELx is valid */ 769 default: 770 saved_state64(state)->far = 0; 771 break; 772 } 773 774 thread->machine.exception_trace_code = (uint16_t)(ARM64_KDBG_CODE_USER | class); 775 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 776 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_START, 777 esr, far, get_saved_state_pc(state), 0, 0); 778 } else { 779 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 780 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_START, 781 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0); 782 } 783 784 if (__improbable(ESR_INSTR_IS_2BYTES(esr))) { 785 /* 786 * We no longer support 32-bit, which means no 2-byte 787 * instructions. 788 */ 789 if (is_user) { 790 panic("Exception on 2-byte instruction, " 791 "context=%p, esr=%#llx, far=%p", 792 context, esr, (void *)far); 793 } else { 794 panic_with_thread_kernel_state("Exception on 2-byte instruction", state); 795 } 796 } 797 798#ifdef CONFIG_XNUPOST 799 if (thread->machine.expected_fault_handler != NULL) { 800 bool matching_fault_pc = false; 801 saved_expected_fault_handler = thread->machine.expected_fault_handler; 802 saved_expected_fault_addr = thread->machine.expected_fault_addr; 803 saved_expected_fault_pc = thread->machine.expected_fault_pc; 804 805 thread->machine.expected_fault_handler = NULL; 806 thread->machine.expected_fault_addr = 0; 807 thread->machine.expected_fault_pc = 0; 808 809#if __has_feature(ptrauth_calls) 810 /* 811 * Compare only the bits of PC which make up the virtual address. 812 * This ignores the upper bits, which may have been corrupted by HW in 813 * platform dependent ways to signal pointer authentication fault. 814 */ 815 uint64_t fault_addr_mask = fault_addr_bitmask(0, 64 - T1SZ_BOOT - 1); 816 uint64_t masked_expected_pc = saved_expected_fault_pc & fault_addr_mask; 817 uint64_t masked_saved_pc = get_saved_state_pc(state) & fault_addr_mask; 818 matching_fault_pc = masked_expected_pc == masked_saved_pc; 819#else 820 matching_fault_pc = 821 (saved_expected_fault_pc == get_saved_state_pc(state)); 822#endif /* ptrauth_call */ 823 if (saved_expected_fault_addr == far || 824 matching_fault_pc) { 825 expected_fault_handler = saved_expected_fault_handler; 826 } 827 } 828#endif /* CONFIG_XNUPOST */ 829 830 if (__improbable(is_platform_error(esr))) { 831 /* 832 * Must gather error info in platform error handler before 833 * thread is preempted to another core/cluster to guarantee 834 * accurate error details 835 */ 836 837 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_SYNC); 838#if CONFIG_SPTM 839 if (__improbable(did_initiate_panic_lockdown)) { 840 panic("Panic lockdown initiated for platform error"); 841 } 842#endif 843 return; 844 } 845 846 if (is_user && class == ESR_EC_DABORT_EL0) { 847 thread_reset_pcs_will_fault(thread); 848 } 849 850#if CONFIG_SPTM 851 if (__improbable(did_initiate_panic_lockdown && current_thread() != NULL)) { 852 /* 853 * If we initiated panic lockdown, we must disable preemption before 854 * enabling interrupts. While unlikely, preempting the panicked thread 855 * after lockdown has occurred may hang the system if all cores end up 856 * blocked while attempting to return to user space. 857 */ 858 disable_preemption(); 859 } 860#endif /* CONFIG_SPTM */ 861 862 /* Inherit the interrupt masks from previous context */ 863 if (SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state))) { 864 ml_set_interrupts_enabled(TRUE); 865 } 866 867 868 switch (class) { 869 case ESR_EC_SVC_64: 870 if (!is_saved_state64(state) || !is_user) { 871 panic("Invalid SVC_64 context"); 872 } 873 874 handle_svc(state); 875 break; 876 877 case ESR_EC_DABORT_EL0: 878 handle_abort(state, esr, far, inspect_data_abort, handle_user_abort, expected_fault_handler); 879 break; 880 881 case ESR_EC_MSR_TRAP: 882 handle_msr_trap(state, esr); 883 break; 884/** 885 * Some APPLEVIRTUALPLATFORM targets do not specify armv8.6, but it's still possible for 886 * them to be hosted by a host that implements ARM_FPAC. There's no way for such a host 887 * to disable it or trap it without substantial performance penalty. Therefore, the FPAC 888 * handler here needs to be built into the guest kernels to prevent the exception to fall 889 * through. 890 */ 891#if __has_feature(ptrauth_calls) 892 case ESR_EC_PAC_FAIL: 893#ifdef CONFIG_XNUPOST 894 if (expected_fault_handler != NULL && expected_fault_handler(state)) { 895 break; 896 } 897#endif /* CONFIG_XNUPOST */ 898 handle_pac_fail(state, esr); 899 __builtin_unreachable(); 900 901#endif /* __has_feature(ptrauth_calls) */ 902 903#if HAS_ARM_FEAT_SME 904 case ESR_EC_SME: 905 handle_sme_trap(state, esr); 906 break; 907#endif /* HAS_ARM_FEAT_SME */ 908 909 case ESR_EC_IABORT_EL0: 910 handle_abort(state, esr, far, inspect_instruction_abort, handle_user_abort, expected_fault_handler); 911 break; 912 913 case ESR_EC_IABORT_EL1: 914#ifdef CONFIG_XNUPOST 915 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) { 916 break; 917 } 918#endif /* CONFIG_XNUPOST */ 919 920 panic_with_thread_kernel_state("Kernel instruction fetch abort", state); 921 922 case ESR_EC_PC_ALIGN: 923 handle_pc_align(state); 924 __builtin_unreachable(); 925 926 case ESR_EC_DABORT_EL1: 927 handle_abort(state, esr, far, inspect_data_abort, handle_kernel_abort, expected_fault_handler); 928 break; 929 930 case ESR_EC_UNCATEGORIZED: 931 assert(!ESR_ISS(esr)); 932 933#if CONFIG_XNUPOST 934 if (!is_user && (expected_fault_handler != NULL) && expected_fault_handler(state)) { 935 /* 936 * The fault handler accepted the exception and handled it on its 937 * own. Don't trap to the debugger/panic. 938 */ 939 break; 940 } 941#endif /* CONFIG_XNUPOST */ 942 handle_uncategorized(&context->ss); 943 break; 944 945 case ESR_EC_SP_ALIGN: 946 handle_sp_align(state); 947 __builtin_unreachable(); 948 949 case ESR_EC_BKPT_AARCH32: 950 handle_user_breakpoint(state, esr); 951 __builtin_unreachable(); 952 953 case ESR_EC_BRK_AARCH64: 954#ifdef CONFIG_XNUPOST 955 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) { 956 break; 957 } 958#endif /* CONFIG_XNUPOST */ 959 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 960 handle_kernel_breakpoint(state, esr); 961 break; 962 } else { 963 handle_user_breakpoint(state, esr); 964 __builtin_unreachable(); 965 } 966 967 case ESR_EC_BKPT_REG_MATCH_EL0: 968 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { 969 handle_user_breakpoint(state, esr); 970 } 971 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p", 972 class, state, class, esr, (void *)far); 973 __builtin_unreachable(); 974 975 case ESR_EC_BKPT_REG_MATCH_EL1: 976 panic_with_thread_kernel_state("Hardware Breakpoint Debug exception from kernel. Panic (by design)", state); 977 __builtin_unreachable(); 978 979 case ESR_EC_SW_STEP_DEBUG_EL0: 980 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { 981 handle_sw_step_debug(state); 982 } 983 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p", 984 class, state, class, esr, (void *)far); 985 __builtin_unreachable(); 986 987 case ESR_EC_SW_STEP_DEBUG_EL1: 988 panic_with_thread_kernel_state("Software Step Debug exception from kernel. Panic (by design)", state); 989 __builtin_unreachable(); 990 991 case ESR_EC_WATCHPT_MATCH_EL0: 992 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { 993 handle_watchpoint(far); 994 } 995 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p", 996 class, state, class, esr, (void *)far); 997 __builtin_unreachable(); 998 999 case ESR_EC_WATCHPT_MATCH_EL1: 1000 /* 1001 * If we hit a watchpoint in kernel mode, probably in a copyin/copyout which we don't want to 1002 * abort. Turn off watchpoints and keep going; we'll turn them back on in return_from_exception.. 1003 */ 1004 if (FSC_DEBUG_FAULT == ISS_SSDE_FSC(esr)) { 1005 arm_debug_set(NULL); 1006 break; /* return to first level handler */ 1007 } 1008 panic("Unsupported Class %u event code. state=%p class=%u esr=%llu far=%p", 1009 class, state, class, esr, (void *)far); 1010 __builtin_unreachable(); 1011 1012 case ESR_EC_TRAP_SIMD_FP: 1013 handle_simd_trap(state, esr); 1014 __builtin_unreachable(); 1015 1016 case ESR_EC_ILLEGAL_INSTR_SET: 1017 panic("Illegal instruction set exception. state=%p class=%u esr=%llu far=%p spsr=0x%x", 1018 state, class, esr, (void *)far, get_saved_state_cpsr(state)); 1019 __builtin_unreachable(); 1020 1021 case ESR_EC_MCR_MRC_CP15_TRAP: 1022 case ESR_EC_MCRR_MRRC_CP15_TRAP: 1023 case ESR_EC_MCR_MRC_CP14_TRAP: 1024 case ESR_EC_LDC_STC_CP14_TRAP: 1025 case ESR_EC_MCRR_MRRC_CP14_TRAP: 1026 handle_user_trapped_instruction32(state, esr); 1027 __builtin_unreachable(); 1028 1029 case ESR_EC_WFI_WFE: 1030 // Use of WFI or WFE instruction when they have been disabled for EL0 1031 handle_wf_trap(state); 1032 __builtin_unreachable(); 1033 1034 case ESR_EC_FLOATING_POINT_64: 1035 handle_fp_trap(state, esr); 1036 __builtin_unreachable(); 1037 case ESR_EC_BTI_FAIL: 1038#ifdef CONFIG_XNUPOST 1039 if ((expected_fault_handler != NULL) && expected_fault_handler(state)) { 1040 break; 1041 } 1042#endif /* CONFIG_XNUPOST */ 1043 handle_bti_fail(state, esr); 1044 __builtin_unreachable(); 1045 1046 default: 1047 handle_uncategorized(state); 1048 } 1049 1050#ifdef CONFIG_XNUPOST 1051 if (saved_expected_fault_handler != NULL) { 1052 thread->machine.expected_fault_handler = saved_expected_fault_handler; 1053 thread->machine.expected_fault_addr = saved_expected_fault_addr; 1054 thread->machine.expected_fault_pc = saved_expected_fault_pc; 1055 } 1056#endif /* CONFIG_XNUPOST */ 1057 1058 if (is_user) { 1059 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 1060 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 1061 esr, far, get_saved_state_pc(state), 0, 0); 1062 thread->machine.exception_trace_code = 0; 1063 } else { 1064 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 1065 MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, ARM64_KDBG_CODE_KERNEL | class) | DBG_FUNC_END, 1066 esr, VM_KERNEL_ADDRHIDE(far), VM_KERNEL_UNSLIDE(get_saved_state_pc(state)), 0, 0); 1067 } 1068 1069#if CONFIG_SPTM 1070 if (__improbable(did_initiate_panic_lockdown)) { 1071#if CONFIG_XNUPOST 1072 bool can_recover = !!(expected_fault_handler); 1073#else 1074 bool can_recover = false; 1075#endif /* CONFIG_XNU_POST */ 1076 1077 if (can_recover) { 1078 /* 1079 * If we matched an exception handler, this was a simulated lockdown 1080 * and so we can recover. Re-enable preemption if we disabled it. 1081 */ 1082 if (current_thread() != NULL) { 1083 enable_preemption(); 1084 } 1085 } else { 1086 /* 1087 * fleh already triggered a lockdown but we, for whatever reason, 1088 * didn't end up finding a reason to panic. Catch all panic in this 1089 * case. 1090 * Note that the panic here has no security benefit as the system is 1091 * already hosed, this is merely for telemetry. 1092 */ 1093 panic_with_thread_kernel_state("Panic lockdown initiated", state); 1094 } 1095 } 1096#endif /* CONFIG_SPTM */ 1097 1098#if MACH_ASSERT 1099 if (preemption_level != sleh_get_preemption_level()) { 1100 panic("synchronous exception changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level()); 1101 } 1102#endif 1103 1104} 1105 1106/* 1107 * Uncategorized exceptions are a catch-all for general execution errors. 1108 * ARM64_TODO: For now, we assume this is for undefined instruction exceptions. 1109 */ 1110static void 1111handle_uncategorized(arm_saved_state_t *state) 1112{ 1113 exception_type_t exception = EXC_BAD_INSTRUCTION; 1114 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; 1115 mach_msg_type_number_t numcodes = 2; 1116 uint32_t instr = 0; 1117 1118 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 1119 1120#if CONFIG_DTRACE 1121 1122 if (PSR64_IS_USER64(get_saved_state_cpsr(state))) { 1123 /* 1124 * For a 64bit user process, we care about all 4 bytes of the 1125 * instr. 1126 */ 1127 if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) { 1128 if (dtrace_user_probe(state) == KERN_SUCCESS) { 1129 return; 1130 } 1131 } 1132 } else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) { 1133 /* 1134 * For a 32bit user process, we check for thumb mode, in 1135 * which case we only care about a 2 byte instruction length. 1136 * For non-thumb mode, we care about all 4 bytes of the instructin. 1137 */ 1138 if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) { 1139 if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) || 1140 ((uint16_t)instr == FASTTRAP_THUMB32_RET_INSTR)) { 1141 if (dtrace_user_probe(state) == KERN_SUCCESS) { 1142 return; 1143 } 1144 } 1145 } else { 1146 if ((instr == FASTTRAP_ARM32_INSTR) || 1147 (instr == FASTTRAP_ARM32_RET_INSTR)) { 1148 if (dtrace_user_probe(state) == KERN_SUCCESS) { 1149 return; 1150 } 1151 } 1152 } 1153 } 1154 1155#endif /* CONFIG_DTRACE */ 1156 1157 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 1158 if (IS_ARM_GDB_TRAP(instr)) { 1159 boolean_t interrupt_state; 1160 exception = EXC_BREAKPOINT; 1161 1162 interrupt_state = ml_set_interrupts_enabled(FALSE); 1163 1164 /* Save off the context here (so that the debug logic 1165 * can see the original state of this thread). 1166 */ 1167 current_thread()->machine.kpcb = state; 1168 1169 /* Hop into the debugger (typically either due to a 1170 * fatal exception, an explicit panic, or a stackshot 1171 * request. 1172 */ 1173 DebuggerCall(exception, state); 1174 1175 current_thread()->machine.kpcb = NULL; 1176 (void) ml_set_interrupts_enabled(interrupt_state); 1177 return; 1178 } else { 1179 panic("Undefined kernel instruction: pc=%p instr=%x", (void*)get_saved_state_pc(state), instr); 1180 } 1181 } 1182 1183 /* 1184 * Check for GDB breakpoint via illegal opcode. 1185 */ 1186 if (IS_ARM_GDB_TRAP(instr)) { 1187 exception = EXC_BREAKPOINT; 1188 codes[0] = EXC_ARM_BREAKPOINT; 1189 codes[1] = instr; 1190 } else { 1191 codes[1] = instr; 1192 } 1193 1194 exception_triage(exception, codes, numcodes); 1195 __builtin_unreachable(); 1196} 1197 1198#if __has_feature(ptrauth_calls) 1199static inline const char * 1200ptrauth_key_to_string(ptrauth_key key) 1201{ 1202 switch (key) { 1203 case ptrauth_key_asia: 1204 return "IA"; 1205 case ptrauth_key_asib: 1206 return "IB"; 1207 case ptrauth_key_asda: 1208 return "DA"; 1209 case ptrauth_key_asdb: 1210 return "DB"; 1211 default: 1212 __builtin_unreachable(); 1213 } 1214} 1215 1216static const char * 1217ptrauth_handle_brk_trap(void *tstate, uint16_t comment) 1218{ 1219 kernel_panic_reason_t pr = PERCPU_GET(panic_reason); 1220 arm_saved_state_t *state = (arm_saved_state_t *)tstate; 1221 1222 ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAP_START); 1223 const char *key_str = ptrauth_key_to_string(key); 1224 1225 snprintf(pr->buf, sizeof(pr->buf), 1226 "Break 0x%04X instruction exception from kernel. " 1227 "Ptrauth failure with %s key resulted in 0x%016llx", 1228 comment, key_str, saved_state64(state)->x[16]); 1229 1230 return pr->buf; 1231} 1232#endif /* __has_feature(ptrauth_calls) */ 1233 1234#if HAS_TELEMETRY_KERNEL_BRK 1235static uint32_t bound_chk_violations_event; 1236 1237static const char * 1238xnu_soft_trap_handle_breakpoint( 1239 void *tstate, 1240 uint16_t comment) 1241{ 1242#if CONFIG_UBSAN_MINIMAL 1243 if (comment == UBSAN_SOFT_TRAP_SIGNED_OF) { 1244 ubsan_handle_brk_trap(tstate, comment); 1245 } 1246#else 1247 (void)tstate; 1248#endif 1249 1250 if (comment == CLANG_SOFT_TRAP_BOUND_CHK) { 1251 os_atomic_inc(&bound_chk_violations_event, relaxed); 1252 } 1253 return NULL; 1254} 1255#endif /* HAS_TELEMETRY_KERNEL_BRK */ 1256 1257static const char * 1258xnu_hard_trap_handle_breakpoint(void *tstate, uint16_t comment) 1259{ 1260 kernel_panic_reason_t pr = PERCPU_GET(panic_reason); 1261 arm_saved_state64_t *state = saved_state64(tstate); 1262 1263 switch (comment) { 1264 case XNU_HARD_TRAP_SAFE_UNLINK: 1265 snprintf(pr->buf, sizeof(pr->buf), 1266 "panic: corrupt list around element %p", 1267 (void *)state->x[8]); 1268 return pr->buf; 1269 1270 case XNU_HARD_TRAP_STRING_CHK: 1271 return "panic: string operation caused an overflow"; 1272 1273 case XNU_HARD_TRAP_ASSERT_FAILURE: 1274 /* 1275 * Read the implicit assert arguments, see: 1276 * ML_TRAP_REGISTER_1: x8 1277 * ML_TRAP_REGISTER_2: x16 1278 * ML_TRAP_REGISTER_3: x17 1279 */ 1280 panic_assert_format(pr->buf, sizeof(pr->buf), 1281 (struct mach_assert_hdr *)state->x[8], 1282 state->x[16], state->x[17]); 1283 return pr->buf; 1284 1285 default: 1286 return NULL; 1287 } 1288} 1289 1290#if __has_feature(ptrauth_calls) 1291KERNEL_BRK_DESCRIPTOR_DEFINE(ptrauth_desc, 1292 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_PTRAUTH, 1293 .base = PTRAUTH_TRAP_START, 1294 .max = PTRAUTH_TRAP_END, 1295 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT, 1296 .handle_breakpoint = ptrauth_handle_brk_trap); 1297#endif 1298 1299KERNEL_BRK_DESCRIPTOR_DEFINE(clang_desc, 1300 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_CLANG, 1301 .base = CLANG_ARM_TRAP_START, 1302 .max = CLANG_ARM_TRAP_END, 1303 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT, 1304 .handle_breakpoint = NULL); 1305 1306KERNEL_BRK_DESCRIPTOR_DEFINE(libcxx_desc, 1307 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_LIBCXX, 1308 .base = LIBCXX_TRAP_START, 1309 .max = LIBCXX_TRAP_END, 1310 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT, 1311 .handle_breakpoint = NULL); 1312 1313#if HAS_TELEMETRY_KERNEL_BRK 1314KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_soft_traps_desc, 1315 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_TELEMETRY, 1316 .base = XNU_SOFT_TRAP_START, 1317 .max = XNU_SOFT_TRAP_END, 1318 .options = BRK_TELEMETRY_OPTIONS_RECOVERABLE_DEFAULT( 1319 /* enable_telemetry */ true), 1320 .handle_breakpoint = xnu_soft_trap_handle_breakpoint); 1321#endif /* HAS_TELEMETRY_KERNEL_BRK */ 1322 1323KERNEL_BRK_DESCRIPTOR_DEFINE(xnu_hard_traps_desc, 1324 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_XNU, 1325 .base = XNU_HARD_TRAP_START, 1326 .max = XNU_HARD_TRAP_END, 1327 .options = BRK_TELEMETRY_OPTIONS_FATAL_DEFAULT, 1328 .handle_breakpoint = xnu_hard_trap_handle_breakpoint); 1329 1330static void 1331#if !HAS_TELEMETRY_KERNEL_BRK 1332__attribute__((noreturn)) 1333#endif 1334handle_kernel_breakpoint(arm_saved_state_t *state, uint64_t esr) 1335{ 1336 uint16_t comment = ISS_BRK_COMMENT(esr); 1337 const struct kernel_brk_descriptor *desc; 1338 const char *msg = NULL; 1339 1340 desc = find_kernel_brk_descriptor_by_comment(comment); 1341 1342 if (!desc) { 1343 goto brk_out; 1344 } 1345 1346#if HAS_TELEMETRY_KERNEL_BRK 1347 if (desc->options.enable_trap_telemetry) { 1348 trap_telemetry_report_exception( 1349 /* trap_type */ desc->type, 1350 /* trap_code */ comment, 1351 /* options */ desc->options.telemetry_options, 1352 /* saved_state */ (void *)state); 1353 } 1354#endif 1355 1356 if (desc->handle_breakpoint) { 1357 msg = desc->handle_breakpoint(state, comment); 1358 } 1359 1360#if HAS_TELEMETRY_KERNEL_BRK 1361 /* Still alive? Check if we should recover. */ 1362 if (desc->options.recoverable) { 1363 add_saved_state_pc(state, 4); 1364 return; 1365 } 1366#endif 1367 1368brk_out: 1369 if (msg == NULL) { 1370 kernel_panic_reason_t pr = PERCPU_GET(panic_reason); 1371 1372 if (comment == CLANG_ARM_TRAP_BOUND_CHK) { 1373 msg = tsnprintf(pr->buf, sizeof(pr->buf), 1374 "Bounds safety trap"); 1375 } else { 1376 msg = tsnprintf(pr->buf, sizeof(pr->buf), 1377 "Break 0x%04X instruction exception from kernel. " 1378 "Panic (by design)", 1379 comment); 1380 } 1381 } 1382 1383 panic_with_thread_kernel_state(msg, state); 1384 __builtin_unreachable(); 1385#undef MSG_FMT 1386} 1387 1388/* 1389 * Similar in spirit to kernel_brk_descriptor, but with less flexible semantics: 1390 * each descriptor defines a `brk` label range for use from userspace. 1391 * When used, system policy may decide to kill the calling process without giving them opportunity to 1392 * catch the exception or continue execution from a signal handler. 1393 * This is used to enforce security boundaries: userspace code may use this mechanism 1394 * to reliably terminate when internal inconsistencies are detected. 1395 * Note that we don't invariably terminate without giving the process a say: we might only enforce 1396 * such a policy if a security feature is enabled, for example. 1397 */ 1398typedef struct user_brk_label_range_descriptor { 1399 uint16_t base; 1400 uint16_t max; 1401} user_brk_label_range_descriptor_t; 1402 1403const user_brk_label_range_descriptor_t user_brk_descriptors[] = { 1404#if __has_feature(ptrauth_calls) 1405 /* PAC failures detected in data by userspace */ 1406 { 1407 /* Use the exact same label range as kernel PAC */ 1408 .base = PTRAUTH_TRAP_START, 1409 .max = PTRAUTH_TRAP_END, 1410 }, 1411#endif /* __has_feature(ptrauth_calls) */ 1412 /* Available for use by system libraries when detecting disallowed conditions */ 1413 { 1414 /* Note this uses the same range as the kernel-specific XNU_HARD_TRAP range */ 1415 .base = 0xB000, 1416 .max = 0xBFFF, 1417 } 1418}; 1419const int user_brk_descriptor_count = sizeof(user_brk_descriptors) / sizeof(user_brk_descriptors[0]); 1420 1421const static inline user_brk_label_range_descriptor_t * 1422find_user_brk_descriptor_by_comment(uint16_t comment) 1423{ 1424 for (int desc_idx = 0; desc_idx < user_brk_descriptor_count; desc_idx++) { 1425 const user_brk_label_range_descriptor_t* des = &user_brk_descriptors[desc_idx]; 1426 if (comment >= des->base && comment <= des->max) { 1427 return des; 1428 } 1429 } 1430 1431 return NULL; 1432} 1433 1434static void 1435handle_user_breakpoint(arm_saved_state_t *state, uint64_t esr __unused) 1436{ 1437 exception_type_t exception = EXC_BREAKPOINT; 1438 mach_exception_data_type_t codes[2] = {EXC_ARM_BREAKPOINT}; 1439 mach_msg_type_number_t numcodes = 2; 1440 1441 if (ESR_EC(esr) == ESR_EC_BRK_AARCH64) { 1442 /* 1443 * Consult the trap labels we know about to decide whether userspace 1444 * should be given the opportunity to handle the exception. 1445 */ 1446 uint16_t brk_label = ISS_BRK_COMMENT(esr); 1447 const struct user_brk_label_range_descriptor* descriptor = find_user_brk_descriptor_by_comment(brk_label); 1448 /* 1449 * Note it's no problem if we don't recognize the label. 1450 * In this case we'll just go through normal exception delivery. 1451 */ 1452 if (descriptor != NULL) { 1453 exception |= EXC_MAY_BE_UNRECOVERABLE_BIT; 1454 1455#if __has_feature(ptrauth_calls) 1456 /* 1457 * We have additional policy specifically for PAC violations. 1458 * To make the rest of the code easier to follow, don't set 1459 * EXC_MAY_BE_UNRECOVERABLE_BIT here and just set EXC_PTRAUTH_BIT instead. 1460 * Conceptually a PAC failure is absolutely 'maybe unrecoverable', but it's 1461 * not really worth excising the discrepency from the plumbing. 1462 */ 1463 if (descriptor->base == PTRAUTH_TRAP_START) { 1464 exception &= ~(EXC_MAY_BE_UNRECOVERABLE_BIT); 1465 exception |= EXC_PTRAUTH_BIT; 1466 } 1467#endif /* __has_feature(ptrauth_calls) */ 1468 } 1469 } 1470 1471 codes[1] = get_saved_state_pc(state); 1472 exception_triage(exception, codes, numcodes); 1473 __builtin_unreachable(); 1474} 1475 1476static void 1477handle_watchpoint(vm_offset_t fault_addr) 1478{ 1479 exception_type_t exception = EXC_BREAKPOINT; 1480 mach_exception_data_type_t codes[2] = {EXC_ARM_DA_DEBUG}; 1481 mach_msg_type_number_t numcodes = 2; 1482 1483 codes[1] = fault_addr; 1484 exception_triage(exception, codes, numcodes); 1485 __builtin_unreachable(); 1486} 1487 1488static void 1489handle_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr, 1490 abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler) 1491{ 1492 fault_status_t fault_code; 1493 vm_prot_t fault_type; 1494 1495 inspect_abort(ESR_ISS(esr), &fault_code, &fault_type); 1496 handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler); 1497} 1498 1499static void 1500inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) 1501{ 1502 getCpuDatap()->cpu_stat.instr_ex_cnt++; 1503 *fault_code = ISS_IA_FSC(iss); 1504 *fault_type = (VM_PROT_READ | VM_PROT_EXECUTE); 1505} 1506 1507static void 1508inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type) 1509{ 1510 getCpuDatap()->cpu_stat.data_ex_cnt++; 1511 *fault_code = ISS_DA_FSC(iss); 1512 1513 /* 1514 * Cache maintenance operations always report faults as write access. 1515 * Change these to read access, unless they report a permission fault. 1516 * Only certain cache maintenance operations (e.g. 'dc ivac') require write 1517 * access to the mapping, but if a cache maintenance operation that only requires 1518 * read access generates a permission fault, then we will not be able to handle 1519 * the fault regardless of whether we treat it as a read or write fault. 1520 */ 1521 if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) { 1522 *fault_type = (VM_PROT_READ | VM_PROT_WRITE); 1523 } else { 1524 *fault_type = (VM_PROT_READ); 1525 } 1526} 1527 1528#if __has_feature(ptrauth_calls) 1529static inline uint64_t 1530fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to) 1531{ 1532 return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from; 1533} 1534 1535static inline bool 1536fault_addr_bit(vm_offset_t fault_addr, unsigned int bit) 1537{ 1538 return (bool)((fault_addr >> bit) & 1); 1539} 1540 1541extern int gARM_FEAT_FPAC; 1542extern int gARM_FEAT_FPACCOMBINE; 1543extern int gARM_FEAT_PAuth2; 1544 1545/** 1546 * Determines whether a fault address taken at EL0 contains a PAC error code 1547 * corresponding to the specified kind of ptrauth key. 1548 */ 1549static bool 1550user_fault_matches_pac_error_code(vm_offset_t fault_addr, uint64_t pc, bool data_key) 1551{ 1552 if (gARM_FEAT_FPACCOMBINE) { 1553 /* 1554 * CPUs with FPACCOMBINE always raise PAC Fail exceptions during 1555 * PAC failure. If the CPU took any other kind of exception, we 1556 * can rule out PAC as the root cause. 1557 */ 1558 return false; 1559 } 1560 1561 if (data_key && gARM_FEAT_FPAC) { 1562 uint32_t instr; 1563 int err = copyin(pc, (char *)&instr, sizeof(instr)); 1564 if (!err && !ARM64_INSTR_IS_LDRAx(instr)) { 1565 /* 1566 * On FPAC-enabled devices, PAC failure can only cause 1567 * data aborts during "combined" LDRAx instructions. If 1568 * PAC fails during a discrete AUTxx + LDR/STR 1569 * instruction sequence, then the AUTxx instruction 1570 * raises a PAC Fail exception rather than poisoning its 1571 * output address. 1572 * 1573 * In principle the same logic applies to instruction 1574 * aborts. But we have no way to identify the exact 1575 * instruction that caused the abort, so we can't tell 1576 * if it was a combined branch + auth instruction. 1577 */ 1578 return false; 1579 } 1580 } 1581 1582 bool instruction_tbi = !(get_tcr() & TCR_TBID0_TBI_DATA_ONLY); 1583 bool tbi = data_key || __improbable(instruction_tbi); 1584 1585 if (gARM_FEAT_PAuth2) { 1586 /* 1587 * EnhancedPAC2 CPUs don't encode error codes at fixed positions, so 1588 * treat all non-canonical address bits like potential poison bits. 1589 */ 1590 uint64_t mask = fault_addr_bitmask(64 - T0SZ_BOOT, 54); 1591 if (!tbi) { 1592 mask |= fault_addr_bitmask(56, 63); 1593 } 1594 return (fault_addr & mask) != 0; 1595 } else { 1596 unsigned int poison_shift; 1597 if (tbi) { 1598 poison_shift = 53; 1599 } else { 1600 poison_shift = 61; 1601 } 1602 1603 /* PAC error codes are always in the form key_number:NOT(key_number) */ 1604 bool poison_bit_1 = fault_addr_bit(fault_addr, poison_shift); 1605 bool poison_bit_2 = fault_addr_bit(fault_addr, poison_shift + 1); 1606 return poison_bit_1 != poison_bit_2; 1607 } 1608} 1609#endif /* __has_feature(ptrauth_calls) */ 1610 1611/** 1612 * Determines whether the userland thread has a JIT region in RW mode, TPRO 1613 * in RW mode, or JCTL_EL0 in pointer signing mode. A fault in any of these trusted 1614 * code paths may indicate an attack on WebKit. Rather than letting a 1615 * potentially-compromised process try to handle the exception, it will be killed 1616 * by the kernel and a crash report will be generated. 1617 */ 1618static self_restrict_mode_t 1619user_fault_in_self_restrict_mode(thread_t thread __unused) 1620{ 1621 self_restrict_mode_t out = SELF_RESTRICT_NONE; 1622 1623 return out; 1624} 1625 1626static void 1627handle_pc_align(arm_saved_state_t *ss) 1628{ 1629 exception_type_t exc; 1630 mach_exception_data_type_t codes[2]; 1631 mach_msg_type_number_t numcodes = 2; 1632 1633 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { 1634 panic_with_thread_kernel_state("PC alignment exception from kernel.", ss); 1635 } 1636 1637 exc = EXC_BAD_ACCESS; 1638#if __has_feature(ptrauth_calls) 1639 uint64_t pc = get_saved_state_pc(ss); 1640 if (user_fault_matches_pac_error_code(pc, pc, false)) { 1641 exc |= EXC_PTRAUTH_BIT; 1642 } 1643#endif /* __has_feature(ptrauth_calls) */ 1644 1645 codes[0] = EXC_ARM_DA_ALIGN; 1646 codes[1] = get_saved_state_pc(ss); 1647 1648 exception_triage(exc, codes, numcodes); 1649 __builtin_unreachable(); 1650} 1651 1652static void 1653handle_sp_align(arm_saved_state_t *ss) 1654{ 1655 exception_type_t exc; 1656 mach_exception_data_type_t codes[2]; 1657 mach_msg_type_number_t numcodes = 2; 1658 1659 if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) { 1660 panic_with_thread_kernel_state("SP alignment exception from kernel.", ss); 1661 } 1662 1663 exc = EXC_BAD_ACCESS; 1664#if __has_feature(ptrauth_calls) 1665 if (user_fault_matches_pac_error_code(get_saved_state_sp(ss), get_saved_state_pc(ss), true)) { 1666 exc |= EXC_PTRAUTH_BIT; 1667 } 1668#endif /* __has_feature(ptrauth_calls) */ 1669 1670 codes[0] = EXC_ARM_SP_ALIGN; 1671 codes[1] = get_saved_state_sp(ss); 1672 1673 exception_triage(exc, codes, numcodes); 1674 __builtin_unreachable(); 1675} 1676 1677static void 1678handle_wf_trap(arm_saved_state_t *state) 1679{ 1680 exception_type_t exc; 1681 mach_exception_data_type_t codes[2]; 1682 mach_msg_type_number_t numcodes = 2; 1683 uint32_t instr = 0; 1684 1685 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 1686 1687 exc = EXC_BAD_INSTRUCTION; 1688 codes[0] = EXC_ARM_UNDEFINED; 1689 codes[1] = instr; 1690 1691 exception_triage(exc, codes, numcodes); 1692 __builtin_unreachable(); 1693} 1694 1695static void 1696handle_fp_trap(arm_saved_state_t *state, uint64_t esr) 1697{ 1698 exception_type_t exc = EXC_ARITHMETIC; 1699 mach_exception_data_type_t codes[2]; 1700 mach_msg_type_number_t numcodes = 2; 1701 uint32_t instr = 0; 1702 1703 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 1704 panic_with_thread_kernel_state("Floating point exception from kernel", state); 1705 } 1706 1707 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 1708 codes[1] = instr; 1709 1710 /* The floating point trap flags are only valid if TFV is set. */ 1711 if (!fp_exceptions_enabled) { 1712 exc = EXC_BAD_INSTRUCTION; 1713 codes[0] = EXC_ARM_UNDEFINED; 1714 } else if (!(esr & ISS_FP_TFV)) { 1715 codes[0] = EXC_ARM_FP_UNDEFINED; 1716 } else if (esr & ISS_FP_UFF) { 1717 codes[0] = EXC_ARM_FP_UF; 1718 } else if (esr & ISS_FP_OFF) { 1719 codes[0] = EXC_ARM_FP_OF; 1720 } else if (esr & ISS_FP_IOF) { 1721 codes[0] = EXC_ARM_FP_IO; 1722 } else if (esr & ISS_FP_DZF) { 1723 codes[0] = EXC_ARM_FP_DZ; 1724 } else if (esr & ISS_FP_IDF) { 1725 codes[0] = EXC_ARM_FP_ID; 1726 } else if (esr & ISS_FP_IXF) { 1727 codes[0] = EXC_ARM_FP_IX; 1728 } else { 1729 panic("Unrecognized floating point exception, state=%p, esr=%#llx", state, esr); 1730 } 1731 1732 exception_triage(exc, codes, numcodes); 1733 __builtin_unreachable(); 1734} 1735 1736 1737 1738/* 1739 * handle_alignment_fault_from_user: 1740 * state: Saved state 1741 * 1742 * Attempts to deal with an alignment fault from userspace (possibly by 1743 * emulating the faulting instruction). If emulation failed due to an 1744 * unservicable fault, the ESR for that fault will be stored in the 1745 * recovery_esr field of the thread by the exception code. 1746 * 1747 * Returns: 1748 * -1: Emulation failed (emulation of state/instr not supported) 1749 * 0: Successfully emulated the instruction 1750 * EFAULT: Emulation failed (probably due to permissions) 1751 * EINVAL: Emulation failed (probably due to a bad address) 1752 */ 1753 1754 1755static int 1756handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr) 1757{ 1758 int ret = -1; 1759 1760#pragma unused (state) 1761#pragma unused (vmfr) 1762 1763 return ret; 1764} 1765 1766 1767 1768#if HAS_ARM_FEAT_SME 1769static void 1770handle_sme_trap(arm_saved_state_t *state, uint64_t esr) 1771{ 1772 exception_type_t exc = EXC_BAD_INSTRUCTION; 1773 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; 1774 mach_msg_type_number_t numcodes = 2; 1775 1776 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) { 1777 panic("SME exception from kernel, state=%p, esr=%#llx", state, esr); 1778 } 1779 if (!arm_sme_version()) { 1780 /* 1781 * If SME is disabled in software but userspace executes an SME 1782 * instruction anyway, then the CPU will still raise an 1783 * SME-specific trap. Triage it as if the CPU raised an 1784 * undefined-instruction trap. 1785 */ 1786 exception_triage(exc, codes, numcodes); 1787 __builtin_unreachable(); 1788 } 1789 1790 if (ISS_SME_SMTC(ESR_ISS(esr)) == ISS_SME_SMTC_CAPCR) { 1791 thread_t thread = current_thread(); 1792 switch (machine_thread_sme_state_alloc(thread)) { 1793 case KERN_SUCCESS: 1794 return; 1795 1796 1797 default: 1798 panic("Failed to allocate SME state for thread %p", thread); 1799 } 1800 } 1801 1802 uint32_t instr; 1803 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 1804 codes[1] = instr; 1805 1806 exception_triage(exc, codes, numcodes); 1807 __builtin_unreachable(); 1808} 1809#endif /* HAS_ARM_FEAT_SME */ 1810 1811static void 1812handle_sw_step_debug(arm_saved_state_t *state) 1813{ 1814 thread_t thread = current_thread(); 1815 exception_type_t exc; 1816 mach_exception_data_type_t codes[2]; 1817 mach_msg_type_number_t numcodes = 2; 1818 1819 if (!PSR64_IS_USER(get_saved_state_cpsr(state))) { 1820 panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state); 1821 } 1822 1823 // Disable single step and unmask interrupts (in the saved state, anticipating next exception return) 1824 if (thread->machine.DebugData != NULL) { 1825 thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1; 1826 } else { 1827 panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state); 1828 } 1829 1830 mask_user_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL); 1831 1832 // Special encoding for gdb single step event on ARM 1833 exc = EXC_BREAKPOINT; 1834 codes[0] = 1; 1835 codes[1] = 0; 1836 1837 exception_triage(exc, codes, numcodes); 1838 __builtin_unreachable(); 1839} 1840 1841#if MACH_ASSERT 1842TUNABLE_WRITEABLE(self_restrict_mode_t, panic_on_jit_guard, "panic_on_jit_guard", SELF_RESTRICT_NONE); 1843#endif /* MACH_ASSERT */ 1844 1845static void 1846handle_user_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr, 1847 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler) 1848{ 1849 exception_type_t exc = EXC_BAD_ACCESS; 1850 mach_exception_data_type_t codes[2]; 1851 mach_msg_type_number_t numcodes = 2; 1852 thread_t thread = current_thread(); 1853 1854 (void)expected_fault_handler; 1855 1856 if (__improbable(!SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)))) { 1857 panic_with_thread_kernel_state("User abort from non-interruptible context", state); 1858 } 1859 1860 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */ 1861 1862 if (!is_servicible_fault(fault_code, esr) && 1863 thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) { 1864 thread_reset_pcs_done_faulting(thread); 1865 } 1866 1867 if (is_vm_fault(fault_code)) { 1868 vm_map_t map = thread->map; 1869 vm_offset_t vm_fault_addr = fault_addr; 1870 kern_return_t result = KERN_FAILURE; 1871 1872 assert(map != kernel_map); 1873 1874 if (!(fault_type & VM_PROT_EXECUTE)) { 1875 vm_fault_addr = VM_USER_STRIP_TBI(fault_addr); 1876 } 1877 1878 /* check to see if it is just a pmap ref/modify fault */ 1879 if (!is_translation_fault(fault_code)) { 1880 result = arm_fast_fault(map->pmap, 1881 vm_fault_addr, 1882 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), TRUE); 1883 } 1884 if (result != KERN_SUCCESS) { 1885 1886 { 1887 /* We have to fault the page in */ 1888 result = vm_fault(map, vm_fault_addr, fault_type, 1889 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, THREAD_ABORTSAFE, 1890 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0); 1891 } 1892 } 1893 if (thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) { 1894 thread_reset_pcs_done_faulting(thread); 1895 } 1896 if (result == KERN_SUCCESS || result == KERN_ABORTED) { 1897 return; 1898 } 1899 1900 /* 1901 * vm_fault() should never return KERN_FAILURE for page faults from user space. 1902 * If it does, we're leaking preemption disables somewhere in the kernel. 1903 */ 1904 if (__improbable(result == KERN_FAILURE)) { 1905 panic("vm_fault() KERN_FAILURE from user fault on thread %p", thread); 1906 } 1907 1908 codes[0] = result; 1909 } else if (is_alignment_fault(fault_code)) { 1910 kern_return_t vmfkr = KERN_SUCCESS; 1911 thread->machine.recover_esr = 0; 1912 thread->machine.recover_far = 0; 1913 int result = handle_alignment_fault_from_user(state, &vmfkr); 1914 if (result == 0) { 1915 /* Successfully emulated, or instruction 1916 * copyin() for decode/emulation failed. 1917 * Continue, or redrive instruction. 1918 */ 1919 thread_exception_return(); 1920 } else if (((result == EFAULT) || (result == EINVAL)) && 1921 (thread->machine.recover_esr == 0)) { 1922 /* 1923 * If we didn't actually take a fault, but got one of 1924 * these errors, then we failed basic sanity checks of 1925 * the fault address. Treat this as an invalid 1926 * address. 1927 */ 1928 codes[0] = KERN_INVALID_ADDRESS; 1929 } else if ((result == EFAULT) && 1930 (thread->machine.recover_esr)) { 1931 /* 1932 * Since alignment aborts are prioritized 1933 * ahead of translation aborts, the misaligned 1934 * atomic emulation flow may have triggered a 1935 * VM pagefault, which the VM could not resolve. 1936 * Report the VM fault error in codes[] 1937 */ 1938 1939 codes[0] = vmfkr; 1940 assertf(vmfkr != KERN_SUCCESS, "Unexpected vmfkr 0x%x", vmfkr); 1941 /* Cause ESR_EC to reflect an EL0 abort */ 1942 thread->machine.recover_esr &= ~ESR_EC_MASK; 1943 thread->machine.recover_esr |= (ESR_EC_DABORT_EL0 << ESR_EC_SHIFT); 1944 set_saved_state_esr(thread->machine.upcb, thread->machine.recover_esr); 1945 set_saved_state_far(thread->machine.upcb, thread->machine.recover_far); 1946 fault_addr = thread->machine.recover_far; 1947 } else { 1948 /* This was just an unsupported alignment 1949 * exception. Misaligned atomic emulation 1950 * timeouts fall in this category. 1951 */ 1952 codes[0] = EXC_ARM_DA_ALIGN; 1953 } 1954 } else if (is_parity_error(fault_code)) { 1955#if defined(APPLE_ARM64_ARCH_FAMILY) 1956 /* 1957 * Platform errors are handled in sleh_sync before interrupts are enabled. 1958 */ 1959#else 1960 panic("User parity error."); 1961#endif 1962 } else { 1963 codes[0] = KERN_FAILURE; 1964 } 1965 1966#if CODE_SIGNING_MONITOR 1967 /* 1968 * If the code reaches here, it means we weren't able to resolve the fault and we're 1969 * going to be sending the task an exception. On systems which have the code signing 1970 * monitor enabled, an execute fault which cannot be handled must result in sending 1971 * a SIGKILL to the task. 1972 */ 1973 if (is_vm_fault(fault_code) && (fault_type & VM_PROT_EXECUTE)) { 1974 csm_code_signing_violation(current_proc(), fault_addr); 1975 } 1976#endif 1977 1978 codes[1] = fault_addr; 1979#if __has_feature(ptrauth_calls) 1980 bool is_data_abort = (ESR_EC(esr) == ESR_EC_DABORT_EL0); 1981 if (user_fault_matches_pac_error_code(fault_addr, get_saved_state_pc(state), is_data_abort)) { 1982 exc |= EXC_PTRAUTH_BIT; 1983 } 1984#endif /* __has_feature(ptrauth_calls) */ 1985 1986 const self_restrict_mode_t self_restrict_mode = user_fault_in_self_restrict_mode(thread); 1987 if ((self_restrict_mode != SELF_RESTRICT_NONE) && 1988 task_is_jit_exception_fatal(get_threadtask(thread))) { 1989 int flags = PX_KTRIAGE; 1990 exception_info_t info = { 1991 .os_reason = OS_REASON_SELF_RESTRICT, 1992 .exception_type = exc, 1993 .mx_code = codes[0], 1994 .mx_subcode = codes[1] 1995 }; 1996 1997#if MACH_ASSERT 1998 /* 1999 * Case: panic_on_jit_guard=1. Catch an early process creation TPRO issue causing 2000 * rdar://129742083. Only panic during early process creation (1 thread, few syscalls 2001 * issued) to avoid spurious panics. 2002 */ 2003 const self_restrict_mode_t self_restrict_panic_mask = panic_on_jit_guard & self_restrict_mode; 2004 bool should_panic = ((self_restrict_panic_mask == SELF_RESTRICT_ANY) && 2005 (current_task()->thread_count == 1) && 2006 (thread->syscalls_unix < 24)); 2007 2008 /* 2009 * Modes other than ANY will force panic, skipping checks that were done in the ANY case, 2010 * but allowing us to filter on a more specific scenario (e.g. TPRO, JIT, etc). This is 2011 * meant to catch a TPRO issue causing rdar://145703251. Restrict to KERN_PROTECTION_FAILURE 2012 * only to avoid failures from the more frequent case of KERN_INVALID_ADDRESS that aren't 2013 * of interest for that radar. 2014 */ 2015 should_panic |= (codes[0] == KERN_PROTECTION_FAILURE) 2016 && ((self_restrict_panic_mask & ~SELF_RESTRICT_ANY) != 0); 2017 2018 printf("\nGUARD_REASON_JIT exc %d codes=<0x%llx,0x%llx> syscalls %d task %p thread %p va 0x%lx code 0x%x type 0x%x esr 0x%llx\n", 2019 exc, codes[0], codes[1], thread->syscalls_unix, current_task(), thread, fault_addr, fault_code, fault_type, esr); 2020 if (should_panic) { 2021 panic("GUARD_REASON_JIT exc %d codes=<0x%llx,0x%llx> syscalls %d task %p thread %p va 0x%lx code 0x%x type 0x%x esr 0x%llx state %p j %d t %d s user 0x%llx (0x%llx) jb 0x%llx (0x%llx)", 2022 exc, codes[0], codes[1], thread->syscalls_unix, current_task(), thread, fault_addr, fault_code, fault_type, esr, state, 2023 0, 0, 0ull, 0ull, 2024 0ull, 0ull 2025 ); 2026 } 2027#endif /* MACH_ASSERT */ 2028 2029 exit_with_mach_exception(current_proc(), info, flags); 2030 } 2031 2032 2033 exception_triage(exc, codes, numcodes); 2034 __builtin_unreachable(); 2035} 2036 2037/** 2038 * Panic because the kernel abort handler tried to apply a recovery handler that 2039 * isn't inside copyio_recover_table[]. 2040 * 2041 * @param state original saved-state 2042 * @param recover invalid recovery handler 2043 */ 2044__attribute__((noreturn, used)) 2045static void 2046panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover) 2047{ 2048 panic("attempt to set invalid recovery handler %p on kernel saved-state %p", recover, state); 2049} 2050 2051/** 2052 * Update a thread saved-state to store an error code in x0 and branch to a 2053 * copyio recovery handler. 2054 * 2055 * @param state original saved-state 2056 * @param esr ESR_ELx value for the fault taken 2057 * @param fault_addr FAR_ELx value for the fault taken 2058 * @param thread target thread 2059 * @param recover destination copyio recovery handler 2060 * @param x0 error code to populate into x0 2061 */ 2062static void 2063handle_kernel_abort_recover_with_error_code( 2064 arm_saved_state_t *state, 2065 uint64_t esr, 2066 vm_offset_t fault_addr, 2067 thread_t thread, 2068 struct copyio_recovery_entry *_Nonnull recover, 2069 uint64_t x0) 2070{ 2071 thread->machine.recover_esr = esr; 2072 thread->machine.recover_far = fault_addr; 2073 saved_state64(state)->x[0] = x0; 2074#if defined(HAS_APPLE_PAC) 2075 MANIPULATE_SIGNED_THREAD_STATE(state, 2076 "adrp x6, _copyio_recover_table_end@page \n" 2077 "add x6, x6, _copyio_recover_table_end@pageoff \n" 2078 "cmp %[recover], x6 \n" 2079 "b.lt 1f \n" 2080 "bl _panic_on_invalid_recovery_handler \n" 2081 "brk #0 \n" 2082 "1: \n" 2083 "adrp x6, _copyio_recover_table@page \n" 2084 "add x6, x6, _copyio_recover_table@pageoff \n" 2085 "subs x7, %[recover], x6 \n" 2086 "b.pl 1f \n" 2087 "bl _panic_on_invalid_recovery_handler \n" 2088 "brk #0 \n" 2089 "1: \n" 2090 "udiv x8, x7, %[SIZEOF_RECOVER] \n" 2091 "mul x8, x8, %[SIZEOF_RECOVER] \n" 2092 "cmp x7, x8 \n" 2093 "b.eq 1f \n" 2094 "bl _panic_on_invalid_recovery_handler \n" 2095 "brk #0 \n" 2096 "1: \n" 2097 "ldr x1, [%[recover], %[CRE_RECOVERY]] \n" 2098 "add x1, x1, x6 \n" 2099 "str x1, [x0, %[SS64_PC]] \n", 2100 [recover] "r"(recover), 2101 [SIZEOF_RECOVER] "r"((sizeof(*recover))), 2102 [CRE_RECOVERY] "i"(offsetof(struct copyio_recovery_entry, cre_recovery)) 2103 ); 2104#else 2105 ptrdiff_t recover_offset = (uintptr_t)recover - (uintptr_t)copyio_recover_table; 2106 if ((uintptr_t)recover < (uintptr_t)copyio_recover_table || 2107 (uintptr_t)recover >= (uintptr_t)copyio_recover_table_end || 2108 (recover_offset % sizeof(*recover)) != 0) { 2109 panic_on_invalid_recovery_handler(state, recover); 2110 } 2111 saved_state64(state)->pc = copyio_recovery_addr(recover->cre_recovery); 2112#endif 2113} 2114 2115static inline void 2116handle_kernel_abort_recover( 2117 arm_saved_state_t *state, 2118 uint64_t esr, 2119 vm_offset_t fault_addr, 2120 thread_t thread, 2121 struct copyio_recovery_entry *_Nonnull recover) 2122{ 2123 handle_kernel_abort_recover_with_error_code(state, esr, fault_addr, thread, recover, EFAULT); 2124} 2125 2126 2127static void 2128handle_kernel_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr, 2129 fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler) 2130{ 2131 thread_t thread = current_thread(); 2132 struct copyio_recovery_entry *recover = find_copyio_recovery_entry( 2133 get_saved_state_pc(state)); 2134 2135#ifndef CONFIG_XNUPOST 2136 (void)expected_fault_handler; 2137#endif /* CONFIG_XNUPOST */ 2138 2139 2140#if CONFIG_DTRACE 2141 if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */ 2142 if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */ 2143 /* 2144 * Point to next instruction, or recovery handler if set. 2145 */ 2146 if (recover) { 2147 handle_kernel_abort_recover(state, esr, VM_USER_STRIP_PTR(fault_addr), thread, recover); 2148 } else { 2149 add_saved_state_pc(state, 4); 2150 } 2151 return; 2152 } else { 2153 panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state); 2154 } 2155 } 2156#endif 2157 2158 if (is_vm_fault(fault_code)) { 2159 kern_return_t result = KERN_FAILURE; 2160 vm_map_t map; 2161 int interruptible; 2162 2163#ifdef CONFIG_XNUPOST 2164 if (expected_fault_handler && expected_fault_handler(state)) { 2165 return; 2166 } 2167#endif /* CONFIG_XNUPOST */ 2168 2169 if (VM_KERNEL_ADDRESS(fault_addr) || thread == THREAD_NULL || recover == 0) { 2170 /* 2171 * If no recovery handler is supplied, always drive the fault against 2172 * the kernel map. If the fault was taken against a userspace VA, indicating 2173 * an unprotected access to user address space, vm_fault() should fail and 2174 * ultimately lead to a panic here. 2175 */ 2176 map = kernel_map; 2177 interruptible = THREAD_UNINT; 2178 2179#if CONFIG_KERNEL_TAGGING 2180 /* 2181 * If kernel tagging is enabled, canonicalize the address here, so that we have a 2182 * chance to find it in the VM ranges. Do not mess with exec fault cases. 2183 */ 2184 if (!((fault_type) & VM_PROT_EXECUTE)) { 2185 fault_addr = vm_memtag_canonicalize(map, fault_addr); 2186 } 2187#endif /* CONFIG_KERNEL_TAGGING */ 2188 } else { 2189 map = thread->map; 2190 2191 /** 2192 * In the case that the recovery handler is set (e.g., during copyio 2193 * and dtrace probes), we don't want the vm_fault() operation to be 2194 * aborted early. Those code paths can't handle restarting the 2195 * vm_fault() operation so don't allow it to return early without 2196 * creating the wanted mapping. 2197 */ 2198 interruptible = (recover) ? THREAD_UNINT : THREAD_ABORTSAFE; 2199 2200 } 2201 2202 /* 2203 * Ensure no faults in the physical aperture. This could happen if 2204 * a page table is incorrectly allocated from the read only region 2205 * when running with KTRR. 2206 */ 2207 if (__improbable(fault_addr >= physmap_base) && (fault_addr < physmap_end)) { 2208 panic_with_thread_kernel_state("Unexpected fault in kernel physical aperture", state); 2209 } 2210 if (__improbable(fault_addr >= gVirtBase && fault_addr < static_memory_end)) { 2211 panic_with_thread_kernel_state("Unexpected fault in kernel static region", state); 2212 } 2213 2214 /* check to see if it is just a pmap ref/modify fault */ 2215 if (!is_translation_fault(fault_code)) { 2216 result = arm_fast_fault(map->pmap, 2217 fault_addr, 2218 fault_type, (fault_code == FSC_ACCESS_FLAG_FAULT_L3), FALSE); 2219 if (result == KERN_SUCCESS) { 2220 return; 2221 } 2222 } 2223 2224 /** 2225 * vm_fault() can be called with preemption disabled (and indeed this is expected for 2226 * certain copyio() scenarios), but can't safely be called with interrupts disabled once 2227 * the system has gone multi-threaded. Other than some early-boot situations such as 2228 * startup kext loading, kernel paging operations should never be triggered by 2229 * non-interruptible code in the first place, so a fault from such a context will 2230 * ultimately produce a kernel data abort panic anyway. In these cases, skip calling 2231 * vm_fault() to avoid masking the real kernel panic with a failed VM locking assertion. 2232 */ 2233 if (__probable(SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)) || 2234 startup_phase < STARTUP_SUB_EARLY_BOOT || 2235 current_cpu_datap()->cpu_hibernate)) { 2236 if (result != KERN_PROTECTION_FAILURE) { 2237 // VM will query this property when deciding to throttle this fault, we don't want to 2238 // throttle kernel faults for copyio faults. The presence of a recovery entry is used as a 2239 // proxy for being in copyio code. 2240 bool const was_recover = thread->recover; 2241 thread->recover = was_recover || recover; 2242 2243 /* 2244 * We have to "fault" the page in. 2245 */ 2246 result = vm_fault(map, fault_addr, fault_type, 2247 /* change_wiring */ FALSE, VM_KERN_MEMORY_NONE, interruptible, 2248 /* caller_pmap */ NULL, /* caller_pmap_addr */ 0); 2249 2250 thread->recover = was_recover; 2251 } 2252 2253 if (result == KERN_SUCCESS) { 2254 return; 2255 } 2256 } 2257 2258 /* 2259 * If we have a recover handler, invoke it now. 2260 */ 2261 if (recover) { 2262 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover); 2263 return; 2264 } 2265 2266 panic_fault_address = fault_addr; 2267 } else if (is_alignment_fault(fault_code)) { 2268 if (recover) { 2269 handle_kernel_abort_recover(state, esr, fault_addr, thread, recover); 2270 return; 2271 } 2272 panic_with_thread_kernel_state("Unaligned kernel data abort.", state); 2273 } else if (is_parity_error(fault_code)) { 2274#if defined(APPLE_ARM64_ARCH_FAMILY) 2275 /* 2276 * Platform errors are handled in sleh_sync before interrupts are enabled. 2277 */ 2278#else 2279 panic_with_thread_kernel_state("Kernel parity error.", state); 2280#endif 2281 } else { 2282 kprintf("Unclassified kernel abort (fault_code=0x%x)\n", fault_code); 2283 } 2284 2285 panic_with_thread_kernel_state("Kernel data abort.", state); 2286} 2287 2288extern void syscall_trace(struct arm_saved_state * regs); 2289 2290static void 2291handle_svc(arm_saved_state_t *state) 2292{ 2293 int trap_no = get_saved_state_svc_number(state); 2294 thread_t thread = current_thread(); 2295 struct proc *p; 2296 2297#define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */ 2298 2299#define TRACE_SYSCALL 1 2300#if TRACE_SYSCALL 2301 syscall_trace(state); 2302#endif 2303 2304 thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */ 2305 2306 if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) { 2307 platform_syscall(state); 2308 panic("Returned from platform_syscall()?"); 2309 } 2310 2311 current_cached_proc_cred_update(); 2312 2313 if (trap_no < 0) { 2314 switch (trap_no) { 2315 case MACH_ARM_TRAP_ABSTIME: 2316 handle_mach_absolute_time_trap(state); 2317 return; 2318 case MACH_ARM_TRAP_CONTTIME: 2319 handle_mach_continuous_time_trap(state); 2320 return; 2321 } 2322 2323 /* Counting perhaps better in the handler, but this is how it's been done */ 2324 thread->syscalls_mach++; 2325 mach_syscall(state); 2326 } else { 2327 /* Counting perhaps better in the handler, but this is how it's been done */ 2328 thread->syscalls_unix++; 2329 p = get_bsdthreadtask_info(thread); 2330 2331 assert(p); 2332 2333 unix_syscall(state, thread, p); 2334 } 2335} 2336 2337static void 2338handle_mach_absolute_time_trap(arm_saved_state_t *state) 2339{ 2340 uint64_t now = mach_absolute_time(); 2341 saved_state64(state)->x[0] = now; 2342} 2343 2344static void 2345handle_mach_continuous_time_trap(arm_saved_state_t *state) 2346{ 2347 uint64_t now = mach_continuous_time(); 2348 saved_state64(state)->x[0] = now; 2349} 2350 2351 2352__attribute__((noreturn)) 2353static void 2354handle_msr_trap(arm_saved_state_t *state, uint64_t esr) 2355{ 2356 exception_type_t exception = EXC_BAD_INSTRUCTION; 2357 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; 2358 mach_msg_type_number_t numcodes = 2; 2359 uint32_t instr = 0; 2360 2361 if (!is_saved_state64(state)) { 2362 panic("MSR/MRS trap (ESR 0x%llx) from 32-bit state", esr); 2363 } 2364 2365 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 2366 panic("MSR/MRS trap (ESR 0x%llx) from kernel", esr); 2367 } 2368 2369 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 2370 codes[1] = instr; 2371 2372 exception_triage(exception, codes, numcodes); 2373 __builtin_unreachable(); 2374} 2375 2376#if __has_feature(ptrauth_calls) 2377static void 2378stringify_gpr(unsigned int r, char reg[4]) 2379{ 2380 switch (r) { 2381 case 29: 2382 strncpy(reg, "fp", 4); 2383 return; 2384 2385 case 30: 2386 strncpy(reg, "lr", 4); 2387 return; 2388 2389 case 31: 2390 strncpy(reg, "xzr", 4); 2391 return; 2392 2393 default: 2394 snprintf(reg, 4, "x%u", r); 2395 return; 2396 } 2397} 2398 2399static void 2400autxx_instruction_extract_reg(uint32_t instr, char reg[4]) 2401{ 2402 unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr); 2403 stringify_gpr(rd, reg); 2404} 2405 2406static const char * 2407autix_system_instruction_extract_reg(uint32_t instr) 2408{ 2409 unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr); 2410 if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 || 2411 crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) { 2412 return "x17"; 2413 } else { 2414 return "lr"; 2415 } 2416} 2417 2418static void 2419bxrax_instruction_extract_reg(uint32_t instr, char reg[4]) 2420{ 2421 unsigned int rn = ARM64_INSTR_BxRAx_RN_GET(instr); 2422 stringify_gpr(rn, reg); 2423} 2424 2425static void 2426handle_pac_fail(arm_saved_state_t *state, uint64_t esr) 2427{ 2428 exception_type_t exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT; 2429 mach_exception_data_type_t codes[2] = {EXC_ARM_PAC_FAIL}; 2430 mach_msg_type_number_t numcodes = 2; 2431 uint32_t instr = 0; 2432 2433 if (!is_saved_state64(state)) { 2434 panic("PAC failure (ESR 0x%llx) from 32-bit state", esr); 2435 } 2436 2437 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 2438 2439 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 2440#define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key" 2441#define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s" 2442#define BXRAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while branching to %s" 2443#define RETAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while returning" 2444#define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT 2445#define MAX_PAC_MSG_FMT BXRAX_MSG_FMT 2446 2447 char msg[strlen(MAX_PAC_MSG_FMT) 2448 - strlen("%s") + strlen("IA") 2449 - strlen("%s") + strlen("xzr") 2450 + 1]; 2451 ptrauth_key key = (ptrauth_key)(esr & 0x3); 2452 const char *key_str = ptrauth_key_to_string(key); 2453 2454 if (ARM64_INSTR_IS_AUTxx(instr)) { 2455 char reg[4]; 2456 autxx_instruction_extract_reg(instr, reg); 2457 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg); 2458 } else if (ARM64_INSTR_IS_AUTIx_SYSTEM(instr)) { 2459 const char *reg = autix_system_instruction_extract_reg(instr); 2460 snprintf(msg, sizeof(msg), AUTXX_MSG_FMT, key_str, reg); 2461 } else if (ARM64_INSTR_IS_BxRAx(instr)) { 2462 char reg[4]; 2463 bxrax_instruction_extract_reg(instr, reg); 2464 snprintf(msg, sizeof(msg), BXRAX_MSG_FMT, key_str, reg); 2465 } else if (ARM64_INSTR_IS_RETAx(instr)) { 2466 snprintf(msg, sizeof(msg), RETAX_MSG_FMT, key_str); 2467 } else { 2468 snprintf(msg, sizeof(msg), GENERIC_MSG_FMT, key_str); 2469 } 2470 panic_with_thread_kernel_state(msg, state); 2471 } 2472 2473 codes[1] = instr; 2474 2475 exception_triage(exception, codes, numcodes); 2476 __builtin_unreachable(); 2477} 2478#endif /* __has_feature(ptrauth_calls) */ 2479 2480__attribute__((noreturn)) 2481static void 2482handle_bti_fail(arm_saved_state_t *state, uint64_t esr) 2483{ 2484 uint32_t btype = (uint32_t) esr & ISS_BTI_BTYPE_MASK; 2485 2486 if (!is_saved_state64(state)) { 2487 /* BTI is an ARMv8 feature, this should not be possible */ 2488 panic("BTI failure for 32-bit state? (ESR=0x%llx)", esr); 2489 } 2490 2491 /* 2492 * We currently only expect BTI to be enabled for kernel pages, so panic if 2493 * we detect otherwise. 2494 */ 2495 if (!PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 2496 panic("Unexpected non-kernel BTI failure? (ESR=0x%llx)", esr); 2497 } 2498 2499#define BTI_FAIL_PTR_FMT "%04x" 2500#define BTI_FAIL_MSG_FMT "Kernel BTI failure (BTYPE=0x" BTI_FAIL_PTR_FMT ")" 2501 /* Replace the pointer format with the length of the pointer message+NULL */ 2502 char msg[strlen(BTI_FAIL_MSG_FMT) - strlen(BTI_FAIL_PTR_FMT) + 8 + 1]; 2503 snprintf(msg, sizeof(msg), BTI_FAIL_MSG_FMT, btype); 2504 panic_with_thread_kernel_state(msg, state); 2505 __builtin_unreachable(); 2506} 2507 2508static void 2509handle_user_trapped_instruction32(arm_saved_state_t *state, uint64_t esr) 2510{ 2511 exception_type_t exception = EXC_BAD_INSTRUCTION; 2512 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; 2513 mach_msg_type_number_t numcodes = 2; 2514 uint32_t instr; 2515 2516 if (is_saved_state64(state)) { 2517 panic("ESR (0x%llx) for instruction trapped from U32, but saved state is 64-bit.", esr); 2518 } 2519 2520 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 2521 panic("ESR (0x%llx) for instruction trapped from U32, actually came from kernel?", esr); 2522 } 2523 2524 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 2525 codes[1] = instr; 2526 2527 exception_triage(exception, codes, numcodes); 2528 __builtin_unreachable(); 2529} 2530 2531static void 2532handle_simd_trap(arm_saved_state_t *state, uint64_t esr) 2533{ 2534 exception_type_t exception = EXC_BAD_INSTRUCTION; 2535 mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED}; 2536 mach_msg_type_number_t numcodes = 2; 2537 uint32_t instr = 0; 2538 2539 if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) { 2540 panic("ESR (0x%llx) for SIMD trap from userland, actually came from kernel?", esr); 2541 } 2542 2543 COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr)); 2544 codes[1] = instr; 2545 2546 exception_triage(exception, codes, numcodes); 2547 __builtin_unreachable(); 2548} 2549 2550void 2551sleh_irq(arm_saved_state_t *state) 2552{ 2553 cpu_data_t * cdp __unused = getCpuDatap(); 2554#if MACH_ASSERT 2555 int preemption_level = sleh_get_preemption_level(); 2556#endif 2557 2558 2559 sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER); 2560 2561#if USE_APPLEARMSMP 2562 PE_handle_ext_interrupt(); 2563#else 2564 /* Run the registered interrupt handler. */ 2565 cdp->interrupt_handler(cdp->interrupt_target, 2566 cdp->interrupt_refCon, 2567 cdp->interrupt_nub, 2568 cdp->interrupt_source); 2569#endif 2570 2571 entropy_collect(); 2572 2573 2574 sleh_interrupt_handler_epilogue(); 2575#if MACH_ASSERT 2576 if (preemption_level != sleh_get_preemption_level()) { 2577 panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level()); 2578 } 2579#endif 2580} 2581 2582void 2583sleh_fiq(arm_saved_state_t *state) 2584{ 2585 unsigned int type = DBG_INTR_TYPE_UNKNOWN; 2586#if MACH_ASSERT 2587 int preemption_level = sleh_get_preemption_level(); 2588#endif 2589 2590#if MONOTONIC_FIQ 2591 uint64_t pmcr0 = 0, upmsr = 0; 2592#endif /* MONOTONIC_FIQ */ 2593 2594#if defined(HAS_IPI) 2595 boolean_t is_ipi = FALSE; 2596 uint64_t ipi_sr = 0; 2597 2598 if (gFastIPI) { 2599 MRS(ipi_sr, "S3_5_C15_C1_1"); 2600 2601 if (ipi_sr & ARM64_IPISR_IPI_PENDING) { 2602 is_ipi = TRUE; 2603 } 2604 } 2605 2606 if (is_ipi) { 2607 type = DBG_INTR_TYPE_IPI; 2608 } else 2609#endif /* defined(HAS_IPI) */ 2610 if (ml_get_timer_pending()) { 2611 type = DBG_INTR_TYPE_TIMER; 2612 } 2613#if MONOTONIC_FIQ 2614 /* Consult the PMI sysregs last, after IPI/timer 2615 * classification. 2616 */ 2617 else if (mt_pmi_pending(&pmcr0, &upmsr)) { 2618 type = DBG_INTR_TYPE_PMI; 2619 } 2620#endif /* MONOTONIC_FIQ */ 2621 2622 sleh_interrupt_handler_prologue(state, type); 2623 2624#if APPLEVIRTUALPLATFORM 2625 uint64_t iar = __builtin_arm_rsr64("ICC_IAR0_EL1"); 2626#endif 2627 2628#if defined(HAS_IPI) 2629 if (type == DBG_INTR_TYPE_IPI) { 2630 /* 2631 * Order is important here: we must ack the IPI by writing IPI_SR 2632 * before we call cpu_signal_handler(). Otherwise, there will be 2633 * a window between the completion of pending-signal processing in 2634 * cpu_signal_handler() and the ack during which a newly-issued 2635 * IPI to this CPU may be lost. ISB is required to ensure the msr 2636 * is retired before execution of cpu_signal_handler(). 2637 */ 2638 MSR("S3_5_C15_C1_1", ARM64_IPISR_IPI_PENDING); 2639 __builtin_arm_isb(ISB_SY); 2640 cpu_signal_handler(); 2641 } else 2642#endif /* defined(HAS_IPI) */ 2643#if MONOTONIC_FIQ 2644 if (type == DBG_INTR_TYPE_PMI) { 2645 ml_interrupt_masked_debug_start(mt_fiq, DBG_INTR_TYPE_PMI); 2646 mt_fiq(getCpuDatap(), pmcr0, upmsr); 2647 ml_interrupt_masked_debug_end(); 2648 } else 2649#endif /* MONOTONIC_FIQ */ 2650 { 2651 /* 2652 * We don't know that this is a timer, but we don't have insight into 2653 * the other interrupts that go down this path. 2654 */ 2655 2656 cpu_data_t *cdp = getCpuDatap(); 2657 2658 cdp->cpu_decrementer = -1; /* Large */ 2659 2660 /* 2661 * ARM64_TODO: whether we're coming from userland is ignored right now. 2662 * We can easily thread it through, but not bothering for the 2663 * moment (AArch32 doesn't either). 2664 */ 2665 ml_interrupt_masked_debug_start(rtclock_intr, DBG_INTR_TYPE_TIMER); 2666 rtclock_intr(TRUE); 2667 ml_interrupt_masked_debug_end(); 2668 } 2669 2670#if APPLEVIRTUALPLATFORM 2671 if (iar != GIC_SPURIOUS_IRQ) { 2672 __builtin_arm_wsr64("ICC_EOIR0_EL1", iar); 2673 __builtin_arm_isb(ISB_SY); 2674 } 2675#endif 2676 2677 sleh_interrupt_handler_epilogue(); 2678#if MACH_ASSERT 2679 if (preemption_level != sleh_get_preemption_level()) { 2680 panic("fiq type %u changed preemption level from %d to %d", type, preemption_level, sleh_get_preemption_level()); 2681 } 2682#endif 2683} 2684 2685void 2686sleh_serror(arm_context_t *context, uint64_t esr, vm_offset_t far) 2687{ 2688 task_vtimer_check(current_thread()); 2689 2690 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START, 2691 esr, VM_KERNEL_ADDRHIDE(far)); 2692 arm_saved_state_t *state = &context->ss; 2693#if MACH_ASSERT 2694 int preemption_level = sleh_get_preemption_level(); 2695#endif 2696 2697 if (PSR64_IS_USER(get_saved_state_cpsr(state))) { 2698 /* Sanitize FAR (only if we came from userspace) */ 2699 saved_state64(state)->far = 0; 2700 } 2701 2702 ASSERT_CONTEXT_SANITY(context); 2703 arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC); 2704#if MACH_ASSERT 2705 if (preemption_level != sleh_get_preemption_level()) { 2706 panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level()); 2707 } 2708#endif 2709 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END, 2710 esr, VM_KERNEL_ADDRHIDE(far)); 2711} 2712 2713void 2714mach_syscall_trace_exit(unsigned int retval, 2715 unsigned int call_number) 2716{ 2717 KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE, 2718 MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) | 2719 DBG_FUNC_END, retval, 0, 0, 0, 0); 2720} 2721 2722__attribute__((noreturn)) 2723void 2724thread_syscall_return(kern_return_t error) 2725{ 2726 thread_t thread; 2727 struct arm_saved_state *state; 2728 2729 thread = current_thread(); 2730 state = get_user_regs(thread); 2731 2732 assert(is_saved_state64(state)); 2733 saved_state64(state)->x[0] = error; 2734 2735#if MACH_ASSERT 2736 kern_allocation_name_t 2737 prior __assert_only = thread_get_kernel_state(thread)->allocation_name; 2738 assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior)); 2739#endif /* MACH_ASSERT */ 2740 2741 if (kdebug_enable) { 2742 /* Invert syscall number (negative for a mach syscall) */ 2743 mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state)); 2744 } 2745 2746 thread_exception_return(); 2747} 2748 2749void 2750syscall_trace( 2751 struct arm_saved_state * regs __unused) 2752{ 2753 /* kprintf("syscall: %d\n", saved_state64(regs)->x[16]); */ 2754} 2755 2756static void 2757sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type) 2758{ 2759 const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state)); 2760 2761 if (is_user == true) { 2762 /* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */ 2763 saved_state64(state)->far = 0; 2764 } 2765 2766 recount_enter_interrupt(); 2767 2768 task_vtimer_check(current_thread()); 2769 2770 uint64_t pc = is_user ? get_saved_state_pc(state) : 2771 VM_KERNEL_UNSLIDE(get_saved_state_pc(state)); 2772 2773 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START, 2774 0, pc, is_user, type); 2775} 2776 2777static void 2778sleh_interrupt_handler_epilogue(void) 2779{ 2780#if KPERF 2781 kperf_interrupt(); 2782#endif /* KPERF */ 2783 KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END); 2784 recount_leave_interrupt(); 2785} 2786 2787void 2788sleh_invalid_stack(arm_context_t *context, uint64_t esr __unused, vm_offset_t far __unused) 2789{ 2790 thread_t thread = current_thread(); 2791 vm_offset_t kernel_stack_bottom, sp; 2792 2793 sp = get_saved_state_sp(&context->ss); 2794 vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr; 2795 kernel_stack_bottom = round_page(kstackptr) - KERNEL_STACK_SIZE; 2796 2797 if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) { 2798 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss); 2799 } 2800 2801 panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption or early boot).", &context->ss); 2802} 2803 2804 2805#if MACH_ASSERT 2806static int trap_handled; 2807static const char * 2808handle_recoverable_kernel_trap( 2809 __unused void *tstate, 2810 uint16_t comment) 2811{ 2812 assert(comment == TEST_RECOVERABLE_SOFT_TRAP); 2813 2814 printf("Recoverable trap handled.\n"); 2815 trap_handled = 1; 2816 2817 return NULL; 2818} 2819 2820KERNEL_BRK_DESCRIPTOR_DEFINE(test_desc, 2821 .type = TRAP_TELEMETRY_TYPE_KERNEL_BRK_TEST, 2822 .base = TEST_RECOVERABLE_SOFT_TRAP, 2823 .max = TEST_RECOVERABLE_SOFT_TRAP, 2824 .options = BRK_TELEMETRY_OPTIONS_RECOVERABLE_DEFAULT( 2825 /* enable_telemetry */ false), 2826 .handle_breakpoint = handle_recoverable_kernel_trap); 2827 2828static int 2829recoverable_kernel_trap_test(__unused int64_t in, int64_t *out) 2830{ 2831 ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP); 2832 2833 *out = trap_handled; 2834 return 0; 2835} 2836 2837SYSCTL_TEST_REGISTER(recoverable_kernel_trap, recoverable_kernel_trap_test); 2838 2839#endif 2840 2841#if CONFIG_SPTM 2842/** 2843 * Evaluate the panic lockdown policy for a synchronous EL1 SP0 exception 2844 * 2845 * Returns true if panic lockdown should be initiated (but does not itself do 2846 * so) 2847 */ 2848__SECURITY_STACK_DISALLOWED_PUSH 2849bool 2850sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr, uint64_t elr, 2851 uint64_t far, uint64_t spsr) 2852{ 2853 const esr_exception_class_t class = ESR_EC(esr); 2854 const bool any_exceptions_masked = spsr & DAIF_STANDARD_DISABLE; 2855 2856 switch (class) { 2857 case ESR_EC_PC_ALIGN: /* PC misaligned (should never happen) */ 2858 case ESR_EC_IABORT_EL1: /* Potential iPAC failure (poisoned PC) */ 2859 case ESR_EC_PAC_FAIL: { /* FPAC fail */ 2860 return true; 2861 } 2862 2863 case ESR_EC_BRK_AARCH64: { 2864 /* 2865 * Breakpoints are used on non-FPAC systems to signal some PAC failures 2866 */ 2867#if HAS_TELEMETRY_KERNEL_BRK 2868 const struct kernel_brk_descriptor *desc; 2869 desc = find_kernel_brk_descriptor_by_comment(ISS_BRK_COMMENT(esr)); 2870 if (desc && desc->options.recoverable) { 2871 /* 2872 * We matched a breakpoint and it's recoverable, skip lockdown. 2873 */ 2874 return false; 2875 } 2876#endif /* HAS_TELEMETRY_KERNEL_BRK */ 2877 2878 /* 2879 * If we don't support telemetry breakpoints and/or didn't match a 2880 * recoverable breakpoint, the exception is fatal. 2881 */ 2882 return true; 2883 } 2884 2885 case ESR_EC_DABORT_EL1: { 2886 const struct copyio_recovery_entry *cre = 2887 find_copyio_recovery_entry(elr); 2888 if (cre) { 2889 2890 /* 2891 * copyio faults are recoverable regardless of whether or not 2892 * exceptions are masked. 2893 */ 2894 return false; 2895 } 2896 2897 2898 2899 /* 2900 * Heuristic: if FAR != XPAC(FAR), the pointer was likely corrupted 2901 * due to PAC. 2902 */ 2903 const uint64_t far_stripped = 2904 (uint64_t)ptrauth_strip((void *)far, ptrauth_key_asda); 2905 2906 if (far != far_stripped) { 2907 /* potential dPAC failure (poisoined address) */ 2908 return true; 2909 } 2910 2911 if (any_exceptions_masked && startup_phase >= STARTUP_SUB_LOCKDOWN) { 2912 /* 2913 * Any data abort taken with exceptions masked is fatal if we're 2914 * past early boot. 2915 */ 2916 return true; 2917 } 2918 2919 return false; 2920 } 2921 2922 case ESR_EC_UNCATEGORIZED: { 2923 /* Undefined instruction (GDBTRAP for stackshots, etc.) */ 2924 return false; 2925 } 2926 2927 case ESR_EC_BTI_FAIL: { 2928 /* Kernel BTI exceptions are always fatal */ 2929 return true; 2930 } 2931 2932 default: { 2933 if (!any_exceptions_masked) { 2934 /* 2935 * When exceptions are not masked, we default-allow exceptions. 2936 */ 2937 return false; 2938 } 2939 2940 if (startup_phase < STARTUP_SUB_LOCKDOWN) { 2941 /* 2942 * Ignore early boot exceptions even if exceptions are masked. 2943 */ 2944 return false; 2945 } 2946 2947 /* Default-deny all others when exceptions are masked */ 2948 return true; 2949 } 2950 } 2951} 2952__SECURITY_STACK_DISALLOWED_POP 2953#endif /* CONFIG_SPTM */