Second-level exception handlers

osfmk/arm64/sleh.c · 2954 lines · browse source

The C second-level exception handlers. sleh_synchronous decodes ESR_EL1 and routes each exception class; sleh_irq, sleh_fiq and sleh_serror handle the asynchronous legs.

copyio_recovery_offset source

static inline ptrdiff_t
copyio_recovery_offset(uintptr_t addr)
{
	return (ptrdiff_t)(addr - (uintptr_t)copyio_recover_table);
}

copyio_recovery_addr source

static inline uintptr_t
copyio_recovery_addr(ptrdiff_t offset)
{
	return (uintptr_t)copyio_recover_table + (uintptr_t)offset;
}

find_copyio_recovery_entry source

static inline struct copyio_recovery_entry *
find_copyio_recovery_entry(uint64_t pc)
{
	ptrdiff_t offset = copyio_recovery_offset(pc);
	struct copyio_recovery_entry *e;

	for (e = copyio_recover_table; e < copyio_recover_table_end; e++) {
		if (offset >= e->cre_start && offset < e->cre_end) {
			return e;
		}
	}

	return NULL;
}

is_vm_fault source

static inline int
is_vm_fault(fault_status_t status)
{
	switch (status) {
	case FSC_TRANSLATION_FAULT_L0:
	case FSC_TRANSLATION_FAULT_L1:
	case FSC_TRANSLATION_FAULT_L2:
	case FSC_TRANSLATION_FAULT_L3:
	case FSC_ACCESS_FLAG_FAULT_L1:
	case FSC_ACCESS_FLAG_FAULT_L2:
	case FSC_ACCESS_FLAG_FAULT_L3:
	case FSC_PERMISSION_FAULT_L1:
	case FSC_PERMISSION_FAULT_L2:
	case FSC_PERMISSION_FAULT_L3:
		return TRUE;
	default:
		return FALSE;
	}
}

is_translation_fault source

static inline int
is_translation_fault(fault_status_t status)
{
	switch (status) {
	case FSC_TRANSLATION_FAULT_L0:
	case FSC_TRANSLATION_FAULT_L1:
	case FSC_TRANSLATION_FAULT_L2:
	case FSC_TRANSLATION_FAULT_L3:
		return TRUE;
	default:
		return FALSE;
	}
}

is_permission_fault source

static inline int
is_permission_fault(fault_status_t status)
{
	switch (status) {
	case FSC_PERMISSION_FAULT_L1:
	case FSC_PERMISSION_FAULT_L2:
	case FSC_PERMISSION_FAULT_L3:
		return TRUE;
	default:
		return FALSE;
	}
}

is_alignment_fault source

static inline int
is_alignment_fault(fault_status_t status)
{
	return status == FSC_ALIGNMENT_FAULT;
}

is_parity_error source

static inline int
is_parity_error(fault_status_t status)
{
	switch (status) {
#if defined(ARM64_BOARD_CONFIG_T6020)
		/*
		 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented,
		 * FSC_SYNC_PARITY_X can be reported for data and instruction aborts
		 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
		 */
#else
	/*
	 * TODO: According to ARM ARM, Async Parity (0b011001) is a DFSC that is
	 * only applicable to AArch32 HSR register. Can this be removed?
	 */
	case FSC_ASYNC_PARITY:
	case FSC_SYNC_PARITY:
	case FSC_SYNC_PARITY_TT_L1:
	case FSC_SYNC_PARITY_TT_L2:
	case FSC_SYNC_PARITY_TT_L3:
		return TRUE;
#endif
	default:
		return FALSE;
	}
}

is_sync_external_abort source

static inline int
is_sync_external_abort(fault_status_t status)
{
	switch (status) {
#if defined(ARM64_BOARD_CONFIG_T6020)
	/*
	 * H14 Erratum (rdar://61553243): Despite having FEAT_RAS implemented,
	 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts
	 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
	 */
	case FSC_SYNC_PARITY:
#endif /* defined(ARM64_BOARD_CONFIG_T6020) */
	case FSC_SYNC_EXT_ABORT:
		return TRUE;
	default:
		return FALSE;
	}
}

is_table_walk_error source

static inline int
is_table_walk_error(fault_status_t status)
{
	switch (status) {
	case FSC_SYNC_EXT_ABORT_TT_L1:
	case FSC_SYNC_EXT_ABORT_TT_L2:
	case FSC_SYNC_EXT_ABORT_TT_L3:
#if defined(ARM64_BOARD_CONFIG_T6020)
	/*
	 * H14 Erratum(rdar://61553243): Despite having FEAT_RAS implemented,
	 * FSC_SYNC_PARITY_x can be reported for data and instruction aborts
	 * and should be interpreted as FSC_SYNC_EXT_ABORT_x
	 */
	case FSC_SYNC_PARITY_TT_L1:
	case FSC_SYNC_PARITY_TT_L2:
	case FSC_SYNC_PARITY_TT_L3:
#endif /* defined(ARM64_BOARD_CONFIG_T6020) */
		return TRUE;
	default:
		return FALSE;
	}
}

is_servicible_fault source

static inline int
is_servicible_fault(fault_status_t status, uint64_t esr)
{
#pragma unused(esr)
	return is_vm_fault(status);
}

arm64_implementation_specific_error source

__dead2 __unused
static void
arm64_implementation_specific_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far)
{
#pragma unused (state, esr, far)
	panic_plain("Unhandled implementation specific error\n");
}

kernel_integrity_error_handler source

static void
kernel_integrity_error_handler(uint64_t esr, vm_offset_t far)
{
#if defined(KERNEL_INTEGRITY_WT)
#if (DEVELOPMENT || DEBUG)
	if (ESR_WT_SERROR(esr)) {
		switch (ESR_WT_REASON(esr)) {
		case WT_REASON_INTEGRITY_FAIL:
			panic_plain("Kernel integrity, violation in frame 0x%016lx.", far);
		case WT_REASON_BAD_SYSCALL:
			panic_plain("Kernel integrity, bad syscall.");
		case WT_REASON_NOT_LOCKED:
			panic_plain("Kernel integrity, not locked.");
		case WT_REASON_ALREADY_LOCKED:
			panic_plain("Kernel integrity, already locked.");
		case WT_REASON_SW_REQ:
			panic_plain("Kernel integrity, software request.");
		case WT_REASON_PT_INVALID:
			panic_plain("Kernel integrity, encountered invalid TTE/PTE while "
			    "walking 0x%016lx.", far);
		case WT_REASON_PT_VIOLATION:
			panic_plain("Kernel integrity, violation in mapping 0x%016lx.",
			    far);
		case WT_REASON_REG_VIOLATION:
			panic_plain("Kernel integrity, violation in system register %d.",
			    (unsigned) far);
		default:
			panic_plain("Kernel integrity, unknown (esr=0x%08llx).", esr);
		}
	}
… more in source

arm64_platform_error source

static void
arm64_platform_error(arm_saved_state_t *state, uint64_t esr, vm_offset_t far, platform_error_source_t source)
{
#if CONFIG_KERNEL_INTEGRITY
	kernel_integrity_error_handler(esr, far);
#endif

	(void)source;
	cpu_data_t *cdp = getCpuDatap();

	if (PE_handle_platform_error(far)) {
		return;
	} else if (cdp->platform_error_handler != NULL) {
		cdp->platform_error_handler(cdp->cpu_id, far);
	} else {
		arm64_implementation_specific_error(state, esr, far);
	}
}

panic_with_thread_kernel_state source

void
panic_with_thread_kernel_state(const char *msg, arm_saved_state_t *ss)
{
	boolean_t ss_valid;

	ss_valid = is_saved_state64(ss);
	arm_saved_state64_t *state = saved_state64(ss);

	os_atomic_cmpxchg(&original_faulting_state, NULL, state, seq_cst);

	// rdar://80659177
	// Read SoCD tracepoints up to twice — once the first time we call panic and
	// another time if we encounter a nested panic after that.
	static int twice = 2;
	if (twice > 0) {
		twice--;
		SOCD_TRACE_XNU(KERNEL_STATE_PANIC,
		    SOCD_TRACE_MODE_STICKY_TRACEPOINT,
		    ADDR(state->pc),
		    PACK_LSB(VALUE(state->lr), VALUE(ss_valid)),
		    PACK_2X32(VALUE(state->esr), VALUE(state->cpsr)),
		    VALUE(state->far));
	}



	panic_plain("%s at pc 0x%016llx, lr 0x%016llx (saved state: %p%s)\n"
	    "\t  x0:  0x%016llx x1:  0x%016llx  x2:  0x%016llx  x3:  0x%016llx\n"
	    "\t  x4:  0x%016llx x5:  0x%016llx  x6:  0x%016llx  x7:  0x%016llx\n"
	    "\t  x8:  0x%016llx x9:  0x%016llx  x10: 0x%016llx  x11: 0x%016llx\n"
… more in source

sleh_synchronous_sp1 source

void
sleh_synchronous_sp1(arm_context_t *context, uint64_t esr, vm_offset_t far __unused)
{
	esr_exception_class_t  class = ESR_EC(esr);
	arm_saved_state_t    * state = &context->ss;

	switch (class) {
	case ESR_EC_UNCATEGORIZED:
	{
#if (DEVELOPMENT || DEBUG)
		uint32_t instr = *((uint32_t*)get_saved_state_pc(state));
		if (IS_ARM_GDB_TRAP(instr)) {
			DebuggerCall(EXC_BREAKPOINT, state);
		}
		OS_FALLTHROUGH; // panic if we return from the debugger
#else
		panic_with_thread_kernel_state("Unexpected debugger trap while SP1 selected", state);
#endif /* (DEVELOPMENT || DEBUG) */
	}
	default:
		panic_with_thread_kernel_state("Synchronous exception taken while SP1 selected", state);
	}
}

thread_exception_return source

__attribute__((noreturn))
void
thread_exception_return()
{
	thread_t thread = current_thread();
	if (thread->machine.exception_trace_code != 0) {
		KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
		    MACHDBG_CODE(DBG_MACH_EXCP_SYNC_ARM, thread->machine.exception_trace_code) | DBG_FUNC_END, 0, 0, 0, 0, 0);
		thread->machine.exception_trace_code = 0;
	}


#if KASAN_TBI
	kasan_unpoison_curstack(true);
#endif /* KASAN_TBI */
	arm64_thread_exception_return();
	__builtin_unreachable();
}

task_vtimer_check source

check whether task vtimers are running and set thread and CPU BSD AST must be called with interrupts masked so updates of fields are atomic must be emitted inline to avoid generating an FBT probe on the exception path

__attribute__((__always_inline__))
static inline void
task_vtimer_check(thread_t thread)
{
	task_t task = get_threadtask_early(thread);

	if (__improbable(task != NULL && task->vtimers)) {
		thread_ast_set(thread, AST_BSD);
		thread->machine.CpuDatap->cpu_pending_ast |= AST_BSD;
	}
}

sleh_get_preemption_level source

A version of get_preemption_level() that works in early boot. If an exception is raised in early boot before the initial thread has been set up, then calling get_preemption_level() in the SLEH will trigger an infinitely-recursing exception. This function handles this edge case.

static inline int
sleh_get_preemption_level(void)
{
	if (__improbable(current_thread() == NULL)) {
		return 0;
	}
	return get_preemption_level();
}

is_platform_error source

MACH_ASSERT

static inline bool
is_platform_error(uint64_t esr)
{
	esr_exception_class_t class = ESR_EC(esr);
	uint32_t iss = ESR_ISS(esr);
	fault_status_t fault_code;

	if (class == ESR_EC_DABORT_EL0 || class == ESR_EC_DABORT_EL1) {
		fault_code = ISS_DA_FSC(iss);
	} else if (class == ESR_EC_IABORT_EL0 || class == ESR_EC_IABORT_EL1) {
		fault_code = ISS_IA_FSC(iss);
	} else {
		return false;
	}

	return is_parity_error(fault_code) || is_sync_external_abort(fault_code) ||
	       is_table_walk_error(fault_code);
}

sleh_synchronous source

void
sleh_synchronous(arm_context_t *context, uint64_t esr, vm_offset_t far, __unused bool did_initiate_panic_lockdown)
{
	esr_exception_class_t  class   = ESR_EC(esr);
	arm_saved_state_t    * state   = &context->ss;
	thread_t               thread  = current_thread();
#if MACH_ASSERT
	int                    preemption_level = sleh_get_preemption_level();
#endif
	expected_fault_handler_t expected_fault_handler = NULL;
#ifdef CONFIG_XNUPOST
	expected_fault_handler_t saved_expected_fault_handler = NULL;
	uintptr_t saved_expected_fault_addr = 0;
	uintptr_t saved_expected_fault_pc = 0;
#endif /* CONFIG_XNUPOST */

	ASSERT_CONTEXT_SANITY(context);

	task_vtimer_check(thread);

#if CONFIG_DTRACE
	/*
	 * Handle kernel DTrace probes as early as possible to minimize the likelihood
	 * that this path will itself trigger a DTrace probe, which would lead to infinite
	 * probe recursion.
	 */
	if (__improbable((class == ESR_EC_UNCATEGORIZED) && tempDTraceTrapHook &&
	    (tempDTraceTrapHook(EXC_BAD_INSTRUCTION, state, 0, 0) == KERN_SUCCESS))) {
#if CONFIG_SPTM
		if (__improbable(did_initiate_panic_lockdown)) {
… more in source

handle_uncategorized source

Uncategorized exceptions are a catch-all for general execution errors. ARM64_TODO: For now, we assume this is for undefined instruction exceptions.

static void
handle_uncategorized(arm_saved_state_t *state)
{
	exception_type_t           exception = EXC_BAD_INSTRUCTION;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_UNDEFINED};
	mach_msg_type_number_t     numcodes  = 2;
	uint32_t                   instr     = 0;

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));

#if CONFIG_DTRACE

	if (PSR64_IS_USER64(get_saved_state_cpsr(state))) {
		/*
		 * For a 64bit user process, we care about all 4 bytes of the
		 * instr.
		 */
		if (instr == FASTTRAP_ARM64_INSTR || instr == FASTTRAP_ARM64_RET_INSTR) {
			if (dtrace_user_probe(state) == KERN_SUCCESS) {
				return;
			}
		}
	} else if (PSR64_IS_USER32(get_saved_state_cpsr(state))) {
		/*
		 * For a 32bit user process, we check for thumb mode, in
		 * which case we only care about a 2 byte instruction length.
		 * For non-thumb mode, we care about all 4 bytes of the instructin.
		 */
		if (get_saved_state_cpsr(state) & PSR64_MODE_USER32_THUMB) {
			if (((uint16_t)instr == FASTTRAP_THUMB32_INSTR) ||
… more in source

ptrauth_key_to_string source

static inline const char *
ptrauth_key_to_string(ptrauth_key key)
{
	switch (key) {
	case ptrauth_key_asia:
		return "IA";
	case ptrauth_key_asib:
		return "IB";
	case ptrauth_key_asda:
		return "DA";
	case ptrauth_key_asdb:
		return "DB";
	default:
		__builtin_unreachable();
	}
}

ptrauth_handle_brk_trap source

static const char *
ptrauth_handle_brk_trap(void *tstate, uint16_t comment)
{
	kernel_panic_reason_t pr = PERCPU_GET(panic_reason);
	arm_saved_state_t *state = (arm_saved_state_t *)tstate;

	ptrauth_key key = (ptrauth_key)(comment - PTRAUTH_TRAP_START);
	const char *key_str = ptrauth_key_to_string(key);

	snprintf(pr->buf, sizeof(pr->buf),
	    "Break 0x%04X instruction exception from kernel. "
	    "Ptrauth failure with %s key resulted in 0x%016llx",
	    comment, key_str, saved_state64(state)->x[16]);

	return pr->buf;
}

xnu_soft_trap_handle_breakpoint source

static const char *
xnu_soft_trap_handle_breakpoint(
	void              *tstate,
	uint16_t          comment)
{
#if CONFIG_UBSAN_MINIMAL
	if (comment == UBSAN_SOFT_TRAP_SIGNED_OF) {
		ubsan_handle_brk_trap(tstate, comment);
	}
#else
	(void)tstate;
#endif

	if (comment == CLANG_SOFT_TRAP_BOUND_CHK) {
		os_atomic_inc(&bound_chk_violations_event, relaxed);
	}
	return NULL;
}

xnu_hard_trap_handle_breakpoint source

HAS_TELEMETRY_KERNEL_BRK

static const char *
xnu_hard_trap_handle_breakpoint(void *tstate, uint16_t comment)
{
	kernel_panic_reason_t pr = PERCPU_GET(panic_reason);
	arm_saved_state64_t *state = saved_state64(tstate);

	switch (comment) {
	case XNU_HARD_TRAP_SAFE_UNLINK:
		snprintf(pr->buf, sizeof(pr->buf),
		    "panic: corrupt list around element %p",
		    (void *)state->x[8]);
		return pr->buf;

	case XNU_HARD_TRAP_STRING_CHK:
		return "panic: string operation caused an overflow";

	case XNU_HARD_TRAP_ASSERT_FAILURE:
		/*
		 * Read the implicit assert arguments, see:
		 * ML_TRAP_REGISTER_1: x8
		 * ML_TRAP_REGISTER_2: x16
		 * ML_TRAP_REGISTER_3: x17
		 */
		panic_assert_format(pr->buf, sizeof(pr->buf),
		    (struct mach_assert_hdr *)state->x[8],
		    state->x[16], state->x[17]);
		return pr->buf;

	default:
		return NULL;
… more in source

handle_kernel_breakpoint source

handle_kernel_breakpoint(arm_saved_state_t *state, uint64_t esr)
{
	uint16_t comment = ISS_BRK_COMMENT(esr);
	const struct kernel_brk_descriptor *desc;
	const char *msg = NULL;

	desc = find_kernel_brk_descriptor_by_comment(comment);

	if (!desc) {
		goto brk_out;
	}

#if HAS_TELEMETRY_KERNEL_BRK
	if (desc->options.enable_trap_telemetry) {
		trap_telemetry_report_exception(
			/* trap_type   */ desc->type,
			/* trap_code   */ comment,
			/* options     */ desc->options.telemetry_options,
			/* saved_state */ (void *)state);
	}
#endif

	if (desc->handle_breakpoint) {
		msg = desc->handle_breakpoint(state, comment);
	}

#if HAS_TELEMETRY_KERNEL_BRK
	/* Still alive? Check if we should recover. */
	if (desc->options.recoverable) {
		add_saved_state_pc(state, 4);
… more in source

find_user_brk_descriptor_by_comment source

const static inline user_brk_label_range_descriptor_t *
find_user_brk_descriptor_by_comment(uint16_t comment)
{
	for (int desc_idx = 0; desc_idx < user_brk_descriptor_count; desc_idx++) {
		const user_brk_label_range_descriptor_t* des = &user_brk_descriptors[desc_idx];
		if (comment >= des->base && comment <= des->max) {
			return des;
		}
	}

	return NULL;
}

handle_user_breakpoint source

static void
handle_user_breakpoint(arm_saved_state_t *state, uint64_t esr __unused)
{
	exception_type_t           exception = EXC_BREAKPOINT;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_BREAKPOINT};
	mach_msg_type_number_t     numcodes  = 2;

	if (ESR_EC(esr) == ESR_EC_BRK_AARCH64) {
		/*
		 * Consult the trap labels we know about to decide whether userspace
		 * should be given the opportunity to handle the exception.
		 */
		uint16_t brk_label = ISS_BRK_COMMENT(esr);
		const struct user_brk_label_range_descriptor* descriptor = find_user_brk_descriptor_by_comment(brk_label);
		/*
		 * Note it's no problem if we don't recognize the label.
		 * In this case we'll just go through normal exception delivery.
		 */
		if (descriptor != NULL) {
			exception |= EXC_MAY_BE_UNRECOVERABLE_BIT;

#if __has_feature(ptrauth_calls)
			/*
			 * We have additional policy specifically for PAC violations.
			 * To make the rest of the code easier to follow, don't set
			 * EXC_MAY_BE_UNRECOVERABLE_BIT here and just set EXC_PTRAUTH_BIT instead.
			 * Conceptually a PAC failure is absolutely 'maybe unrecoverable', but it's
			 * not really worth excising the discrepency from the plumbing.
			 */
			if (descriptor->base == PTRAUTH_TRAP_START) {
… more in source

handle_watchpoint source

static void
handle_watchpoint(vm_offset_t fault_addr)
{
	exception_type_t           exception = EXC_BREAKPOINT;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_DA_DEBUG};
	mach_msg_type_number_t     numcodes  = 2;

	codes[1] = fault_addr;
	exception_triage(exception, codes, numcodes);
	__builtin_unreachable();
}

handle_abort source

static void
handle_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
    abort_inspector_t inspect_abort, abort_handler_t handler, expected_fault_handler_t expected_fault_handler)
{
	fault_status_t fault_code;
	vm_prot_t      fault_type;

	inspect_abort(ESR_ISS(esr), &fault_code, &fault_type);
	handler(state, esr, fault_addr, fault_code, fault_type, expected_fault_handler);
}

inspect_instruction_abort source

static void
inspect_instruction_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
{
	getCpuDatap()->cpu_stat.instr_ex_cnt++;
	*fault_code = ISS_IA_FSC(iss);
	*fault_type = (VM_PROT_READ | VM_PROT_EXECUTE);
}

inspect_data_abort source

static void
inspect_data_abort(uint32_t iss, fault_status_t *fault_code, vm_prot_t *fault_type)
{
	getCpuDatap()->cpu_stat.data_ex_cnt++;
	*fault_code = ISS_DA_FSC(iss);

	/*
	 * Cache maintenance operations always report faults as write access.
	 * Change these to read access, unless they report a permission fault.
	 * Only certain cache maintenance operations (e.g. 'dc ivac') require write
	 * access to the mapping, but if a cache maintenance operation that only requires
	 * read access generates a permission fault, then we will not be able to handle
	 * the fault regardless of whether we treat it as a read or write fault.
	 */
	if ((iss & ISS_DA_WNR) && (!(iss & ISS_DA_CM) || is_permission_fault(*fault_code))) {
		*fault_type = (VM_PROT_READ | VM_PROT_WRITE);
	} else {
		*fault_type = (VM_PROT_READ);
	}
}

fault_addr_bitmask source

static inline uint64_t
fault_addr_bitmask(unsigned int bit_from, unsigned int bit_to)
{
	return ((1ULL << (bit_to - bit_from + 1)) - 1) << bit_from;
}

fault_addr_bit source

static inline bool
fault_addr_bit(vm_offset_t fault_addr, unsigned int bit)
{
	return (bool)((fault_addr >> bit) & 1);
}

user_fault_matches_pac_error_code source

Determines whether a fault address taken at EL0 contains a PAC error code corresponding to the specified kind of ptrauth key.

static bool
user_fault_matches_pac_error_code(vm_offset_t fault_addr, uint64_t pc, bool data_key)
{
	if (gARM_FEAT_FPACCOMBINE) {
		/*
		 * CPUs with FPACCOMBINE always raise PAC Fail exceptions during
		 * PAC failure.  If the CPU took any other kind of exception, we
		 * can rule out PAC as the root cause.
		 */
		return false;
	}

	if (data_key && gARM_FEAT_FPAC) {
		uint32_t instr;
		int err = copyin(pc, (char *)&instr, sizeof(instr));
		if (!err && !ARM64_INSTR_IS_LDRAx(instr)) {
			/*
			 * On FPAC-enabled devices, PAC failure can only cause
			 * data aborts during "combined" LDRAx instructions.  If
			 * PAC fails during a discrete AUTxx + LDR/STR
			 * instruction sequence, then the AUTxx instruction
			 * raises a PAC Fail exception rather than poisoning its
			 * output address.
			 *
			 * In principle the same logic applies to instruction
			 * aborts.  But we have no way to identify the exact
			 * instruction that caused the abort, so we can't tell
			 * if it was a combined branch + auth instruction.
			 */
			return false;
… more in source

user_fault_in_self_restrict_mode source

Determines whether the userland thread has a JIT region in RW mode, TPRO in RW mode, or JCTL_EL0 in pointer signing mode. A fault in any of these trusted code paths may indicate an attack on WebKit. Rather than letting a potentially-compromised process try to handle the exception, it will be killed by the kernel and a crash report will be generated.

static self_restrict_mode_t
user_fault_in_self_restrict_mode(thread_t thread __unused)
{
	self_restrict_mode_t out = SELF_RESTRICT_NONE;

	return out;
}

handle_pc_align source

static void
handle_pc_align(arm_saved_state_t *ss)
{
	exception_type_t exc;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t numcodes = 2;

	if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
		panic_with_thread_kernel_state("PC alignment exception from kernel.", ss);
	}

	exc = EXC_BAD_ACCESS;
#if __has_feature(ptrauth_calls)
	uint64_t pc = get_saved_state_pc(ss);
	if (user_fault_matches_pac_error_code(pc, pc, false)) {
		exc |= EXC_PTRAUTH_BIT;
	}
#endif /* __has_feature(ptrauth_calls) */

	codes[0] = EXC_ARM_DA_ALIGN;
	codes[1] = get_saved_state_pc(ss);

	exception_triage(exc, codes, numcodes);
	__builtin_unreachable();
}

handle_sp_align source

static void
handle_sp_align(arm_saved_state_t *ss)
{
	exception_type_t exc;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t numcodes = 2;

	if (!PSR64_IS_USER(get_saved_state_cpsr(ss))) {
		panic_with_thread_kernel_state("SP alignment exception from kernel.", ss);
	}

	exc = EXC_BAD_ACCESS;
#if __has_feature(ptrauth_calls)
	if (user_fault_matches_pac_error_code(get_saved_state_sp(ss), get_saved_state_pc(ss), true)) {
		exc |= EXC_PTRAUTH_BIT;
	}
#endif /* __has_feature(ptrauth_calls) */

	codes[0] = EXC_ARM_SP_ALIGN;
	codes[1] = get_saved_state_sp(ss);

	exception_triage(exc, codes, numcodes);
	__builtin_unreachable();
}

handle_wf_trap source

static void
handle_wf_trap(arm_saved_state_t *state)
{
	exception_type_t exc;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t numcodes = 2;
	uint32_t instr = 0;

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));

	exc = EXC_BAD_INSTRUCTION;
	codes[0] = EXC_ARM_UNDEFINED;
	codes[1] = instr;

	exception_triage(exc, codes, numcodes);
	__builtin_unreachable();
}

handle_fp_trap source

static void
handle_fp_trap(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t exc = EXC_ARITHMETIC;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t numcodes = 2;
	uint32_t instr = 0;

	if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
		panic_with_thread_kernel_state("Floating point exception from kernel", state);
	}

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
	codes[1] = instr;

	/* The floating point trap flags are only valid if TFV is set. */
	if (!fp_exceptions_enabled) {
		exc = EXC_BAD_INSTRUCTION;
		codes[0] = EXC_ARM_UNDEFINED;
	} else if (!(esr & ISS_FP_TFV)) {
		codes[0] = EXC_ARM_FP_UNDEFINED;
	} else if (esr & ISS_FP_UFF) {
		codes[0] = EXC_ARM_FP_UF;
	} else if (esr & ISS_FP_OFF) {
		codes[0] = EXC_ARM_FP_OF;
	} else if (esr & ISS_FP_IOF) {
		codes[0] = EXC_ARM_FP_IO;
	} else if (esr & ISS_FP_DZF) {
		codes[0] = EXC_ARM_FP_DZ;
	} else if (esr & ISS_FP_IDF) {
… more in source

handle_alignment_fault_from_user source

handle_alignment_fault_from_user: state: Saved state Attempts to deal with an alignment fault from userspace (possibly by emulating the faulting instruction). If emulation failed due to an unservicable fault, the ESR for that fault will be stored in the recovery_esr field of the thread by the exception code. Returns: -1: Emulation failed (emulation of state/instr not supported) 0: Successfully emulated the instruction EFAULT: Emulation failed (probably due to permissions) EINVAL: Emulation failed (probably due to a bad address)

static int
handle_alignment_fault_from_user(arm_saved_state_t *state, kern_return_t *vmfr)
{
	int ret = -1;

#pragma unused (state)
#pragma unused (vmfr)

	return ret;
}

handle_sme_trap source

static void
handle_sme_trap(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t exc = EXC_BAD_INSTRUCTION;
	mach_exception_data_type_t codes[2] = {EXC_ARM_UNDEFINED};
	mach_msg_type_number_t numcodes = 2;

	if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
		panic("SME exception from kernel, state=%p, esr=%#llx", state, esr);
	}
	if (!arm_sme_version()) {
		/*
		 * If SME is disabled in software but userspace executes an SME
		 * instruction anyway, then the CPU will still raise an
		 * SME-specific trap.  Triage it as if the CPU raised an
		 * undefined-instruction trap.
		 */
		exception_triage(exc, codes, numcodes);
		__builtin_unreachable();
	}

	if (ISS_SME_SMTC(ESR_ISS(esr)) == ISS_SME_SMTC_CAPCR) {
		thread_t thread = current_thread();
		switch (machine_thread_sme_state_alloc(thread)) {
		case KERN_SUCCESS:
			return;


		default:
			panic("Failed to allocate SME state for thread %p", thread);
… more in source

handle_sw_step_debug source

HAS_ARM_FEAT_SME

static void
handle_sw_step_debug(arm_saved_state_t *state)
{
	thread_t thread = current_thread();
	exception_type_t exc;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t numcodes = 2;

	if (!PSR64_IS_USER(get_saved_state_cpsr(state))) {
		panic_with_thread_kernel_state("SW_STEP_DEBUG exception from kernel.", state);
	}

	// Disable single step and unmask interrupts (in the saved state, anticipating next exception return)
	if (thread->machine.DebugData != NULL) {
		thread->machine.DebugData->uds.ds64.mdscr_el1 &= ~0x1;
	} else {
		panic_with_thread_kernel_state("SW_STEP_DEBUG exception thread DebugData is NULL.", state);
	}

	mask_user_saved_state_cpsr(thread->machine.upcb, 0, PSR64_SS | DAIF_ALL);

	// Special encoding for gdb single step event on ARM
	exc = EXC_BREAKPOINT;
	codes[0] = 1;
	codes[1] = 0;

	exception_triage(exc, codes, numcodes);
	__builtin_unreachable();
}

handle_user_abort source

MACH_ASSERT

static void
handle_user_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
    fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
{
	exception_type_t           exc      = EXC_BAD_ACCESS;
	mach_exception_data_type_t codes[2];
	mach_msg_type_number_t     numcodes = 2;
	thread_t                   thread   = current_thread();

	(void)expected_fault_handler;

	if (__improbable(!SPSR_INTERRUPTS_ENABLED(get_saved_state_cpsr(state)))) {
		panic_with_thread_kernel_state("User abort from non-interruptible context", state);
	}

	thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling abort from userspace */

	if (!is_servicible_fault(fault_code, esr) &&
	    thread->t_rr_state.trr_fault_state != TRR_FAULT_NONE) {
		thread_reset_pcs_done_faulting(thread);
	}

	if (is_vm_fault(fault_code)) {
		vm_map_t        map = thread->map;
		vm_offset_t     vm_fault_addr = fault_addr;
		kern_return_t   result = KERN_FAILURE;

		assert(map != kernel_map);

		if (!(fault_type & VM_PROT_EXECUTE)) {
… more in source

panic_on_invalid_recovery_handler source

Panic because the kernel abort handler tried to apply a recovery handler that isn't inside copyio_recover_table[]. @param state original saved-state @param recover invalid recovery handler

__attribute__((noreturn, used))
static void
panic_on_invalid_recovery_handler(arm_saved_state_t *state, struct copyio_recovery_entry *recover)
{
	panic("attempt to set invalid recovery handler %p on kernel saved-state %p", recover, state);
}

handle_kernel_abort_recover_with_error_code source

Update a thread saved-state to store an error code in x0 and branch to a copyio recovery handler. @param state original saved-state @param esr ESR_ELx value for the fault taken @param fault_addr FAR_ELx value for the fault taken @param thread target thread @param recover destination copyio recovery handler @param x0 error code to populate into x0

static void
handle_kernel_abort_recover_with_error_code(
	arm_saved_state_t              *state,
	uint64_t                        esr,
	vm_offset_t                     fault_addr,
	thread_t                        thread,
	struct copyio_recovery_entry   *_Nonnull recover,
	uint64_t                       x0)
{
	thread->machine.recover_esr = esr;
	thread->machine.recover_far = fault_addr;
	saved_state64(state)->x[0] = x0;
#if defined(HAS_APPLE_PAC)
	MANIPULATE_SIGNED_THREAD_STATE(state,
	    "adrp	x6, _copyio_recover_table_end@page		\n"
	    "add	x6, x6, _copyio_recover_table_end@pageoff	\n"
	    "cmp	%[recover], x6					\n"
	    "b.lt	1f						\n"
	    "bl		_panic_on_invalid_recovery_handler		\n"
	    "brk	#0						\n"
	    "1:								\n"
	    "adrp	x6, _copyio_recover_table@page			\n"
	    "add	x6, x6, _copyio_recover_table@pageoff		\n"
	    "subs	x7, %[recover], x6				\n"
	    "b.pl	1f						\n"
	    "bl		_panic_on_invalid_recovery_handler		\n"
	    "brk	#0						\n"
	    "1:								\n"
	    "udiv	x8, x7, %[SIZEOF_RECOVER]			\n"
	    "mul	x8, x8, %[SIZEOF_RECOVER]			\n"
… more in source

handle_kernel_abort_recover source

static inline void
handle_kernel_abort_recover(
	arm_saved_state_t              *state,
	uint64_t                        esr,
	vm_offset_t                     fault_addr,
	thread_t                        thread,
	struct copyio_recovery_entry   *_Nonnull recover)
{
	handle_kernel_abort_recover_with_error_code(state, esr, fault_addr, thread, recover, EFAULT);
}

handle_kernel_abort source

static void
handle_kernel_abort(arm_saved_state_t *state, uint64_t esr, vm_offset_t fault_addr,
    fault_status_t fault_code, vm_prot_t fault_type, expected_fault_handler_t expected_fault_handler)
{
	thread_t thread = current_thread();
	struct copyio_recovery_entry *recover = find_copyio_recovery_entry(
		get_saved_state_pc(state));

#ifndef CONFIG_XNUPOST
	(void)expected_fault_handler;
#endif /* CONFIG_XNUPOST */


#if CONFIG_DTRACE
	if (is_vm_fault(fault_code) && thread->t_dtrace_inprobe) { /* Executing under dtrace_probe? */
		if (dtrace_tally_fault(fault_addr)) { /* Should a fault under dtrace be ignored? */
			/*
			 * Point to next instruction, or recovery handler if set.
			 */
			if (recover) {
				handle_kernel_abort_recover(state, esr, VM_USER_STRIP_PTR(fault_addr), thread, recover);
			} else {
				add_saved_state_pc(state, 4);
			}
			return;
		} else {
			panic_with_thread_kernel_state("Unexpected page fault under dtrace_probe", state);
		}
	}
#endif
… more in source

handle_svc source

static void
handle_svc(arm_saved_state_t *state)
{
	int      trap_no = get_saved_state_svc_number(state);
	thread_t thread  = current_thread();
	struct   proc *p;

#define handle_svc_kprintf(x...) /* kprintf("handle_svc: " x) */

#define TRACE_SYSCALL 1
#if TRACE_SYSCALL
	syscall_trace(state);
#endif

	thread->iotier_override = THROTTLE_LEVEL_NONE; /* Reset IO tier override before handling SVC from userspace */

	if (trap_no == (int)PLATFORM_SYSCALL_TRAP_NO) {
		platform_syscall(state);
		panic("Returned from platform_syscall()?");
	}

	current_cached_proc_cred_update();

	if (trap_no < 0) {
		switch (trap_no) {
		case MACH_ARM_TRAP_ABSTIME:
			handle_mach_absolute_time_trap(state);
			return;
		case MACH_ARM_TRAP_CONTTIME:
			handle_mach_continuous_time_trap(state);
… more in source

handle_mach_absolute_time_trap source

static void
handle_mach_absolute_time_trap(arm_saved_state_t *state)
{
	uint64_t now = mach_absolute_time();
	saved_state64(state)->x[0] = now;
}

handle_mach_continuous_time_trap source

static void
handle_mach_continuous_time_trap(arm_saved_state_t *state)
{
	uint64_t now = mach_continuous_time();
	saved_state64(state)->x[0] = now;
}

handle_msr_trap source

__attribute__((noreturn))
static void
handle_msr_trap(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t           exception = EXC_BAD_INSTRUCTION;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_UNDEFINED};
	mach_msg_type_number_t     numcodes  = 2;
	uint32_t                   instr     = 0;

	if (!is_saved_state64(state)) {
		panic("MSR/MRS trap (ESR 0x%llx) from 32-bit state", esr);
	}

	if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
		panic("MSR/MRS trap (ESR 0x%llx) from kernel", esr);
	}

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
	codes[1] = instr;

	exception_triage(exception, codes, numcodes);
	__builtin_unreachable();
}

stringify_gpr source

static void
stringify_gpr(unsigned int r, char reg[4])
{
	switch (r) {
	case 29:
		strncpy(reg, "fp", 4);
		return;

	case 30:
		strncpy(reg, "lr", 4);
		return;

	case 31:
		strncpy(reg, "xzr", 4);
		return;

	default:
		snprintf(reg, 4, "x%u", r);
		return;
	}
}

autxx_instruction_extract_reg source

static void
autxx_instruction_extract_reg(uint32_t instr, char reg[4])
{
	unsigned int rd = ARM64_INSTR_AUTxx_RD_GET(instr);
	stringify_gpr(rd, reg);
}

autix_system_instruction_extract_reg source

static const char *
autix_system_instruction_extract_reg(uint32_t instr)
{
	unsigned int crm_op2 = ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_GET(instr);
	if (crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIA1716 ||
	    crm_op2 == ARM64_INSTR_AUTIx_SYSTEM_CRM_OP2_AUTIB1716) {
		return "x17";
	} else {
		return "lr";
	}
}

bxrax_instruction_extract_reg source

static void
bxrax_instruction_extract_reg(uint32_t instr, char reg[4])
{
	unsigned int rn = ARM64_INSTR_BxRAx_RN_GET(instr);
	stringify_gpr(rn, reg);
}

handle_pac_fail source

static void
handle_pac_fail(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t           exception = EXC_BAD_ACCESS | EXC_PTRAUTH_BIT;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_PAC_FAIL};
	mach_msg_type_number_t     numcodes  = 2;
	uint32_t                   instr     = 0;

	if (!is_saved_state64(state)) {
		panic("PAC failure (ESR 0x%llx) from 32-bit state", esr);
	}

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));

	if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
#define GENERIC_PAC_FAILURE_MSG_FMT "PAC failure from kernel with %s key"
#define AUTXX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while authing %s"
#define BXRAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while branching to %s"
#define RETAX_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT " while returning"
#define GENERIC_MSG_FMT GENERIC_PAC_FAILURE_MSG_FMT
#define MAX_PAC_MSG_FMT BXRAX_MSG_FMT

		char msg[strlen(MAX_PAC_MSG_FMT)
		- strlen("%s") + strlen("IA")
		- strlen("%s") + strlen("xzr")
		+ 1];
		ptrauth_key key = (ptrauth_key)(esr & 0x3);
		const char *key_str = ptrauth_key_to_string(key);

		if (ARM64_INSTR_IS_AUTxx(instr)) {
… more in source

handle_bti_fail source

__has_feature(ptrauth_calls)

__attribute__((noreturn))
static void
handle_bti_fail(arm_saved_state_t *state, uint64_t esr)
{
	uint32_t btype = (uint32_t) esr & ISS_BTI_BTYPE_MASK;

	if (!is_saved_state64(state)) {
		/* BTI is an ARMv8 feature, this should not be possible */
		panic("BTI failure for 32-bit state? (ESR=0x%llx)", esr);
	}

	/*
	 * We currently only expect BTI to be enabled for kernel pages, so panic if
	 * we detect otherwise.
	 */
	if (!PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
		panic("Unexpected non-kernel BTI failure? (ESR=0x%llx)", esr);
	}

#define BTI_FAIL_PTR_FMT "%04x"
#define BTI_FAIL_MSG_FMT "Kernel BTI failure (BTYPE=0x" BTI_FAIL_PTR_FMT ")"
	/* Replace the pointer format with the length of the pointer message+NULL */
	char msg[strlen(BTI_FAIL_MSG_FMT) - strlen(BTI_FAIL_PTR_FMT) + 8 + 1];
	snprintf(msg, sizeof(msg), BTI_FAIL_MSG_FMT, btype);
	panic_with_thread_kernel_state(msg, state);
	__builtin_unreachable();
}

handle_user_trapped_instruction32 source

static void
handle_user_trapped_instruction32(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t           exception = EXC_BAD_INSTRUCTION;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_UNDEFINED};
	mach_msg_type_number_t     numcodes  = 2;
	uint32_t                   instr;

	if (is_saved_state64(state)) {
		panic("ESR (0x%llx) for instruction trapped from U32, but saved state is 64-bit.", esr);
	}

	if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
		panic("ESR (0x%llx) for instruction trapped from U32, actually came from kernel?", esr);
	}

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
	codes[1] = instr;

	exception_triage(exception, codes, numcodes);
	__builtin_unreachable();
}

handle_simd_trap source

static void
handle_simd_trap(arm_saved_state_t *state, uint64_t esr)
{
	exception_type_t           exception = EXC_BAD_INSTRUCTION;
	mach_exception_data_type_t codes[2]  = {EXC_ARM_UNDEFINED};
	mach_msg_type_number_t     numcodes  = 2;
	uint32_t                   instr     = 0;

	if (PSR64_IS_KERNEL(get_saved_state_cpsr(state))) {
		panic("ESR (0x%llx) for SIMD trap from userland, actually came from kernel?", esr);
	}

	COPYIN(get_saved_state_pc(state), (char *)&instr, sizeof(instr));
	codes[1] = instr;

	exception_triage(exception, codes, numcodes);
	__builtin_unreachable();
}

sleh_irq source

void
sleh_irq(arm_saved_state_t *state)
{
	cpu_data_t * cdp __unused             = getCpuDatap();
#if MACH_ASSERT
	int preemption_level = sleh_get_preemption_level();
#endif


	sleh_interrupt_handler_prologue(state, DBG_INTR_TYPE_OTHER);

#if USE_APPLEARMSMP
	PE_handle_ext_interrupt();
#else
	/* Run the registered interrupt handler. */
	cdp->interrupt_handler(cdp->interrupt_target,
	    cdp->interrupt_refCon,
	    cdp->interrupt_nub,
	    cdp->interrupt_source);
#endif

	entropy_collect();


	sleh_interrupt_handler_epilogue();
#if MACH_ASSERT
	if (preemption_level != sleh_get_preemption_level()) {
		panic("irq handler %p changed preemption level from %d to %d", cdp->interrupt_handler, preemption_level, sleh_get_preemption_level());
	}
#endif
… more in source

sleh_fiq source

void
sleh_fiq(arm_saved_state_t *state)
{
	unsigned int type   = DBG_INTR_TYPE_UNKNOWN;
#if MACH_ASSERT
	int preemption_level = sleh_get_preemption_level();
#endif

#if MONOTONIC_FIQ
	uint64_t pmcr0 = 0, upmsr = 0;
#endif /* MONOTONIC_FIQ */

#if defined(HAS_IPI)
	boolean_t    is_ipi = FALSE;
	uint64_t     ipi_sr = 0;

	if (gFastIPI) {
		MRS(ipi_sr, "S3_5_C15_C1_1");

		if (ipi_sr & ARM64_IPISR_IPI_PENDING) {
			is_ipi = TRUE;
		}
	}

	if (is_ipi) {
		type = DBG_INTR_TYPE_IPI;
	} else
#endif /* defined(HAS_IPI) */
	if (ml_get_timer_pending()) {
		type = DBG_INTR_TYPE_TIMER;
… more in source

sleh_serror source

void
sleh_serror(arm_context_t *context, uint64_t esr, vm_offset_t far)
{
	task_vtimer_check(current_thread());

	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_START,
	    esr, VM_KERNEL_ADDRHIDE(far));
	arm_saved_state_t *state = &context->ss;
#if MACH_ASSERT
	int preemption_level = sleh_get_preemption_level();
#endif

	if (PSR64_IS_USER(get_saved_state_cpsr(state))) {
		/* Sanitize FAR (only if we came from userspace) */
		saved_state64(state)->far = 0;
	}

	ASSERT_CONTEXT_SANITY(context);
	arm64_platform_error(state, esr, far, PLAT_ERR_SRC_ASYNC);
#if MACH_ASSERT
	if (preemption_level != sleh_get_preemption_level()) {
		panic("serror changed preemption level from %d to %d", preemption_level, sleh_get_preemption_level());
	}
#endif
	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_SERR_ARM, 0) | DBG_FUNC_END,
	    esr, VM_KERNEL_ADDRHIDE(far));
}

mach_syscall_trace_exit source

void
mach_syscall_trace_exit(unsigned int retval,
    unsigned int call_number)
{
	KERNEL_DEBUG_CONSTANT_IST(KDEBUG_TRACE,
	    MACHDBG_CODE(DBG_MACH_EXCP_SC, (call_number)) |
	    DBG_FUNC_END, retval, 0, 0, 0, 0);
}

thread_syscall_return source

__attribute__((noreturn))
void
thread_syscall_return(kern_return_t error)
{
	thread_t thread;
	struct arm_saved_state *state;

	thread = current_thread();
	state = get_user_regs(thread);

	assert(is_saved_state64(state));
	saved_state64(state)->x[0] = error;

#if MACH_ASSERT
	kern_allocation_name_t
	prior __assert_only = thread_get_kernel_state(thread)->allocation_name;
	assertf(prior == NULL, "thread_set_allocation_name(\"%s\") not cleared", kern_allocation_get_name(prior));
#endif /* MACH_ASSERT */

	if (kdebug_enable) {
		/* Invert syscall number (negative for a mach syscall) */
		mach_syscall_trace_exit(error, (-1) * get_saved_state_svc_number(state));
	}

	thread_exception_return();
}

syscall_trace source

void
syscall_trace(
	struct arm_saved_state * regs __unused)
{
	/* kprintf("syscall: %d\n", saved_state64(regs)->x[16]);  */
}

sleh_interrupt_handler_prologue source

static void
sleh_interrupt_handler_prologue(arm_saved_state_t *state, unsigned int type)
{
	const bool is_user = PSR64_IS_USER(get_saved_state_cpsr(state));

	if (is_user == true) {
		/* Sanitize FAR (only if the interrupt occurred while the CPU was in usermode) */
		saved_state64(state)->far = 0;
	}

	recount_enter_interrupt();

	task_vtimer_check(current_thread());

	uint64_t pc = is_user ? get_saved_state_pc(state) :
	    VM_KERNEL_UNSLIDE(get_saved_state_pc(state));

	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_START,
	    0, pc, is_user, type);
}

sleh_interrupt_handler_epilogue source

static void
sleh_interrupt_handler_epilogue(void)
{
#if KPERF
	kperf_interrupt();
#endif /* KPERF */
	KDBG_RELEASE(MACHDBG_CODE(DBG_MACH_EXCP_INTR, 0) | DBG_FUNC_END);
	recount_leave_interrupt();
}

sleh_invalid_stack source

void
sleh_invalid_stack(arm_context_t *context, uint64_t esr __unused, vm_offset_t far __unused)
{
	thread_t thread = current_thread();
	vm_offset_t kernel_stack_bottom, sp;

	sp = get_saved_state_sp(&context->ss);
	vm_offset_t kstackptr = (vm_offset_t)thread->machine.kstackptr;
	kernel_stack_bottom = round_page(kstackptr) - KERNEL_STACK_SIZE;

	if ((sp < kernel_stack_bottom) && (sp >= (kernel_stack_bottom - PAGE_SIZE))) {
		panic_with_thread_kernel_state("Invalid kernel stack pointer (probable overflow).", &context->ss);
	}

	panic_with_thread_kernel_state("Invalid kernel stack pointer (probable corruption or early boot).", &context->ss);
}

handle_recoverable_kernel_trap source

static const char *
handle_recoverable_kernel_trap(
	__unused void     *tstate,
	uint16_t          comment)
{
	assert(comment == TEST_RECOVERABLE_SOFT_TRAP);

	printf("Recoverable trap handled.\n");
	trap_handled = 1;

	return NULL;
}

recoverable_kernel_trap_test source

static int
recoverable_kernel_trap_test(__unused int64_t in, int64_t *out)
{
	ml_recoverable_trap(TEST_RECOVERABLE_SOFT_TRAP);

	*out = trap_handled;
	return 0;
}

sleh_panic_lockdown_should_initiate_el1_sp0_sync source

Evaluate the panic lockdown policy for a synchronous EL1 SP0 exception Returns true if panic lockdown should be initiated (but does not itself do so)

__SECURITY_STACK_DISALLOWED_PUSH
bool
sleh_panic_lockdown_should_initiate_el1_sp0_sync(uint64_t esr, uint64_t elr,
    uint64_t far, uint64_t spsr)
{
	const esr_exception_class_t class = ESR_EC(esr);
	const bool any_exceptions_masked = spsr & DAIF_STANDARD_DISABLE;

	switch (class) {
	case ESR_EC_PC_ALIGN:   /* PC misaligned (should never happen) */
	case ESR_EC_IABORT_EL1: /* Potential iPAC failure (poisoned PC) */
	case ESR_EC_PAC_FAIL: { /* FPAC fail */
		return true;
	}

	case ESR_EC_BRK_AARCH64: {
		/*
		 * Breakpoints are used on non-FPAC systems to signal some PAC failures
		 */
#if HAS_TELEMETRY_KERNEL_BRK
		const struct kernel_brk_descriptor *desc;
		desc = find_kernel_brk_descriptor_by_comment(ISS_BRK_COMMENT(esr));
		if (desc && desc->options.recoverable) {
			/*
			 * We matched a breakpoint and it's recoverable, skip lockdown.
			 */
			return false;
		}
#endif /* HAS_TELEMETRY_KERNEL_BRK */
… more in source

copyio_recovery_entry source struct

Fault copyio_recovery_entry in copyin/copyout routines. Offets are expressed in bytes from &copy_recovery_table

struct copyio_recovery_entry {
	ptrdiff_t cre_start;
	ptrdiff_t cre_end;
	ptrdiff_t cre_recovery;
}

abort_inspector_t source typedef

typedef void (*abort_inspector_t)(uint32_t, fault_status_t *, vm_prot_t *);

abort_handler_t source typedef

typedef void (*abort_handler_t)(arm_saved_state_t *, uint64_t, vm_offset_t, fault_status_t, vm_prot_t, expected_fault_handler_t);

arm64_instr_t source typedef

typedef uint32_t arm64_instr_t;

user_brk_label_range_descriptor_t source typedef

Similar in spirit to kernel_brk_descriptor, but with less flexible semantics: each descriptor defines a `brk` label range for use from userspace. When used, system policy may decide to kill the calling process without giving them opportunity to catch the exception or continue execution from a signal handler. This is used to enforce security boundaries: userspace code may use this mechanism to reliably terminate when internal inconsistencies are detected. Note that we don't invariably terminate without giving the process a say: we might only enforce such a policy if a security feature is enabled, for example.

typedef struct user_brk_label_range_descriptor { uint16_t base; uint16_t max; } user_brk_label_range_descriptor_t;

arm_fast_fault source

extern kern_return_t arm_fast_fault(pmap_t, vm_map_address_t, vm_prot_t, bool, bool);

current_cached_proc_cred_update source

extern void current_cached_proc_cred_update(void);

unix_syscall source

extern void unix_syscall(struct arm_saved_state * regs, thread_t thread_act, struct proc * proc);

mach_syscall source

extern void mach_syscall(struct arm_saved_state*);

dtrace_user_probe source

CONFIG_SPTM

extern kern_return_t dtrace_user_probe(arm_saved_state_t* regs);

dtrace_tally_fault source

extern boolean_t dtrace_tally_fault(user_addr_t);

arm64_thread_exception_return source

extern void arm64_thread_exception_return(void) __dead2;