Panic and debugger entry

static inline __pure2 struct debugger_state *
current_debugger_state(void)
{
	return PERCPU_GET(debugger_state);
}

boolean_t
kernel_debugging_restricted(void)
{
#if XNU_TARGET_OS_OSX
#if CONFIG_CSR
	if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0) {
		return TRUE;
	}
#endif /* CONFIG_CSR */
	return FALSE;
#else /* XNU_TARGET_OS_OSX */
	return FALSE;
#endif /* XNU_TARGET_OS_OSX */
}

__startup_func
static void
panic_init(void)
{
	unsigned long uuidlen = 0;
	void *uuid;

	uuid = getuuidfromheader(&_mh_execute_header, &uuidlen);
	if ((uuid != NULL) && (uuidlen == sizeof(uuid_t))) {
		kernel_uuid = uuid;
		uuid_unparse_upper(*(uuid_t *)uuid, kernel_uuid_string);
	}

	/*
	 * Take the value of the debug boot-arg into account
	 */
#if MACH_KDP
	if (!kernel_debugging_restricted() && debug_boot_arg) {
		if (debug_boot_arg & DB_HALT) {
			halt_in_debugger = 1;
		}

#if defined(__arm64__)
		if (debug_boot_arg & DB_NMI) {
			panicDebugging  = TRUE;
		}
#else
		panicDebugging = TRUE;
#endif /* defined(__arm64__) */
	}
… more in source

void
extended_debug_log_init(void)
{
	assert(coprocessor_paniclog_flush);
	/*
	 * Allocate an extended panic log buffer that has space for the panic
	 * stackshot at the end. Update the debug buf pointers appropriately
	 * to point at this new buffer.
	 *
	 * iBoot pre-initializes the panic region with the NULL character. We set this here
	 * so we can accurately calculate the CRC for the region without needing to flush the
	 * full region over SMC.
	 */
	char *new_debug_buf = kalloc_data(EXTENDED_DEBUG_BUF_SIZE, Z_WAITOK | Z_ZERO);

	panic_info = (struct macos_panic_header *)new_debug_buf;
	debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));
	debug_buf_size = (EXTENDED_DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));

	extended_debug_log_enabled = TRUE;

	/*
	 * Insert a compiler barrier so we don't free the other panic stackshot buffer
	 * until after we've marked the new one as available
	 */
	__compiler_barrier();
	kmem_free(kernel_map, panic_stackshot_buf, panic_stackshot_buf_len);
	panic_stackshot_buf = 0;
	panic_stackshot_buf_len = 0;
}

void
debug_log_init(void)
{
#if defined(__arm64__)
	if (!gPanicBase) {
		printf("debug_log_init: Error!! gPanicBase is still not initialized\n");
		return;
	}
	/* Shift debug buf start location and size by the length of the panic header */
	debug_buf_base = (char *)gPanicBase + sizeof(struct embedded_panic_header);
	debug_buf_ptr = debug_buf_base;
	debug_buf_size = gPanicSize - sizeof(struct embedded_panic_header);

#if CONFIG_EXT_PANICLOG
	ext_paniclog_init();
#endif
#else
	kern_return_t kr = KERN_SUCCESS;
	bzero(panic_info, DEBUG_BUF_SIZE);

	assert(debug_buf_base != NULL);
	assert(debug_buf_ptr != NULL);
	assert(debug_buf_size != 0);

	/*
	 * We allocate a buffer to store a panic time stackshot. If we later discover that this is a
	 * system that supports flushing a stackshot via an extended debug log (see above), we'll free this memory
	 * as it's not necessary on this platform. This information won't be available until the IOPlatform has come
	 * up.
	 */
… more in source

void
phys_carveout_init(void)
{
	if (!PE_i_can_has_debugger(NULL)) {
		return;
	}

#if __arm__ || __arm64__
#if DEVELOPMENT || DEBUG
#endif /* DEVELOPMENT || DEBUG  */
#endif /* __arm__ || __arm64__ */

	struct carveout {
		const char *name;
		vm_offset_t *va;
		uint32_t requested_size;
		uintptr_t *pa;
		size_t *allocated_size;
		uint64_t present;
	} carveouts[] = {
		{
			"phys_carveout",
			&phys_carveout,
			phys_carveout_mb,
			&phys_carveout_pa,
			&phys_carveout_size,
			phys_carveout_mb != 0,
		}
	};
… more in source

boolean_t
debug_is_in_phys_carveout(vm_map_offset_t va)
{
	return phys_carveout_size && va >= phys_carveout &&
	       va < (phys_carveout + phys_carveout_size);
}

boolean_t
debug_can_coredump_phys_carveout(void)
{
	return phys_carveout_core;
}

static boolean_t
DebuggerLock(void)
{
	int my_cpu = cpu_number();
	int debugger_exp_cpu = DEBUGGER_NO_CPU;
	assert(ml_get_interrupts_enabled() == FALSE);

	if (atomic_load(&debugger_cpu) == my_cpu) {
		return true;
	}

	if (!atomic_compare_exchange_strong(&debugger_cpu, &debugger_exp_cpu, my_cpu)) {
		return false;
	}

	return true;
}

static void
DebuggerUnlock(void)
{
	assert(atomic_load_explicit(&debugger_cpu, memory_order_relaxed) == cpu_number());

	/*
	 * We don't do an atomic exchange here in case
	 * there's another CPU spinning to acquire the debugger_lock
	 * and we never get a chance to update it. We already have the
	 * lock so we can simply store DEBUGGER_NO_CPU and follow with
	 * a barrier.
	 */
	atomic_store(&debugger_cpu, DEBUGGER_NO_CPU);
	OSMemoryBarrier();

	return;
}

static kern_return_t
DebuggerHaltOtherCores(boolean_t proceed_on_failure, bool is_stackshot)
{
#if defined(__arm64__)
	return DebuggerXCallEnter(proceed_on_failure, is_stackshot);
#else /* defined(__arm64__) */
#pragma unused(proceed_on_failure)
	mp_kdp_enter(proceed_on_failure, is_stackshot);
	return KERN_SUCCESS;
#endif
}

static void
DebuggerResumeOtherCores(void)
{
#if defined(__arm64__)
	DebuggerXCallReturn();
#else /* defined(__arm64__) */
	mp_kdp_exit();
#endif
}

__printflike(3, 0)
static void
DebuggerSaveState(debugger_op db_op, const char *db_message, const char *db_panic_str,
    va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
    boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char *db_panic_initiator)
{
	CPUDEBUGGEROP = db_op;

	/*
	 * Note:
	 * if CPUDEBUGGERCOUNT == 1 then we are in the normal case - record the panic data
	 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR == NULL then we are in a nested panic that happened before DebuggerSaveState was called, so store the nested panic data
	 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR != NULL then we are in a nested panic that happened after DebuggerSaveState was called, so leave the original panic data
	 *
	 * TODO: is it safe to flatten this to if (CPUPANICSTR == NULL)?
	 */
	if (CPUDEBUGGERCOUNT == 1 || CPUPANICSTR == NULL) {
		CPUDEBUGGERMSG = db_message;
		CPUPANICSTR = db_panic_str;
		CPUPANICARGS = db_panic_args;
		CPUPANICDATAPTR = db_panic_data_ptr;
		CPUPANICCALLER = db_panic_caller;
		CPUPANICINITIATOR = db_panic_initiator;

#if CONFIG_EXCLAVES
		char *panic_str;
		if (exclaves_panic_get_string(&panic_str) == KERN_SUCCESS) {
			CPUPANICSTR = panic_str;
		}
#endif
… more in source

kern_return_t
DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str,
    va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
    boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char* db_panic_initiator)
{
	kern_return_t ret;

#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
	if (!PE_arm_debug_and_trace_initialized()) {
		/*
		 * In practice this can only happen if we panicked very early,
		 * when only the boot CPU is online and before it has finished
		 * initializing the debug and trace infrastructure. We're going
		 * to hang soon, so let's at least make sure the message passed
		 * to panic() is actually logged.
		 */
		char buf[EARLY_PANIC_BUFLEN];
		vsnprintf(buf, EARLY_PANIC_BUFLEN, db_panic_str, *db_panic_args);
		paniclog_append_noflush("%s\n", buf);
	}
#endif

	assert(ml_get_interrupts_enabled() == FALSE);
	DebuggerSaveState(db_op, db_message, db_panic_str, db_panic_args,
	    db_panic_options, db_panic_data_ptr,
	    db_proceed_on_sync_failure, db_panic_caller, db_panic_initiator);

	/*
	 * On ARM this generates an uncategorized exception -> sleh code ->
	 *   DebuggerCall -> kdp_trap -> handle_debugger_trap
… more in source

void __attribute__((noinline))
Assert(const char*file, int line, const char *expression)
{
	panic_plain("%s:%d Assertion failed: %s", file, line, expression);
}

void
panic_assert_format(char *buf, size_t len, struct mach_assert_hdr *hdr, long a, long b)
{
	struct mach_assert_default *adef;
	struct mach_assert_3x      *a3x;

	static_assert(MACH_ASSERT_TRAP_CODE == XNU_HARD_TRAP_ASSERT_FAILURE);

	switch (hdr->type) {
	case MACH_ASSERT_DEFAULT:
		adef = __container_of(hdr, struct mach_assert_default, hdr);
		snprintf(buf, len, "%s:%d Assertion failed: %s",
		    hdr->filename, hdr->lineno, adef->expr);
		break;

	case MACH_ASSERT_3P:
		a3x = __container_of(hdr, struct mach_assert_3x, hdr);
		snprintf(buf, len, "%s:%d Assertion failed: "
		    "%s %s %s (%p %s %p)",
		    hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
		    (void *)a, a3x->op, (void *)b);
		break;

	case MACH_ASSERT_3S:
		a3x = __container_of(hdr, struct mach_assert_3x, hdr);
		snprintf(buf, len, "%s:%d Assertion failed: "
		    "%s %s %s (0x%lx %s 0x%lx, %ld %s %ld)",
		    hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
		    a, a3x->op, b, a, a3x->op, b);
		break;
… more in source

boolean_t
debug_is_current_cpu_in_panic_state(void)
{
	return current_debugger_state()->db_entry_count > 0;
}

static void
check_and_handle_nested_panic(uint64_t panic_options_mask, unsigned long panic_caller, const char *db_panic_str, va_list *db_panic_args)
{
	if ((CPUDEBUGGERCOUNT > 1) && (CPUDEBUGGERCOUNT < max_debugger_entry_count)) {
		// Note: this is the first indication in the panic log or serial that we are off the rails...
		//
		// if we panic *before* the paniclog is finalized then this will end up in the ips report with a panic_caller addr that gives us a clue
		// if we panic *after* the log is finalized then we will only see it in the serial log
		//
		paniclog_append_noflush("Nested panic detected - entry count: %d panic_caller: 0x%016lx\n", CPUDEBUGGERCOUNT, panic_caller);
		print_curr_backtrace();
		paniclog_flush();

		// print the *new* panic string to the console, we might not get it by other means...
		// TODO: I tried to write this stuff to the paniclog, but the serial output gets corrupted and the panicstring in the ips file is <mysterious>
		// rdar://87846117 (NestedPanic: output panic string to paniclog)
		if (db_panic_str) {
			printf("Nested panic string:\n");
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wformat-nonliteral"
#pragma clang diagnostic ignored "-Wformat"
			_doprnt(db_panic_str, db_panic_args, PE_kputc, 0);
#pragma clang diagnostic pop
			printf("\n<end nested panic string>\n");
		}
	}

	// Stage 1 bailout
	//
	// Try to complete the normal panic flow, i.e. try to make sure the callouts happen and we flush the paniclog.  If this fails with another nested
… more in source

void
Debugger(const char *message)
{
	DebuggerWithContext(0, NULL, message, DEBUGGER_OPTION_NONE, (unsigned long)(char *)__builtin_return_address(0));
}

void
DebuggerWithContext(unsigned int reason, void *ctx, const char *message,
    uint64_t debugger_options_mask, unsigned long debugger_caller)
{
	spl_t previous_interrupts_state;
	boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers;

#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
	read_lbr();
#endif
	previous_interrupts_state = ml_set_interrupts_enabled(FALSE);
	disable_preemption();

	/* track depth of debugger/panic entry */
	CPUDEBUGGERCOUNT++;

	/* emit a tracepoint as early as possible in case of hang */
	SOCD_TRACE_XNU(PANIC,
	    ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
	    PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
	    VALUE(debugger_options_mask),
	    ADDR(message),
	    ADDR(debugger_caller));

	/* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
	check_and_handle_nested_panic(debugger_options_mask, debugger_caller, message, NULL);

	/* Handle any necessary platform specific actions before we proceed */
	PEInitiatePanic();
… more in source

void
kdp_register_callout(kdp_callout_fn_t fn, void * arg)
{
	struct kdp_callout * kcp;
	struct kdp_callout * list_head;

	kcp = zalloc_permanent_type(struct kdp_callout);

	kcp->callout_fn = fn;
	kcp->callout_arg = arg;
	kcp->callout_in_progress = FALSE;

	/* Lock-less list insertion using compare and exchange. */
	do {
		list_head = kdp_callout_list;
		kcp->callout_next = list_head;
	} while (!OSCompareAndSwapPtr(list_head, kcp, &kdp_callout_list));
}

static void
kdp_callouts(kdp_event_t event)
{
	struct kdp_callout      *kcp = kdp_callout_list;

	while (kcp) {
		if (!kcp->callout_in_progress) {
			kcp->callout_in_progress = TRUE;
			kcp->callout_fn(kcp->callout_arg, event);
			kcp->callout_in_progress = FALSE;
		}
		kcp = kcp->callout_next;
	}
}

void
register_additional_panic_data_buffer(const char *producer_name, void *buf, int len)
{
	if (panic_data_buffers != NULL) {
		panic("register_additional_panic_data_buffer called with buffer already registered");
	}

	if (producer_name == NULL || (strlen(producer_name) == 0)) {
		panic("register_additional_panic_data_buffer called with invalid producer_name");
	}

	if (buf == NULL) {
		panic("register_additional_panic_data_buffer called with invalid buffer pointer");
	}

	if ((len <= 0) || (len > ADDITIONAL_PANIC_DATA_BUFFER_MAX_LEN)) {
		panic("register_additional_panic_data_buffer called with invalid length");
	}

	struct additional_panic_data_buffer *new_panic_data_buffer = zalloc_permanent_type(struct additional_panic_data_buffer);
	new_panic_data_buffer->producer_name = producer_name;
	new_panic_data_buffer->buf = buf;
	new_panic_data_buffer->len = len;

	if (!OSCompareAndSwapPtr(NULL, new_panic_data_buffer, &panic_data_buffers)) {
		panic("register_additional_panic_data_buffer called with buffer already registered");
	}

	return;
}

void
panic(const char *str, ...)
{
	va_list panic_str_args;

	va_start(panic_str_args, str);
	panic_trap_to_debugger(str, &panic_str_args, 0, NULL, 0, NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
	va_end(panic_str_args);
}

void
panic_with_data(uuid_t uuid, void *addr, uint32_t len, uint64_t debugger_options_mask, const char *str, ...)
{
	va_list panic_str_args;

	ext_paniclog_panic_with_data(uuid, addr, len);

#if CONFIG_EXCLAVES
	/*
	 * Before trapping, inform the exclaves scheduler that we're going down
	 * so it can grab an exclaves stackshot.
	 */
	if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
	    exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
		(void) exclaves_scheduler_request_watchdog_panic();
	}
#endif /* CONFIG_EXCLAVES */

	va_start(panic_str_args, str);
	panic_trap_to_debugger(str, &panic_str_args, 0, NULL, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
	    NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
	va_end(panic_str_args);
}

void
panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
{
	va_list panic_str_args;

#if CONFIG_EXCLAVES
	/*
	 * Before trapping, inform the exclaves scheduler that we're going down
	 * so it can grab an exclaves stackshot.
	 */
	if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
	    exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
		(void) exclaves_scheduler_request_watchdog_panic();
	}
#endif /* CONFIG_EXCLAVES */

	va_start(panic_str_args, str);
	panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
	    NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
	va_end(panic_str_args);
}

void
panic_with_options_and_initiator(const char* initiator, unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
{
	va_list panic_str_args;

	va_start(panic_str_args, str);
	panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
	    NULL, (unsigned long)(char *)__builtin_return_address(0), initiator);
	va_end(panic_str_args);
}

boolean_t
panic_validate_ptr(void *ptr, vm_size_t size, const char *what)
{
	if (ptr == NULL) {
		paniclog_append_noflush("NULL %s pointer\n", what);
		return false;
	}

	if (!ml_validate_nofault((vm_offset_t)ptr, size)) {
		paniclog_append_noflush("Invalid %s pointer: %p (size %d)\n",
		    what, ptr, (uint32_t)size);
		return false;
	}

	return true;
}

boolean_t
panic_get_thread_proc_task(struct thread *thread, struct task **task, struct proc **proc)
{
	if (!PANIC_VALIDATE_PTR(thread)) {
		return false;
	}

	if (!PANIC_VALIDATE_PTR(thread->t_tro)) {
		return false;
	}

	if (!PANIC_VALIDATE_PTR(thread->t_tro->tro_task)) {
		return false;
	}

	if (task) {
		*task = thread->t_tro->tro_task;
	}

	if (!panic_validate_ptr(thread->t_tro->tro_proc,
	    sizeof(struct proc *), "bsd_info")) {
		*proc = NULL;
	} else {
		*proc = thread->t_tro->tro_proc;
	}

	return true;
}

void
panic_with_thread_context(unsigned int reason, void *ctx, uint64_t debugger_options_mask, thread_t thread, const char *str, ...)
{
	va_list panic_str_args;
	__assert_only os_ref_count_t th_ref_count;

	assert_thread_magic(thread);
	th_ref_count = os_ref_get_count_raw(&thread->ref_count);
	assertf(th_ref_count > 0, "panic_with_thread_context called with invalid thread %p with refcount %u", thread, th_ref_count);

	/* Take a reference on the thread so it doesn't disappear by the time we try to backtrace it */
	thread_reference(thread);

	va_start(panic_str_args, str);
	panic_trap_to_debugger(str, &panic_str_args, reason, ctx, ((debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK) | DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE),
	    thread, (unsigned long)(char *)__builtin_return_address(0), "");

	va_end(panic_str_args);
}

__mockable void
panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void *ctx,
    uint64_t panic_options_mask, void *panic_data_ptr, unsigned long panic_caller, const char *panic_initiator)
{
#pragma clang diagnostic pop

#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
	read_lbr();
#endif

	/* optionally call sync, to reduce lost logs on restart, avoid on recursive panic. Unsafe due to unbounded sync() duration */
	if ((panic_options_mask & DEBUGGER_OPTION_SYNC_ON_PANIC_UNSAFE) && (CPUDEBUGGERCOUNT == 0)) {
		sync_internal();
	}

	/* Turn off I/O tracing once we've panicked */
	iotrace_disable();

	/* call machine-layer panic handler */
	ml_panic_trap_to_debugger(panic_format_str, panic_args, reason, ctx, panic_options_mask, panic_caller, panic_initiator);

	/* track depth of debugger/panic entry */
	CPUDEBUGGERCOUNT++;

	__unused uint32_t panic_initiator_crc = panic_initiator ? crc32(0, panic_initiator, strnlen(panic_initiator, MAX_PANIC_INITIATOR_SIZE)) : 0;

	/* emit a tracepoint as early as possible in case of hang */
	SOCD_TRACE_XNU(PANIC,
	    ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
	    PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
… more in source

void __attribute__((noinline, optnone))
panic_spin_forever(void)
{
	for (;;) {
#if defined(__arm__) || defined(__arm64__)
		/* On arm32, which doesn't have a WFE timeout, this may not return.  But that should be OK on this path. */
		__builtin_arm_wfe();
#else
		cpu_pause();
#endif
	}
}

void
panic_stackshot_release_lock(void)
{
	assert(!not_in_kdp);
	DebuggerUnlock();
}

static void
kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags)
{
	if ((type == kPEPanicRestartCPU) && (debugger_flags & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS)) {
		PEHaltRestart(kPEPanicRestartCPUNoCallouts);
	} else {
		PEHaltRestart(type);
	}
	halt_all_cpus(TRUE);
}

void
kdp_machine_reboot(void)
{
	kdp_machine_reboot_type(kPEPanicRestartCPU, 0);
}

static __attribute__((unused)) void
panic_debugger_log(const char *string, ...)
{
	va_list panic_debugger_log_args;

	va_start(panic_debugger_log_args, string);
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wformat-nonliteral"
#pragma clang diagnostic ignored "-Wformat"
	_doprnt(string, &panic_debugger_log_args, consdebug_putc, 16);
#pragma clang diagnostic pop
	va_end(panic_debugger_log_args);

#if defined(__arm64__)
	paniclog_flush();
#endif
}

static void
debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
{
#if DEVELOPMENT || DEBUG
	INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_PRELOG);
#endif

#if defined(__x86_64__)
	kprintf("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
#endif
	/*
	 * DB_HALT (halt_in_debugger) can be requested on startup, we shouldn't generate
	 * a coredump/paniclog for this type of debugger entry. If KDP isn't configured,
	 * we'll just spin in kdp_raise_exception.
	 */
	if (debugger_current_op == DBOP_DEBUGGER && halt_in_debugger) {
		kdp_raise_exception(exception, code, subcode, state);
		if (debugger_safe_to_return && !debugger_is_panic) {
			return;
		}
	}

#ifdef CONFIG_KCOV
	/* Try not to break core dump path by sanitizer. */
	kcov_panic_disable();
#endif

	if ((debugger_current_op == DBOP_PANIC) ||
	    ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
		/*
… more in source

void
handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
{
	unsigned int initial_not_in_kdp = not_in_kdp;
	kern_return_t ret = KERN_SUCCESS;
	debugger_op db_prev_op = debugger_current_op;

	if (!DebuggerLock()) {
		/*
		 * We lost the race to be the first to panic.
		 * Return here so that we will enter the panic stop
		 * infinite loop and take the debugger IPI from the
		 * first CPU that got the debugger lock.
		 */
		return;
	}

	DEBUGGER_TRAP_TIMESTAMP(0);

	ret = DebuggerHaltOtherCores(CPUDEBUGGERSYNC, (CPUDEBUGGEROP == DBOP_STACKSHOT));

	DEBUGGER_TRAP_TIMESTAMP(1);

#if SCHED_HYGIENE_DEBUG
	if (serialmode & SERIALMODE_OUTPUT) {
		ml_spin_debug_reset(current_thread());
	}
#endif /* SCHED_HYGIENE_DEBUG */
	if (ret != KERN_SUCCESS) {
		CPUDEBUGGERRET = ret;
… more in source

__attribute__((noinline, not_tail_called))
void
log(__unused int level, char *fmt, ...)
{
	void *caller = __builtin_return_address(0);
	va_list listp;
	va_list listp2;


#ifdef lint
	level++;
#endif /* lint */
#ifdef  MACH_BSD
	va_start(listp, fmt);
	va_copy(listp2, listp);

	disable_preemption();
	_doprnt(fmt, &listp, cons_putc_locked, 0);
	enable_preemption();

	va_end(listp);

#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wformat-nonliteral"
#pragma clang diagnostic ignored "-Wformat"
	os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, listp2, caller);
#pragma clang diagnostic pop
	va_end(listp2);
#endif
}

boolean_t
oslog_is_safe(void)
{
	return kernel_debugger_entry_count == 0 &&
	       not_in_kdp == 1 &&
	       get_preemption_level() == 0 &&
	       ml_get_interrupts_enabled() == TRUE;
}

boolean_t
debug_mode_active(void)
{
	return (0 != kernel_debugger_entry_count != 0) || (0 == not_in_kdp);
}

void
debug_putc(char c)
{
	if ((debug_buf_size != 0) &&
	    ((debug_buf_ptr - debug_buf_base) < (int)debug_buf_size) &&
	    (!is_debug_ptr_in_ext_paniclog())) {
		*debug_buf_ptr = c;
		debug_buf_ptr++;
	}
}

int
packA(char *inbuf, uint32_t length, uint32_t buflen)
{
	unsigned int i, j = 0;
	pasc_t pack;

	length = MIN(((length + 7) & ~7), buflen);

	for (i = 0; i < length; i += 8) {
		pack.a = inbuf[i];
		pack.b = inbuf[i + 1];
		pack.c = inbuf[i + 2];
		pack.d = inbuf[i + 3];
		pack.e = inbuf[i + 4];
		pack.f = inbuf[i + 5];
		pack.g = inbuf[i + 6];
		pack.h = inbuf[i + 7];
		bcopy((char *) &pack, inbuf + j, 7);
		j += 7;
	}
	return j;
}

void
unpackA(char *inbuf, uint32_t length)
{
	pasc_t packs;
	unsigned i = 0;
	length = (length * 8) / 7;

	while (i < length) {
		packs = *(pasc_t *)&inbuf[i];
		bcopy(&inbuf[i + 7], &inbuf[i + 8], MAX(0, (int) (length - i - 8)));
		inbuf[i++] = packs.a;
		inbuf[i++] = packs.b;
		inbuf[i++] = packs.c;
		inbuf[i++] = packs.d;
		inbuf[i++] = packs.e;
		inbuf[i++] = packs.f;
		inbuf[i++] = packs.g;
		inbuf[i++] = packs.h;
	}
}

__private_extern__ void
panic_display_process_name(void)
{
	proc_name_t proc_name = {};
	struct proc *cbsd_info = NULL;
	task_t ctask = NULL;
	vm_size_t size;

	if (!panic_get_thread_proc_task(current_thread(), &ctask, &cbsd_info)) {
		goto out;
	}

	if (cbsd_info == NULL) {
		goto out;
	}

	size = ml_nofault_copy((vm_offset_t)proc_longname_address(cbsd_info),
	    (vm_offset_t)&proc_name, sizeof(proc_name));

	if (size == 0 || proc_name[0] == '\0') {
		size = ml_nofault_copy((vm_offset_t)proc_name_address(cbsd_info),
		    (vm_offset_t)&proc_name,
		    MIN(sizeof(command_t), sizeof(proc_name)));
		if (size > 0) {
			proc_name[size - 1] = '\0';
		}
	}

out:
	proc_name[sizeof(proc_name) - 1] = '\0';
… more in source

unsigned
panic_active(void)
{
	return debugger_current_op == DBOP_PANIC ||
	       (debugger_current_op == DBOP_DEBUGGER && debugger_is_panic);
}

void
populate_model_name(char *model_string)
{
	strlcpy(model_name, model_string, sizeof(model_name));
}

void
panic_display_model_name(void)
{
	char tmp_model_name[sizeof(model_name)];

	if (ml_nofault_copy((vm_offset_t) &model_name, (vm_offset_t) &tmp_model_name, sizeof(model_name)) != sizeof(model_name)) {
		return;
	}

	tmp_model_name[sizeof(tmp_model_name) - 1] = '\0';

	if (tmp_model_name[0] != 0) {
		paniclog_append_noflush("System model name: %s\n", tmp_model_name);
	}
}

void
panic_display_kernel_uuid(void)
{
	char tmp_kernel_uuid[sizeof(kernel_uuid_string)];

	if (ml_nofault_copy((vm_offset_t) &kernel_uuid_string, (vm_offset_t) &tmp_kernel_uuid, sizeof(kernel_uuid_string)) != sizeof(kernel_uuid_string)) {
		return;
	}

	if (tmp_kernel_uuid[0] != '\0') {
		paniclog_append_noflush("Kernel UUID: %s\n", tmp_kernel_uuid);
	}
}

static void
panic_display_component_uuid(char const *component_name, void *component_address)
{
	uuid_t *component_uuid;
	unsigned long component_uuid_len = 0;
	uuid_string_t component_uuid_string;

	component_uuid = getuuidfromheader((kernel_mach_header_t *)component_address, &component_uuid_len);

	if (component_uuid != NULL && component_uuid_len == sizeof(uuid_t)) {
		uuid_unparse_upper(*component_uuid, component_uuid_string);
		paniclog_append_noflush("%s UUID: %s\n", component_name, component_uuid_string);
	}
}

void
panic_display_kernel_aslr(void)
{
#if CONFIG_SPTM
	{
		struct debug_header const *dh = SPTMArgs->debug_header;

		paniclog_append_noflush("Debug Header address: %p\n", dh);

		if (dh != NULL) {
			void *component_address;

			paniclog_append_noflush("Debug Header entry count: %d\n", dh->count);

			switch (dh->count) {
			default: // 3 or more
				component_address = dh->image[DEBUG_HEADER_ENTRY_TXM];
				paniclog_append_noflush("TXM load address: %p\n", component_address);

				panic_display_component_uuid("TXM", component_address);
				OS_FALLTHROUGH;
			case 2:
				component_address = dh->image[DEBUG_HEADER_ENTRY_XNU];
				paniclog_append_noflush("Debug Header kernelcache load address: %p\n", component_address);

				panic_display_component_uuid("Debug Header kernelcache", component_address);
				OS_FALLTHROUGH;
			case 1:
				component_address = dh->image[DEBUG_HEADER_ENTRY_SPTM];
				paniclog_append_noflush("SPTM load address: %p\n", component_address);
… more in source

void
panic_display_hibb(void)
{
#if defined(__i386__) || defined (__x86_64__)
	paniclog_append_noflush("__HIB  text base: %p\n", (void *) vm_hib_base);
#endif
}

__private_extern__ void
panic_display_ecc_errors(void)
{
	uint32_t count = ecc_log_get_correction_count();

	if (count > 0) {
		paniclog_append_noflush("ECC Corrections:%u\n", count);
	}
}

void
panic_display_compressor_stats(void)
{
	int isswaplow = vm_swap_low_on_space();
#if CONFIG_FREEZE
	uint32_t incore_seg_count;
	uint32_t incore_compressed_pages;
	if (freezer_incore_cseg_acct) {
		incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
		incore_compressed_pages = c_segment_pages_compressed_incore;
	} else {
		incore_seg_count = c_segment_count;
		incore_compressed_pages = c_segment_pages_compressed;
	}

	paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
	    (incore_compressed_pages * 100) / c_segment_pages_compressed_limit,
	    (incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
	    (incore_seg_count * 100) / c_segments_limit,
	    (incore_seg_count > c_segments_nearing_limit) ? "BAD":"OK",
	    vm_num_swap_files,
	    isswaplow ? "LOW":"OK");
#else /* CONFIG_FREEZE */
	paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
	    (c_segment_pages_compressed * 100) / c_segment_pages_compressed_limit,
	    (c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
	    (c_segment_count * 100) / c_segments_limit,
	    (c_segment_count > c_segments_nearing_limit) ? "BAD":"OK",
	    vm_num_swap_files,
	    isswaplow ? "LOW":"OK");
… more in source

int
telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, bool mark __unused)
{
	return KERN_NOT_SUPPORTED;
}

__startup_func
static void
kern_feature_override_init(void)
{
	/*
	 * update kern_feature_override based on the serverperfmode=1 boot-arg
	 * being present, but do not look at the device-tree setting on purpose.
	 *
	 * scale_setup() will update serverperfmode=1 based on the DT later.
	 */

	if (serverperfmode) {
		kern_feature_overrides |= KF_SERVER_PERF_MODE_OVRD;
	}
}

__static_if_init_func
static void
kern_feature_override_apply(const char *args)
{
	uint64_t kf_ovrd;

	/*
	 * Compute the value of kern_feature_override like it will look like
	 * after kern_feature_override_init().
	 */
	kf_ovrd = static_if_boot_arg_uint64(args, "validation_disables", 0);
	if (static_if_boot_arg_uint64(args, "serverperfmode", 0)) {
		kf_ovrd |= KF_SERVER_PERF_MODE_OVRD;
	}

#if DEBUG_RW
	lck_rw_assert_init(args, kf_ovrd);
#endif /* DEBUG_RW */
#if MACH_ASSERT
	if (kf_ovrd & KF_MACH_ASSERT_OVRD) {
		static_if_key_disable(mach_assert);
	}
#endif /* MACH_ASSERT */
#if SCHED_HYGIENE_DEBUG
	if ((int64_t)static_if_boot_arg_uint64(args, "wdt", 0) != -1) {
		if (kf_ovrd & KF_SCHED_HYGIENE_DEBUG_PMC_OVRD) {
			static_if_key_disable(sched_debug_pmc);
		}
		if (kf_ovrd & KF_PREEMPTION_DISABLED_DEBUG_OVRD) {
			static_if_key_disable(sched_debug_preemption_disable);
… more in source

boolean_t
kern_feature_override(uint32_t fmask)
{
	return (kern_feature_overrides & fmask) == fmask;
}

static boolean_t
device_corefile_valid_on_ephemeral(void)
{
#ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
	DTEntry node;
	const uint32_t *value = NULL;
	unsigned int size = 0;
	if (kSuccess != SecureDTLookupEntry(NULL, "/product", &node)) {
		return TRUE;
	}
	if (kSuccess != SecureDTGetProperty(node, "ephemeral-data-mode", (void const **) &value, &size)) {
		return TRUE;
	}

	if (size != sizeof(uint32_t)) {
		return TRUE;
	}

	if ((*value) && (kern_dump_should_enforce_encryption() == true)) {
		return FALSE;
	}
#endif /* ifdef CONFIG_KDP_COREDUMP_ENCRYPTION */

	return TRUE;
}

boolean_t
on_device_corefile_enabled(void)
{
	assert(startup_phase >= STARTUP_SUB_TUNABLES);
#if CONFIG_KDP_INTERACTIVE_DEBUGGING
	if (debug_boot_arg == 0) {
		return FALSE;
	}
	if (debug_boot_arg & DB_DISABLE_LOCAL_CORE) {
		return FALSE;
	}
#if !XNU_TARGET_OS_OSX
	if (device_corefile_valid_on_ephemeral() == FALSE) {
		return FALSE;
	}
	/*
	 * outside of macOS, if there's a debug boot-arg set and local
	 * cores aren't explicitly disabled, we always write a corefile.
	 */
	return TRUE;
#else /* !XNU_TARGET_OS_OSX */
	/*
	 * on macOS, if corefiles on panic are requested and local cores
	 * aren't disabled we write a local core.
	 */
	if (debug_boot_arg & (DB_KERN_DUMP_ON_NMI | DB_KERN_DUMP_ON_PANIC)) {
		return TRUE;
	}
#endif /* !XNU_TARGET_OS_OSX */
#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
… more in source

boolean_t
panic_stackshot_to_disk_enabled(void)
{
	assert(startup_phase >= STARTUP_SUB_TUNABLES);
#if defined(__x86_64__)
	if (PEGetCoprocessorVersion() < kCoprocessorVersion2) {
		/* Only enabled on pre-Gibraltar machines where it hasn't been disabled explicitly */
		if ((debug_boot_arg != 0) && (debug_boot_arg & DB_DISABLE_STACKSHOT_TO_DISK)) {
			return FALSE;
		}

		return TRUE;
	}
#endif
	return FALSE;
}

const char *
sysctl_debug_get_preoslog(size_t *size)
{
	int result = 0;
	void *preoslog_pa = NULL;
	int preoslog_size = 0;

	result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
	if (result || preoslog_pa == NULL || preoslog_size == 0) {
		kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
		*size = 0;
		return NULL;
	}

	/*
	 *  Beware:
	 *  On release builds, we would need to call IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size) to free the preoslog buffer.
	 *  On Development & Debug builds, we retain the buffer so it can be extracted from coredumps.
	 */
	*size = preoslog_size;
	return (char *)(ml_static_ptovirt((vm_offset_t)(preoslog_pa)));
}

void
sysctl_debug_free_preoslog(void)
{
#if RELEASE
	int result = 0;
	void *preoslog_pa = NULL;
	int preoslog_size = 0;

	result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
	if (result || preoslog_pa == NULL || preoslog_size == 0) {
		kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
		return;
	}

	IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size);
#else
	/*  On Development & Debug builds, we retain the buffer so it can be extracted from coredumps. */
#endif // RELEASE
}

void
check_for_failure_injection(failure_injection_stage_t current_stage)
{
	// Can't call this function with the default initialization for xnu_upsi_injection_stage
	assert(current_stage != 0);

	// Check condition to inject a panic/stall/hang
	if (current_stage != xnu_upsi_injection_stage) {
		return;
	}

	// Do the requested action
	switch (xnu_upsi_injection_action) {
	case INJECTION_ACTION_PANIC:
		panic("Test panic at stage 0x%llx", current_stage);
	case INJECTION_ACTION_WATCHDOG_TIMEOUT:
	case INJECTION_ACTION_DEADLOOP:
		SPINNING_FOREVER();
		break;
	default:
		break;
	}
}

static inline void
awl_set_scratch_reg_hv_bit(void)
{
#if defined(__arm64__)
#define WATCHDOG_DIAG0     "S3_5_c15_c2_6"
	uint64_t awl_diag0 = __builtin_arm_rsr64(WATCHDOG_DIAG0);
	awl_diag0 |= AWL_HV_ENTRY_FLAG;
	__builtin_arm_wsr64(WATCHDOG_DIAG0, awl_diag0);
#endif // defined(__arm64__)
}

void
awl_mark_hv_entry(void)
{
	if (__probable(*PERCPU_GET(hv_entry_detected) || !awl_scratch_reg_supported)) {
		return;
	}
	*PERCPU_GET(hv_entry_detected) = true;

	awl_set_scratch_reg_hv_bit();
}

static bool
awl_pm_state_change_cbk(void *param __unused, enum cpu_event event, unsigned int cpu_or_cluster __unused)
{
	if (event == CPU_BOOTED) {
		if (*PERCPU_GET(hv_entry_detected)) {
			awl_set_scratch_reg_hv_bit();
		}
	}

	return true;
}

__startup_func
static void
set_awl_scratch_exists_flag_and_subscribe_for_pm(void)
{
	DTEntry base = NULL;

	if (SecureDTLookupEntry(NULL, "/arm-io/wdt", &base) != kSuccess) {
		return;
	}
	const uint8_t *data = NULL;
	unsigned int data_size = sizeof(uint8_t);

	if (base != NULL && SecureDTGetProperty(base, "awl-scratch-supported", (const void **)&data, &data_size) == kSuccess) {
		for (unsigned int i = 0; i < data_size; i++) {
			if (data[i] != 0) {
				awl_scratch_reg_supported = true;
				cpu_event_register_callback(awl_pm_state_change_cbk, NULL);
				break;
			}
		}
	}
}

static inline boolean_t
debug_fatal_panic_begin(void)
{
#if CONFIG_SPTM
	/*
	 * Since we're going down, initiate panic lockdown.
	 *
	 * Whether or not this call to panic lockdown can be subverted is murky.
	 * This doesn't really matter, however, because any security critical panics
	 * events will have already initiated lockdown from the exception vector
	 * before calling panic. Thus, lockdown from panic itself is fine as merely
	 * a "best effort".
	 */
#if DEVELOPMENT || DEBUG
	panic_lockdown_record_debug_data();
#endif /* DEVELOPMENT || DEBUG */
	sptm_xnu_panic_begin();

	pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
	uint16_t sptm_cpu_id = sptm_pcpu->sptm_cpu_id;
	uint64_t sptm_panicking_cpu_id;

	if (sptm_get_panicking_cpu_id(&sptm_panicking_cpu_id) == LIBSPTM_SUCCESS &&
	    sptm_panicking_cpu_id != sptm_cpu_id) {
		return false;
	}
#endif /* CONFIG_SPTM */
	return true;
}

struct debugger_state {
	uint64_t        db_panic_options;
	debugger_op     db_current_op;
	boolean_t       db_proceed_on_sync_failure;
	const char     *db_message;
	const char     *db_panic_str;
	va_list        *db_panic_args;
	void           *db_panic_data_ptr;
	unsigned long   db_panic_caller;
	const char     *db_panic_initiator;
	/* incremented whenever we panic or call Debugger (current CPU panic level) */
	uint32_t        db_entry_count;
	kern_return_t   db_op_return;
}

struct pasc {
	unsigned a: 7;
	unsigned b: 7;
	unsigned c: 7;
	unsigned d: 7;
	unsigned e: 7;
	unsigned f: 7;
	unsigned g: 7;
	unsigned h: 7;
}

typedef struct pasc pasc_t;

extern int vsnprintf(char *, size_t, const char *, va_list);

extern int IODTGetLoaderInfo( const char *key, void **infoAddr, int *infosize );

extern void IODTFreeLoaderInfo( const char *key, void *infoAddr, int infoSize );

extern void PE_panic_hook(const char*);

extern int sync_internal(void);

extern char *proc_name_address(void *);

extern char *proc_longname_address(void *);