osfmk/kern/debug.c
1/*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57#include <mach_assert.h>
58#include <mach_kdp.h>
59#include <kdp/kdp.h>
60#include <kdp/kdp_core.h>
61#include <kdp/kdp_internal.h>
62#include <kdp/kdp_callout.h>
63#include <kern/cpu_number.h>
64#include <kern/kalloc.h>
65#include <kern/percpu.h>
66#include <kern/spl.h>
67#include <kern/thread.h>
68#include <kern/assert.h>
69#include <kern/sched_prim.h>
70#include <kern/socd_client.h>
71#include <kern/misc_protos.h>
72#include <kern/clock.h>
73#include <kern/telemetry.h>
74#include <kern/trap_telemetry.h>
75#include <kern/ecc.h>
76#include <kern/kern_stackshot.h>
77#include <kern/kern_cdata.h>
78#include <kern/zalloc_internal.h>
79#include <kern/iotrace.h>
80#include <pexpert/device_tree.h>
81#include <vm/vm_kern_xnu.h>
82#include <vm/vm_map.h>
83#include <vm/pmap.h>
84#include <vm/vm_compressor_xnu.h>
85#include <stdarg.h>
86#include <stdatomic.h>
87#include <sys/pgo.h>
88#include <console/serial_protos.h>
89#include <IOKit/IOBSD.h>
90#include <libkern/crc.h>
91
92#if !(MACH_KDP && CONFIG_KDP_INTERACTIVE_DEBUGGING)
93#include <kdp/kdp_udp.h>
94#endif
95#include <kern/processor.h>
96
97#if defined(__i386__) || defined(__x86_64__)
98#include <IOKit/IOBSD.h>
99
100#include <i386/cpu_threads.h>
101#include <i386/pmCPU.h>
102#include <i386/lbr.h>
103#endif
104
105#include <IOKit/IOPlatformExpert.h>
106#include <machine/machine_cpu.h>
107#include <machine/pal_routines.h>
108
109#include <sys/kdebug.h>
110#include <libkern/OSKextLibPrivate.h>
111#include <libkern/OSAtomic.h>
112#include <libkern/kernel_mach_header.h>
113#include <libkern/section_keywords.h>
114#include <uuid/uuid.h>
115#include <mach_debug/zone_info.h>
116#include <mach/resource_monitors.h>
117#include <machine/machine_routines.h>
118#include <sys/proc_require.h>
119
120#include <os/log_private.h>
121
122#include <kern/ext_paniclog.h>
123
124#if defined(__arm64__)
125#include <pexpert/pexpert.h> /* For gPanicBase */
126#include <arm/caches_internal.h>
127#include <arm/misc_protos.h>
128extern volatile struct xnu_hw_shmem_dbg_command_info *hwsd_info;
129#endif
130
131#include <san/kcov.h>
132
133#if CONFIG_XNUPOST
134#include <tests/xnupost.h>
135extern int vsnprintf(char *, size_t, const char *, va_list);
136#endif
137
138#if CONFIG_CSR
139#include <sys/csr.h>
140#endif
141
142#if CONFIG_EXCLAVES
143#include <xnuproxy/panic.h>
144#include "exclaves_panic.h"
145#endif
146
147#if CONFIG_SPTM
148#include <arm64/sptm/sptm.h>
149#include <arm64/sptm/pmap/pmap_data.h>
150#endif /* CONFIG_SPTM */
151
152extern int IODTGetLoaderInfo( const char *key, void **infoAddr, int *infosize );
153extern void IODTFreeLoaderInfo( const char *key, void *infoAddr, int infoSize );
154extern unsigned int debug_boot_arg;
155
156unsigned int halt_in_debugger = 0;
157unsigned int current_debugger = 0;
158unsigned int active_debugger = 0;
159SECURITY_READ_ONLY_LATE(unsigned int) panicDebugging = FALSE;
160unsigned int kernel_debugger_entry_count = 0;
161
162#if DEVELOPMENT || DEBUG
163unsigned int panic_test_failure_mode = PANIC_TEST_FAILURE_MODE_BADPTR;
164unsigned int panic_test_action_count = 1;
165unsigned int panic_test_case = PANIC_TEST_CASE_DISABLED;
166#endif
167
168#if defined(__arm64__)
169struct additional_panic_data_buffer *panic_data_buffers = NULL;
170#endif
171
172#if defined(__arm64__)
173/*
174 * Magic number; this should be identical to the armv7 encoding for trap.
175 */
176#define TRAP_DEBUGGER __asm__ volatile(".long 0xe7ffdeff")
177#elif defined (__x86_64__)
178#define TRAP_DEBUGGER __asm__("int3")
179#else
180#error No TRAP_DEBUGGER for this architecture
181#endif
182
183#if defined(__i386__) || defined(__x86_64__)
184#define panic_stop() pmCPUHalt(PM_HALT_PANIC)
185#else
186#define panic_stop() panic_spin_forever()
187#endif
188
189#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
190/*
191 * More than enough for any typical format string passed to panic();
192 * anything longer will be truncated but that's better than nothing.
193 */
194#define EARLY_PANIC_BUFLEN 256
195#endif
196
197struct debugger_state {
198 uint64_t db_panic_options;
199 debugger_op db_current_op;
200 boolean_t db_proceed_on_sync_failure;
201 const char *db_message;
202 const char *db_panic_str;
203 va_list *db_panic_args;
204 void *db_panic_data_ptr;
205 unsigned long db_panic_caller;
206 const char *db_panic_initiator;
207 /* incremented whenever we panic or call Debugger (current CPU panic level) */
208 uint32_t db_entry_count;
209 kern_return_t db_op_return;
210};
211static struct debugger_state PERCPU_DATA(debugger_state);
212struct kernel_panic_reason PERCPU_DATA(panic_reason);
213
214/* __pure2 is correct if this function is called with preemption disabled */
215static inline __pure2 struct debugger_state *
216current_debugger_state(void)
217{
218 return PERCPU_GET(debugger_state);
219}
220
221#define CPUDEBUGGEROP current_debugger_state()->db_current_op
222#define CPUDEBUGGERMSG current_debugger_state()->db_message
223#define CPUPANICSTR current_debugger_state()->db_panic_str
224#define CPUPANICARGS current_debugger_state()->db_panic_args
225#define CPUPANICOPTS current_debugger_state()->db_panic_options
226#define CPUPANICDATAPTR current_debugger_state()->db_panic_data_ptr
227#define CPUDEBUGGERSYNC current_debugger_state()->db_proceed_on_sync_failure
228#define CPUDEBUGGERCOUNT current_debugger_state()->db_entry_count
229#define CPUDEBUGGERRET current_debugger_state()->db_op_return
230#define CPUPANICCALLER current_debugger_state()->db_panic_caller
231#define CPUPANICINITIATOR current_debugger_state()->db_panic_initiator
232
233
234/*
235 * Usage:
236 * panic_test_action_count is in the context of other flags, e.g. for IO errors it is "succeed this many times then fail" and for nesting it is "panic this many times then succeed"
237 * panic_test_failure_mode is a bit map of things to do
238 * panic_test_case is what sort of test we are injecting
239 *
240 * For more details see definitions in debugger.h
241 *
242 * Note that not all combinations are sensible, but some actions can be combined, e.g.
243 * - BADPTR+SPIN with action count = 3 will cause panic->panic->spin
244 * - BADPTR with action count = 2 will cause 2 nested panics (in addition to the initial panic)
245 * - IO_ERR with action 15 will cause 14 successful IOs, then fail on the next one
246 */
247#if DEVELOPMENT || DEBUG
248#define INJECT_NESTED_PANIC_IF_REQUESTED(requested) \
249MACRO_BEGIN \
250 if ((panic_test_case & requested) && panic_test_action_count) { \
251 panic_test_action_count--; \
252 volatile int *panic_test_badpointer = (int *)4; \
253 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_SPIN) && (!panic_test_action_count)) { printf("inject spin...\n"); while(panic_test_badpointer); } \
254 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_BADPTR) && (panic_test_action_count+1)) { printf("inject badptr...\n"); *panic_test_badpointer = 0; } \
255 if ((panic_test_failure_mode & PANIC_TEST_FAILURE_MODE_PANIC) && (panic_test_action_count+1)) { printf("inject panic...\n"); panic("nested panic level %d", panic_test_action_count); } \
256 } \
257MACRO_END
258
259#endif /* DEVELOPMENT || DEBUG */
260
261debugger_op debugger_current_op = DBOP_NONE;
262const char *debugger_panic_str = NULL;
263va_list *debugger_panic_args = NULL;
264void *debugger_panic_data = NULL;
265uint64_t debugger_panic_options = 0;
266const char *debugger_message = NULL;
267unsigned long debugger_panic_caller = 0;
268const char *debugger_panic_initiator = "";
269
270void panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args,
271 unsigned int reason, void *ctx, uint64_t panic_options_mask, void *panic_data,
272 unsigned long panic_caller, const char *panic_initiator) __dead2 __printflike(1, 0);
273static void kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags);
274void panic_spin_forever(void) __dead2;
275void panic_stackshot_release_lock(void);
276extern void PE_panic_hook(const char*);
277extern int sync_internal(void);
278
279#define NESTEDDEBUGGERENTRYMAX 5
280static TUNABLE(unsigned int, max_debugger_entry_count, "nested_panic_max",
281 NESTEDDEBUGGERENTRYMAX);
282
283SECURITY_READ_ONLY_LATE(bool) awl_scratch_reg_supported = false;
284static bool PERCPU_DATA(hv_entry_detected); // = false
285static void awl_set_scratch_reg_hv_bit(void);
286void awl_mark_hv_entry(void);
287static bool awl_pm_state_change_cbk(void *param, enum cpu_event event, unsigned int cpu_or_cluster);
288
289#if !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING
290static boolean_t device_corefile_valid_on_ephemeral(void);
291#endif /* !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING */
292
293#if defined(__arm64__)
294#define DEBUG_BUF_SIZE (4096)
295
296/* debug_buf is directly linked with iBoot panic region for arm targets */
297char *debug_buf_base = NULL;
298char *debug_buf_ptr = NULL;
299unsigned int debug_buf_size = 0;
300
301SECURITY_READ_ONLY_LATE(boolean_t) kdp_explicitly_requested = FALSE;
302#else /* defined(__arm64__) */
303#define DEBUG_BUF_SIZE ((3 * PAGE_SIZE) + offsetof(struct macos_panic_header, mph_data))
304/* EXTENDED_DEBUG_BUF_SIZE definition is now in debug.h */
305static_assert(((EXTENDED_DEBUG_BUF_SIZE % PANIC_FLUSH_BOUNDARY) == 0), "Extended debug buf size must match SMC alignment requirements");
306
307char debug_buf[DEBUG_BUF_SIZE];
308struct macos_panic_header *panic_info = (struct macos_panic_header *)debug_buf;
309char *debug_buf_base = (debug_buf + offsetof(struct macos_panic_header, mph_data));
310char *debug_buf_ptr = (debug_buf + offsetof(struct macos_panic_header, mph_data));
311
312/*
313 * We don't include the size of the panic header in the length of the data we actually write.
314 * On co-processor platforms, we lose sizeof(struct macos_panic_header) bytes from the end of
315 * the end of the log because we only support writing (3*PAGESIZE) bytes.
316 */
317unsigned int debug_buf_size = (DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
318
319boolean_t extended_debug_log_enabled = FALSE;
320#endif /* defined(__arm64__) */
321
322#if defined(XNU_TARGET_OS_OSX)
323#define KDBG_TRACE_PANIC_FILENAME "/var/tmp/panic.trace"
324#else
325#define KDBG_TRACE_PANIC_FILENAME "/var/log/panic.trace"
326#endif
327
328static inline boolean_t debug_fatal_panic_begin(void);
329
330/* Debugger state */
331atomic_int debugger_cpu = DEBUGGER_NO_CPU;
332boolean_t debugger_allcpus_halted = FALSE;
333boolean_t debugger_safe_to_return = TRUE;
334unsigned int debugger_context = 0;
335
336static char model_name[64];
337unsigned char *kernel_uuid;
338
339boolean_t kernelcache_uuid_valid = FALSE;
340uuid_t kernelcache_uuid;
341uuid_string_t kernelcache_uuid_string;
342
343boolean_t pageablekc_uuid_valid = FALSE;
344uuid_t pageablekc_uuid;
345uuid_string_t pageablekc_uuid_string;
346
347boolean_t auxkc_uuid_valid = FALSE;
348uuid_t auxkc_uuid;
349uuid_string_t auxkc_uuid_string;
350
351
352/*
353 * By default we treat Debugger() the same as calls to panic(), unless
354 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
355 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
356 *
357 * Return from Debugger() is currently only implemented on x86
358 */
359static boolean_t debugger_is_panic = TRUE;
360
361TUNABLE(unsigned int, debug_boot_arg, "debug", 0);
362
363TUNABLE_DEV_WRITEABLE(unsigned int, verbose_panic_flow_logging, "verbose_panic_flow_logging", 0);
364
365char kernel_uuid_string[37]; /* uuid_string_t */
366char kernelcache_uuid_string[37]; /* uuid_string_t */
367char panic_disk_error_description[512];
368size_t panic_disk_error_description_size = sizeof(panic_disk_error_description);
369
370extern unsigned int write_trace_on_panic;
371int kext_assertions_enable =
372#if DEBUG || DEVELOPMENT
373 TRUE;
374#else
375 FALSE;
376#endif
377
378/*
379 * Maintain the physically-contiguous carveouts for the carveout bootargs.
380 */
381TUNABLE_WRITEABLE(boolean_t, phys_carveout_core, "phys_carveout_core", 1);
382
383TUNABLE(uint32_t, phys_carveout_mb, "phys_carveout_mb", 0);
384SECURITY_READ_ONLY_LATE(vm_offset_t) phys_carveout = 0;
385SECURITY_READ_ONLY_LATE(uintptr_t) phys_carveout_pa = 0;
386SECURITY_READ_ONLY_LATE(size_t) phys_carveout_size = 0;
387
388
389#if CONFIG_SPTM && (DEVELOPMENT || DEBUG)
390/**
391 * Extra debug state which is set when panic lockdown is initiated.
392 * This information is intended to help when debugging issues with the panic
393 * path.
394 */
395struct panic_lockdown_initiator_state debug_panic_lockdown_initiator_state;
396#endif /* CONFIG_SPTM && (DEVELOPMENT || DEBUG) */
397
398/*
399 * Returns whether kernel debugging is expected to be restricted
400 * on the device currently based on CSR or other platform restrictions.
401 */
402boolean_t
403kernel_debugging_restricted(void)
404{
405#if XNU_TARGET_OS_OSX
406#if CONFIG_CSR
407 if (csr_check(CSR_ALLOW_KERNEL_DEBUGGER) != 0) {
408 return TRUE;
409 }
410#endif /* CONFIG_CSR */
411 return FALSE;
412#else /* XNU_TARGET_OS_OSX */
413 return FALSE;
414#endif /* XNU_TARGET_OS_OSX */
415}
416
417__startup_func
418static void
419panic_init(void)
420{
421 unsigned long uuidlen = 0;
422 void *uuid;
423
424 uuid = getuuidfromheader(&_mh_execute_header, &uuidlen);
425 if ((uuid != NULL) && (uuidlen == sizeof(uuid_t))) {
426 kernel_uuid = uuid;
427 uuid_unparse_upper(*(uuid_t *)uuid, kernel_uuid_string);
428 }
429
430 /*
431 * Take the value of the debug boot-arg into account
432 */
433#if MACH_KDP
434 if (!kernel_debugging_restricted() && debug_boot_arg) {
435 if (debug_boot_arg & DB_HALT) {
436 halt_in_debugger = 1;
437 }
438
439#if defined(__arm64__)
440 if (debug_boot_arg & DB_NMI) {
441 panicDebugging = TRUE;
442 }
443#else
444 panicDebugging = TRUE;
445#endif /* defined(__arm64__) */
446 }
447
448#if defined(__arm64__)
449 char kdpname[80];
450
451 kdp_explicitly_requested = PE_parse_boot_argn("kdp_match_name", kdpname, sizeof(kdpname));
452#endif /* defined(__arm64__) */
453
454#endif /* MACH_KDP */
455
456#if defined (__x86_64__)
457 /*
458 * By default we treat Debugger() the same as calls to panic(), unless
459 * we have debug boot-args present and the DB_KERN_DUMP_ON_NMI *NOT* set.
460 * If DB_KERN_DUMP_ON_NMI is *NOT* set, return from Debugger() is supported.
461 * This is because writing an on-device corefile is a destructive operation.
462 *
463 * Return from Debugger() is currently only implemented on x86
464 */
465 if (PE_i_can_has_debugger(NULL) && !(debug_boot_arg & DB_KERN_DUMP_ON_NMI)) {
466 debugger_is_panic = FALSE;
467 }
468#endif
469}
470STARTUP(TUNABLES, STARTUP_RANK_MIDDLE, panic_init);
471
472#if defined (__x86_64__)
473void
474extended_debug_log_init(void)
475{
476 assert(coprocessor_paniclog_flush);
477 /*
478 * Allocate an extended panic log buffer that has space for the panic
479 * stackshot at the end. Update the debug buf pointers appropriately
480 * to point at this new buffer.
481 *
482 * iBoot pre-initializes the panic region with the NULL character. We set this here
483 * so we can accurately calculate the CRC for the region without needing to flush the
484 * full region over SMC.
485 */
486 char *new_debug_buf = kalloc_data(EXTENDED_DEBUG_BUF_SIZE, Z_WAITOK | Z_ZERO);
487
488 panic_info = (struct macos_panic_header *)new_debug_buf;
489 debug_buf_ptr = debug_buf_base = (new_debug_buf + offsetof(struct macos_panic_header, mph_data));
490 debug_buf_size = (EXTENDED_DEBUG_BUF_SIZE - offsetof(struct macos_panic_header, mph_data));
491
492 extended_debug_log_enabled = TRUE;
493
494 /*
495 * Insert a compiler barrier so we don't free the other panic stackshot buffer
496 * until after we've marked the new one as available
497 */
498 __compiler_barrier();
499 kmem_free(kernel_map, panic_stackshot_buf, panic_stackshot_buf_len);
500 panic_stackshot_buf = 0;
501 panic_stackshot_buf_len = 0;
502}
503#endif /* defined (__x86_64__) */
504
505void
506debug_log_init(void)
507{
508#if defined(__arm64__)
509 if (!gPanicBase) {
510 printf("debug_log_init: Error!! gPanicBase is still not initialized\n");
511 return;
512 }
513 /* Shift debug buf start location and size by the length of the panic header */
514 debug_buf_base = (char *)gPanicBase + sizeof(struct embedded_panic_header);
515 debug_buf_ptr = debug_buf_base;
516 debug_buf_size = gPanicSize - sizeof(struct embedded_panic_header);
517
518#if CONFIG_EXT_PANICLOG
519 ext_paniclog_init();
520#endif
521#else
522 kern_return_t kr = KERN_SUCCESS;
523 bzero(panic_info, DEBUG_BUF_SIZE);
524
525 assert(debug_buf_base != NULL);
526 assert(debug_buf_ptr != NULL);
527 assert(debug_buf_size != 0);
528
529 /*
530 * We allocate a buffer to store a panic time stackshot. If we later discover that this is a
531 * system that supports flushing a stackshot via an extended debug log (see above), we'll free this memory
532 * as it's not necessary on this platform. This information won't be available until the IOPlatform has come
533 * up.
534 */
535 kr = kmem_alloc(kernel_map, &panic_stackshot_buf, PANIC_STACKSHOT_BUFSIZE,
536 KMA_DATA_SHARED | KMA_ZERO, VM_KERN_MEMORY_DIAG);
537 assert(kr == KERN_SUCCESS);
538 if (kr == KERN_SUCCESS) {
539 panic_stackshot_buf_len = PANIC_STACKSHOT_BUFSIZE;
540 }
541#endif
542}
543
544void
545phys_carveout_init(void)
546{
547 if (!PE_i_can_has_debugger(NULL)) {
548 return;
549 }
550
551#if __arm__ || __arm64__
552#if DEVELOPMENT || DEBUG
553#endif /* DEVELOPMENT || DEBUG */
554#endif /* __arm__ || __arm64__ */
555
556 struct carveout {
557 const char *name;
558 vm_offset_t *va;
559 uint32_t requested_size;
560 uintptr_t *pa;
561 size_t *allocated_size;
562 uint64_t present;
563 } carveouts[] = {
564 {
565 "phys_carveout",
566 &phys_carveout,
567 phys_carveout_mb,
568 &phys_carveout_pa,
569 &phys_carveout_size,
570 phys_carveout_mb != 0,
571 }
572 };
573
574 for (int i = 0; i < (sizeof(carveouts) / sizeof(struct carveout)); i++) {
575 if (carveouts[i].present) {
576 size_t temp_carveout_size = 0;
577 if (os_mul_overflow(carveouts[i].requested_size, 1024 * 1024, &temp_carveout_size)) {
578 panic("%s_mb size overflowed (%uMB)",
579 carveouts[i].name, carveouts[i].requested_size);
580 return;
581 }
582
583 kmem_alloc_contig(kernel_map, carveouts[i].va,
584 temp_carveout_size, PAGE_MASK, 0, 0,
585 KMA_NOFAIL | KMA_PERMANENT | KMA_NOPAGEWAIT | KMA_DATA_SHARED |
586 KMA_NOSOFTLIMIT,
587 VM_KERN_MEMORY_DIAG);
588
589 *carveouts[i].pa = kvtophys(*carveouts[i].va);
590 *carveouts[i].allocated_size = temp_carveout_size;
591 }
592 }
593}
594
595boolean_t
596debug_is_in_phys_carveout(vm_map_offset_t va)
597{
598 return phys_carveout_size && va >= phys_carveout &&
599 va < (phys_carveout + phys_carveout_size);
600}
601
602boolean_t
603debug_can_coredump_phys_carveout(void)
604{
605 return phys_carveout_core;
606}
607
608static boolean_t
609DebuggerLock(void)
610{
611 int my_cpu = cpu_number();
612 int debugger_exp_cpu = DEBUGGER_NO_CPU;
613 assert(ml_get_interrupts_enabled() == FALSE);
614
615 if (atomic_load(&debugger_cpu) == my_cpu) {
616 return true;
617 }
618
619 if (!atomic_compare_exchange_strong(&debugger_cpu, &debugger_exp_cpu, my_cpu)) {
620 return false;
621 }
622
623 return true;
624}
625
626static void
627DebuggerUnlock(void)
628{
629 assert(atomic_load_explicit(&debugger_cpu, memory_order_relaxed) == cpu_number());
630
631 /*
632 * We don't do an atomic exchange here in case
633 * there's another CPU spinning to acquire the debugger_lock
634 * and we never get a chance to update it. We already have the
635 * lock so we can simply store DEBUGGER_NO_CPU and follow with
636 * a barrier.
637 */
638 atomic_store(&debugger_cpu, DEBUGGER_NO_CPU);
639 OSMemoryBarrier();
640
641 return;
642}
643
644static kern_return_t
645DebuggerHaltOtherCores(boolean_t proceed_on_failure, bool is_stackshot)
646{
647#if defined(__arm64__)
648 return DebuggerXCallEnter(proceed_on_failure, is_stackshot);
649#else /* defined(__arm64__) */
650#pragma unused(proceed_on_failure)
651 mp_kdp_enter(proceed_on_failure, is_stackshot);
652 return KERN_SUCCESS;
653#endif
654}
655
656static void
657DebuggerResumeOtherCores(void)
658{
659#if defined(__arm64__)
660 DebuggerXCallReturn();
661#else /* defined(__arm64__) */
662 mp_kdp_exit();
663#endif
664}
665
666__printflike(3, 0)
667static void
668DebuggerSaveState(debugger_op db_op, const char *db_message, const char *db_panic_str,
669 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
670 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char *db_panic_initiator)
671{
672 CPUDEBUGGEROP = db_op;
673
674 /*
675 * Note:
676 * if CPUDEBUGGERCOUNT == 1 then we are in the normal case - record the panic data
677 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR == NULL then we are in a nested panic that happened before DebuggerSaveState was called, so store the nested panic data
678 * if CPUDEBUGGERCOUNT > 1 and CPUPANICSTR != NULL then we are in a nested panic that happened after DebuggerSaveState was called, so leave the original panic data
679 *
680 * TODO: is it safe to flatten this to if (CPUPANICSTR == NULL)?
681 */
682 if (CPUDEBUGGERCOUNT == 1 || CPUPANICSTR == NULL) {
683 CPUDEBUGGERMSG = db_message;
684 CPUPANICSTR = db_panic_str;
685 CPUPANICARGS = db_panic_args;
686 CPUPANICDATAPTR = db_panic_data_ptr;
687 CPUPANICCALLER = db_panic_caller;
688 CPUPANICINITIATOR = db_panic_initiator;
689
690#if CONFIG_EXCLAVES
691 char *panic_str;
692 if (exclaves_panic_get_string(&panic_str) == KERN_SUCCESS) {
693 CPUPANICSTR = panic_str;
694 }
695#endif
696 }
697
698 CPUDEBUGGERSYNC = db_proceed_on_sync_failure;
699 CPUDEBUGGERRET = KERN_SUCCESS;
700
701 /* Reset these on any nested panics */
702 // follow up in rdar://88497308 (nested panics should not clobber panic flags)
703 CPUPANICOPTS = db_panic_options;
704
705 return;
706}
707
708/*
709 * Save the requested debugger state/action into the current processor's
710 * percu state and trap to the debugger.
711 */
712kern_return_t
713DebuggerTrapWithState(debugger_op db_op, const char *db_message, const char *db_panic_str,
714 va_list *db_panic_args, uint64_t db_panic_options, void *db_panic_data_ptr,
715 boolean_t db_proceed_on_sync_failure, unsigned long db_panic_caller, const char* db_panic_initiator)
716{
717 kern_return_t ret;
718
719#if defined(__arm64__) && (DEVELOPMENT || DEBUG)
720 if (!PE_arm_debug_and_trace_initialized()) {
721 /*
722 * In practice this can only happen if we panicked very early,
723 * when only the boot CPU is online and before it has finished
724 * initializing the debug and trace infrastructure. We're going
725 * to hang soon, so let's at least make sure the message passed
726 * to panic() is actually logged.
727 */
728 char buf[EARLY_PANIC_BUFLEN];
729 vsnprintf(buf, EARLY_PANIC_BUFLEN, db_panic_str, *db_panic_args);
730 paniclog_append_noflush("%s\n", buf);
731 }
732#endif
733
734 assert(ml_get_interrupts_enabled() == FALSE);
735 DebuggerSaveState(db_op, db_message, db_panic_str, db_panic_args,
736 db_panic_options, db_panic_data_ptr,
737 db_proceed_on_sync_failure, db_panic_caller, db_panic_initiator);
738
739 /*
740 * On ARM this generates an uncategorized exception -> sleh code ->
741 * DebuggerCall -> kdp_trap -> handle_debugger_trap
742 * So that is how XNU ensures that only one core can panic.
743 * The rest of the cores are halted by IPI if possible; if that
744 * fails it will fall back to dbgwrap.
745 */
746 TRAP_DEBUGGER;
747
748 ret = CPUDEBUGGERRET;
749
750 DebuggerSaveState(DBOP_NONE, NULL, NULL, NULL, 0, NULL, FALSE, 0, NULL);
751
752 return ret;
753}
754
755void __attribute__((noinline))
756Assert(const char*file, int line, const char *expression)
757{
758 panic_plain("%s:%d Assertion failed: %s", file, line, expression);
759}
760
761void
762panic_assert_format(char *buf, size_t len, struct mach_assert_hdr *hdr, long a, long b)
763{
764 struct mach_assert_default *adef;
765 struct mach_assert_3x *a3x;
766
767 static_assert(MACH_ASSERT_TRAP_CODE == XNU_HARD_TRAP_ASSERT_FAILURE);
768
769 switch (hdr->type) {
770 case MACH_ASSERT_DEFAULT:
771 adef = __container_of(hdr, struct mach_assert_default, hdr);
772 snprintf(buf, len, "%s:%d Assertion failed: %s",
773 hdr->filename, hdr->lineno, adef->expr);
774 break;
775
776 case MACH_ASSERT_3P:
777 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
778 snprintf(buf, len, "%s:%d Assertion failed: "
779 "%s %s %s (%p %s %p)",
780 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
781 (void *)a, a3x->op, (void *)b);
782 break;
783
784 case MACH_ASSERT_3S:
785 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
786 snprintf(buf, len, "%s:%d Assertion failed: "
787 "%s %s %s (0x%lx %s 0x%lx, %ld %s %ld)",
788 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
789 a, a3x->op, b, a, a3x->op, b);
790 break;
791
792 case MACH_ASSERT_3U:
793 a3x = __container_of(hdr, struct mach_assert_3x, hdr);
794 snprintf(buf, len, "%s:%d Assertion failed: "
795 "%s %s %s (0x%lx %s 0x%lx, %lu %s %lu)",
796 hdr->filename, hdr->lineno, a3x->a, a3x->op, a3x->b,
797 a, a3x->op, b, a, a3x->op, b);
798 break;
799 }
800}
801
802boolean_t
803debug_is_current_cpu_in_panic_state(void)
804{
805 return current_debugger_state()->db_entry_count > 0;
806}
807
808/*
809 * check if we are in a nested panic, report findings, take evasive action where necessary
810 *
811 * see also PE_update_panicheader_nestedpanic
812 */
813static void
814check_and_handle_nested_panic(uint64_t panic_options_mask, unsigned long panic_caller, const char *db_panic_str, va_list *db_panic_args)
815{
816 if ((CPUDEBUGGERCOUNT > 1) && (CPUDEBUGGERCOUNT < max_debugger_entry_count)) {
817 // Note: this is the first indication in the panic log or serial that we are off the rails...
818 //
819 // if we panic *before* the paniclog is finalized then this will end up in the ips report with a panic_caller addr that gives us a clue
820 // if we panic *after* the log is finalized then we will only see it in the serial log
821 //
822 paniclog_append_noflush("Nested panic detected - entry count: %d panic_caller: 0x%016lx\n", CPUDEBUGGERCOUNT, panic_caller);
823 print_curr_backtrace();
824 paniclog_flush();
825
826 // print the *new* panic string to the console, we might not get it by other means...
827 // TODO: I tried to write this stuff to the paniclog, but the serial output gets corrupted and the panicstring in the ips file is <mysterious>
828 // rdar://87846117 (NestedPanic: output panic string to paniclog)
829 if (db_panic_str) {
830 printf("Nested panic string:\n");
831#pragma clang diagnostic push
832#pragma clang diagnostic ignored "-Wformat-nonliteral"
833#pragma clang diagnostic ignored "-Wformat"
834 _doprnt(db_panic_str, db_panic_args, PE_kputc, 0);
835#pragma clang diagnostic pop
836 printf("\n<end nested panic string>\n");
837 }
838 }
839
840 // Stage 1 bailout
841 //
842 // Try to complete the normal panic flow, i.e. try to make sure the callouts happen and we flush the paniclog. If this fails with another nested
843 // panic then we will land in Stage 2 below...
844 //
845 if (CPUDEBUGGERCOUNT == max_debugger_entry_count) {
846 uint32_t panic_details = 0;
847
848 // if this is a force-reset panic then capture a log and reboot immediately.
849 if (panic_options_mask & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
850 panic_details |= kPanicDetailsForcePowerOff;
851 }
852
853 // normally the kPEPanicBegin is sent from debugger_collect_diagnostics(), but we might nested-panic before we get
854 // there. To be safe send another notification, the function called below will only send kPEPanicBegin if it has not yet been sent.
855 //
856 PEHaltRestartInternal(kPEPanicBegin, panic_details);
857
858 paniclog_append_noflush("Nested panic count exceeds limit %d, machine will reset or spin\n", max_debugger_entry_count);
859 PE_update_panicheader_nestedpanic();
860 paniclog_flush();
861
862 if (!panicDebugging) {
863 // note that this will also send kPEPanicEnd
864 kdp_machine_reboot_type(kPEPanicRestartCPU, panic_options_mask);
865 }
866
867 // prints to console
868 paniclog_append_noflush("\nNested panic stall. Stage 1 bailout. Please go to https://panic.apple.com to report this panic\n");
869 panic_spin_forever();
870 }
871
872 // Stage 2 bailout
873 //
874 // Things are severely hosed, we have nested to the point of bailout and then nested again during the bailout path. Try to issue
875 // a chipreset as quickly as possible, hopefully something in the panic log is salvageable, since we flushed it during Stage 1.
876 //
877 if (CPUDEBUGGERCOUNT == max_debugger_entry_count + 1) {
878 if (!panicDebugging) {
879 // note that:
880 // - this code path should be audited for prints, as that is a common cause of nested panics
881 // - this code path should take the fastest route to the actual reset, and not call any un-necessary code
882 kdp_machine_reboot_type(kPEPanicRestartCPU, panic_options_mask & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS);
883 }
884
885 // prints to console, but another nested panic will land in Stage 3 where we simply spin, so that is sort of ok...
886 paniclog_append_noflush("\nIn Nested panic stall. Stage 2 bailout. Please go to https://panic.apple.com to report this panic\n");
887 panic_spin_forever();
888 }
889
890 // Stage 3 bailout
891 //
892 // We are done here, we were unable to reset the platform without another nested panic. Spin until the watchdog kicks in.
893 //
894 if (CPUDEBUGGERCOUNT > max_debugger_entry_count + 1) {
895 kdp_machine_reboot_type(kPEHangCPU, 0);
896 }
897}
898
899void
900Debugger(const char *message)
901{
902 DebuggerWithContext(0, NULL, message, DEBUGGER_OPTION_NONE, (unsigned long)(char *)__builtin_return_address(0));
903}
904
905/*
906 * Enter the Debugger
907 *
908 * This is similar to, but not the same as a panic
909 *
910 * Key differences:
911 * - we get here from a debugger entry action (e.g. NMI)
912 * - the system is resumable on x86 (in theory, however it is not clear if this is tested)
913 * - rdar://57738811 (xnu: support resume from debugger via KDP on arm devices)
914 *
915 */
916void
917DebuggerWithContext(unsigned int reason, void *ctx, const char *message,
918 uint64_t debugger_options_mask, unsigned long debugger_caller)
919{
920 spl_t previous_interrupts_state;
921 boolean_t old_doprnt_hide_pointers = doprnt_hide_pointers;
922
923#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
924 read_lbr();
925#endif
926 previous_interrupts_state = ml_set_interrupts_enabled(FALSE);
927 disable_preemption();
928
929 /* track depth of debugger/panic entry */
930 CPUDEBUGGERCOUNT++;
931
932 /* emit a tracepoint as early as possible in case of hang */
933 SOCD_TRACE_XNU(PANIC,
934 ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
935 PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
936 VALUE(debugger_options_mask),
937 ADDR(message),
938 ADDR(debugger_caller));
939
940 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
941 check_and_handle_nested_panic(debugger_options_mask, debugger_caller, message, NULL);
942
943 /* Handle any necessary platform specific actions before we proceed */
944 PEInitiatePanic();
945
946#if DEVELOPMENT || DEBUG
947 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
948#endif
949
950 PE_panic_hook(message);
951
952 doprnt_hide_pointers = FALSE;
953
954 if (ctx != NULL) {
955 DebuggerSaveState(DBOP_DEBUGGER, message,
956 NULL, NULL, debugger_options_mask, NULL, TRUE, 0, "");
957 handle_debugger_trap(reason, 0, 0, ctx);
958 DebuggerSaveState(DBOP_NONE, NULL, NULL,
959 NULL, 0, NULL, FALSE, 0, "");
960 } else {
961 DebuggerTrapWithState(DBOP_DEBUGGER, message,
962 NULL, NULL, debugger_options_mask, NULL, TRUE, 0, NULL);
963 }
964
965 /* resume from the debugger */
966
967 CPUDEBUGGERCOUNT--;
968 doprnt_hide_pointers = old_doprnt_hide_pointers;
969 enable_preemption();
970 ml_set_interrupts_enabled(previous_interrupts_state);
971}
972
973static struct kdp_callout {
974 struct kdp_callout * callout_next;
975 kdp_callout_fn_t callout_fn;
976 boolean_t callout_in_progress;
977 void * callout_arg;
978} * kdp_callout_list = NULL;
979
980/*
981 * Called from kernel context to register a kdp event callout.
982 */
983void
984kdp_register_callout(kdp_callout_fn_t fn, void * arg)
985{
986 struct kdp_callout * kcp;
987 struct kdp_callout * list_head;
988
989 kcp = zalloc_permanent_type(struct kdp_callout);
990
991 kcp->callout_fn = fn;
992 kcp->callout_arg = arg;
993 kcp->callout_in_progress = FALSE;
994
995 /* Lock-less list insertion using compare and exchange. */
996 do {
997 list_head = kdp_callout_list;
998 kcp->callout_next = list_head;
999 } while (!OSCompareAndSwapPtr(list_head, kcp, &kdp_callout_list));
1000}
1001
1002static void
1003kdp_callouts(kdp_event_t event)
1004{
1005 struct kdp_callout *kcp = kdp_callout_list;
1006
1007 while (kcp) {
1008 if (!kcp->callout_in_progress) {
1009 kcp->callout_in_progress = TRUE;
1010 kcp->callout_fn(kcp->callout_arg, event);
1011 kcp->callout_in_progress = FALSE;
1012 }
1013 kcp = kcp->callout_next;
1014 }
1015}
1016
1017#if defined(__arm64__)
1018/*
1019 * Register an additional buffer with data to include in the panic log
1020 *
1021 * <rdar://problem/50137705> tracks supporting more than one buffer
1022 *
1023 * Note that producer_name and buf should never be de-allocated as we reference these during panic.
1024 */
1025void
1026register_additional_panic_data_buffer(const char *producer_name, void *buf, int len)
1027{
1028 if (panic_data_buffers != NULL) {
1029 panic("register_additional_panic_data_buffer called with buffer already registered");
1030 }
1031
1032 if (producer_name == NULL || (strlen(producer_name) == 0)) {
1033 panic("register_additional_panic_data_buffer called with invalid producer_name");
1034 }
1035
1036 if (buf == NULL) {
1037 panic("register_additional_panic_data_buffer called with invalid buffer pointer");
1038 }
1039
1040 if ((len <= 0) || (len > ADDITIONAL_PANIC_DATA_BUFFER_MAX_LEN)) {
1041 panic("register_additional_panic_data_buffer called with invalid length");
1042 }
1043
1044 struct additional_panic_data_buffer *new_panic_data_buffer = zalloc_permanent_type(struct additional_panic_data_buffer);
1045 new_panic_data_buffer->producer_name = producer_name;
1046 new_panic_data_buffer->buf = buf;
1047 new_panic_data_buffer->len = len;
1048
1049 if (!OSCompareAndSwapPtr(NULL, new_panic_data_buffer, &panic_data_buffers)) {
1050 panic("register_additional_panic_data_buffer called with buffer already registered");
1051 }
1052
1053 return;
1054}
1055#endif /* defined(__arm64__) */
1056
1057/*
1058 * An overview of the xnu panic path:
1059 *
1060 * Several panic wrappers (panic(), panic_with_options(), etc.) all funnel into panic_trap_to_debugger().
1061 * panic_trap_to_debugger() sets the panic state in the current processor's debugger_state prior
1062 * to trapping into the debugger. Once we trap to the debugger, we end up in handle_debugger_trap()
1063 * which tries to acquire the panic lock by atomically swapping the current CPU number into debugger_cpu.
1064 * debugger_cpu acts as a synchronization point, from which the winning CPU can halt the other cores and
1065 * continue to debugger_collect_diagnostics() where we write the paniclog, corefile (if appropriate) and proceed
1066 * according to the device's boot-args.
1067 */
1068#undef panic
1069void
1070panic(const char *str, ...)
1071{
1072 va_list panic_str_args;
1073
1074 va_start(panic_str_args, str);
1075 panic_trap_to_debugger(str, &panic_str_args, 0, NULL, 0, NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1076 va_end(panic_str_args);
1077}
1078
1079void
1080panic_with_data(uuid_t uuid, void *addr, uint32_t len, uint64_t debugger_options_mask, const char *str, ...)
1081{
1082 va_list panic_str_args;
1083
1084 ext_paniclog_panic_with_data(uuid, addr, len);
1085
1086#if CONFIG_EXCLAVES
1087 /*
1088 * Before trapping, inform the exclaves scheduler that we're going down
1089 * so it can grab an exclaves stackshot.
1090 */
1091 if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
1092 exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
1093 (void) exclaves_scheduler_request_watchdog_panic();
1094 }
1095#endif /* CONFIG_EXCLAVES */
1096
1097 va_start(panic_str_args, str);
1098 panic_trap_to_debugger(str, &panic_str_args, 0, NULL, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1099 NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1100 va_end(panic_str_args);
1101}
1102
1103void
1104panic_with_options(unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
1105{
1106 va_list panic_str_args;
1107
1108#if CONFIG_EXCLAVES
1109 /*
1110 * Before trapping, inform the exclaves scheduler that we're going down
1111 * so it can grab an exclaves stackshot.
1112 */
1113 if ((debugger_options_mask & DEBUGGER_OPTION_USER_WATCHDOG) != 0 &&
1114 exclaves_get_boot_stage() != EXCLAVES_BOOT_STAGE_NONE) {
1115 (void) exclaves_scheduler_request_watchdog_panic();
1116 }
1117#endif /* CONFIG_EXCLAVES */
1118
1119 va_start(panic_str_args, str);
1120 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1121 NULL, (unsigned long)(char *)__builtin_return_address(0), NULL);
1122 va_end(panic_str_args);
1123}
1124
1125void
1126panic_with_options_and_initiator(const char* initiator, unsigned int reason, void *ctx, uint64_t debugger_options_mask, const char *str, ...)
1127{
1128 va_list panic_str_args;
1129
1130 va_start(panic_str_args, str);
1131 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, (debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK),
1132 NULL, (unsigned long)(char *)__builtin_return_address(0), initiator);
1133 va_end(panic_str_args);
1134}
1135
1136boolean_t
1137panic_validate_ptr(void *ptr, vm_size_t size, const char *what)
1138{
1139 if (ptr == NULL) {
1140 paniclog_append_noflush("NULL %s pointer\n", what);
1141 return false;
1142 }
1143
1144 if (!ml_validate_nofault((vm_offset_t)ptr, size)) {
1145 paniclog_append_noflush("Invalid %s pointer: %p (size %d)\n",
1146 what, ptr, (uint32_t)size);
1147 return false;
1148 }
1149
1150 return true;
1151}
1152
1153boolean_t
1154panic_get_thread_proc_task(struct thread *thread, struct task **task, struct proc **proc)
1155{
1156 if (!PANIC_VALIDATE_PTR(thread)) {
1157 return false;
1158 }
1159
1160 if (!PANIC_VALIDATE_PTR(thread->t_tro)) {
1161 return false;
1162 }
1163
1164 if (!PANIC_VALIDATE_PTR(thread->t_tro->tro_task)) {
1165 return false;
1166 }
1167
1168 if (task) {
1169 *task = thread->t_tro->tro_task;
1170 }
1171
1172 if (!panic_validate_ptr(thread->t_tro->tro_proc,
1173 sizeof(struct proc *), "bsd_info")) {
1174 *proc = NULL;
1175 } else {
1176 *proc = thread->t_tro->tro_proc;
1177 }
1178
1179 return true;
1180}
1181
1182#if defined (__x86_64__)
1183/*
1184 * panic_with_thread_context() is used on x86 platforms to specify a different thread that should be backtraced in the paniclog.
1185 * We don't generally need this functionality on embedded platforms because embedded platforms include a panic time stackshot
1186 * from customer devices. We plumb the thread pointer via the debugger trap mechanism and backtrace the kernel stack from the
1187 * thread when writing the panic log.
1188 *
1189 * NOTE: panic_with_thread_context() should be called with an explicit thread reference held on the passed thread.
1190 */
1191void
1192panic_with_thread_context(unsigned int reason, void *ctx, uint64_t debugger_options_mask, thread_t thread, const char *str, ...)
1193{
1194 va_list panic_str_args;
1195 __assert_only os_ref_count_t th_ref_count;
1196
1197 assert_thread_magic(thread);
1198 th_ref_count = os_ref_get_count_raw(&thread->ref_count);
1199 assertf(th_ref_count > 0, "panic_with_thread_context called with invalid thread %p with refcount %u", thread, th_ref_count);
1200
1201 /* Take a reference on the thread so it doesn't disappear by the time we try to backtrace it */
1202 thread_reference(thread);
1203
1204 va_start(panic_str_args, str);
1205 panic_trap_to_debugger(str, &panic_str_args, reason, ctx, ((debugger_options_mask & ~DEBUGGER_INTERNAL_OPTIONS_MASK) | DEBUGGER_INTERNAL_OPTION_THREAD_BACKTRACE),
1206 thread, (unsigned long)(char *)__builtin_return_address(0), "");
1207
1208 va_end(panic_str_args);
1209}
1210#endif /* defined (__x86_64__) */
1211
1212#pragma clang diagnostic push
1213#pragma clang diagnostic ignored "-Wmissing-noreturn"
1214__mockable void
1215panic_trap_to_debugger(const char *panic_format_str, va_list *panic_args, unsigned int reason, void *ctx,
1216 uint64_t panic_options_mask, void *panic_data_ptr, unsigned long panic_caller, const char *panic_initiator)
1217{
1218#pragma clang diagnostic pop
1219
1220#if defined(__x86_64__) && (DEVELOPMENT || DEBUG)
1221 read_lbr();
1222#endif
1223
1224 /* optionally call sync, to reduce lost logs on restart, avoid on recursive panic. Unsafe due to unbounded sync() duration */
1225 if ((panic_options_mask & DEBUGGER_OPTION_SYNC_ON_PANIC_UNSAFE) && (CPUDEBUGGERCOUNT == 0)) {
1226 sync_internal();
1227 }
1228
1229 /* Turn off I/O tracing once we've panicked */
1230 iotrace_disable();
1231
1232 /* call machine-layer panic handler */
1233 ml_panic_trap_to_debugger(panic_format_str, panic_args, reason, ctx, panic_options_mask, panic_caller, panic_initiator);
1234
1235 /* track depth of debugger/panic entry */
1236 CPUDEBUGGERCOUNT++;
1237
1238 __unused uint32_t panic_initiator_crc = panic_initiator ? crc32(0, panic_initiator, strnlen(panic_initiator, MAX_PANIC_INITIATOR_SIZE)) : 0;
1239
1240 /* emit a tracepoint as early as possible in case of hang */
1241 SOCD_TRACE_XNU(PANIC,
1242 ((CPUDEBUGGERCOUNT <= 2) ? SOCD_TRACE_MODE_STICKY_TRACEPOINT : SOCD_TRACE_MODE_NONE),
1243 PACK_2X32(VALUE(cpu_number()), VALUE(CPUDEBUGGERCOUNT)),
1244 PACK_2X32(VALUE(panic_initiator_crc), VALUE(panic_options_mask & 0xFFFFFFFF)),
1245 ADDR(panic_format_str),
1246 ADDR(panic_caller));
1247
1248 /* do max nested panic/debugger check, this will report nesting to the console and spin forever if we exceed a limit */
1249 check_and_handle_nested_panic(panic_options_mask, panic_caller, panic_format_str, panic_args);
1250
1251 /* If we're in a stackshot, signal that we've started panicking and wait for other CPUs to coalesce and spin before proceeding */
1252 stackshot_cpu_signal_panic();
1253
1254 /* Handle any necessary platform specific actions before we proceed */
1255 PEInitiatePanic();
1256
1257#if DEVELOPMENT || DEBUG
1258 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_ENTRY);
1259#endif
1260
1261 PE_panic_hook(panic_format_str);
1262
1263#if defined (__x86_64__)
1264 plctrace_disable();
1265#endif
1266
1267 if (write_trace_on_panic && kdebug_enable) {
1268 if (get_preemption_level() == 0 && !ml_at_interrupt_context()) {
1269 ml_set_interrupts_enabled(TRUE);
1270 KDBG_RELEASE(TRACE_PANIC);
1271 kdbg_dump_trace_to_file(KDBG_TRACE_PANIC_FILENAME, false);
1272 }
1273 }
1274
1275 ml_set_interrupts_enabled(FALSE);
1276 disable_preemption();
1277
1278 if (!debug_fatal_panic_begin()) {
1279 /*
1280 * This CPU lost the race to be the first to panic. Re-enable
1281 * interrupts and dead loop here awaiting the debugger xcall from
1282 * the CPU that first panicked.
1283 */
1284 ml_set_interrupts_enabled(TRUE);
1285 panic_stop();
1286 }
1287
1288#if defined (__x86_64__)
1289 pmSafeMode(x86_lcpu(), PM_SAFE_FL_SAFE);
1290#endif /* defined (__x86_64__) */
1291
1292 /* Never hide pointers from panic logs. */
1293 doprnt_hide_pointers = FALSE;
1294
1295 if (ctx != NULL) {
1296 /*
1297 * We called into panic from a trap, no need to trap again. Set the
1298 * state on the current CPU and then jump to handle_debugger_trap.
1299 */
1300 DebuggerSaveState(DBOP_PANIC, "panic",
1301 panic_format_str, panic_args,
1302 panic_options_mask, panic_data_ptr, TRUE, panic_caller, panic_initiator);
1303 handle_debugger_trap(reason, 0, 0, ctx);
1304 }
1305
1306#if defined(__arm64__) && !APPLEVIRTUALPLATFORM
1307 /*
1308 * Signal to fastsim that it should open debug ports (nop on hardware)
1309 */
1310 __asm__ volatile ("hint #0x45");
1311#endif /* defined(__arm64__) && !APPLEVIRTUALPLATFORM */
1312
1313 DebuggerTrapWithState(DBOP_PANIC, "panic", panic_format_str,
1314 panic_args, panic_options_mask, panic_data_ptr, TRUE, panic_caller, panic_initiator);
1315
1316 /*
1317 * Not reached.
1318 */
1319 panic_stop();
1320 __builtin_unreachable();
1321}
1322
1323/* We rely on this symbol being visible in the debugger for triage automation */
1324void __attribute__((noinline, optnone))
1325panic_spin_forever(void)
1326{
1327 for (;;) {
1328#if defined(__arm__) || defined(__arm64__)
1329 /* On arm32, which doesn't have a WFE timeout, this may not return. But that should be OK on this path. */
1330 __builtin_arm_wfe();
1331#else
1332 cpu_pause();
1333#endif
1334 }
1335}
1336
1337void
1338panic_stackshot_release_lock(void)
1339{
1340 assert(!not_in_kdp);
1341 DebuggerUnlock();
1342}
1343
1344static void
1345kdp_machine_reboot_type(unsigned int type, uint64_t debugger_flags)
1346{
1347 if ((type == kPEPanicRestartCPU) && (debugger_flags & DEBUGGER_OPTION_SKIP_PANICEND_CALLOUTS)) {
1348 PEHaltRestart(kPEPanicRestartCPUNoCallouts);
1349 } else {
1350 PEHaltRestart(type);
1351 }
1352 halt_all_cpus(TRUE);
1353}
1354
1355void
1356kdp_machine_reboot(void)
1357{
1358 kdp_machine_reboot_type(kPEPanicRestartCPU, 0);
1359}
1360
1361static __attribute__((unused)) void
1362panic_debugger_log(const char *string, ...)
1363{
1364 va_list panic_debugger_log_args;
1365
1366 va_start(panic_debugger_log_args, string);
1367#pragma clang diagnostic push
1368#pragma clang diagnostic ignored "-Wformat-nonliteral"
1369#pragma clang diagnostic ignored "-Wformat"
1370 _doprnt(string, &panic_debugger_log_args, consdebug_putc, 16);
1371#pragma clang diagnostic pop
1372 va_end(panic_debugger_log_args);
1373
1374#if defined(__arm64__)
1375 paniclog_flush();
1376#endif
1377}
1378
1379/*
1380 * Gather and save diagnostic information about a panic (or Debugger call).
1381 *
1382 * On embedded, Debugger and Panic are treated very similarly -- WDT uses Debugger so we can
1383 * theoretically return from it. On desktop, Debugger is treated as a conventional debugger -- i.e no
1384 * paniclog is written and no core is written unless we request a core on NMI.
1385 *
1386 * This routine handles kicking off local coredumps, paniclogs, calling into the Debugger/KDP (if it's configured),
1387 * and calling out to any other functions we have for collecting diagnostic info.
1388 */
1389static void
1390debugger_collect_diagnostics(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1391{
1392#if DEVELOPMENT || DEBUG
1393 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_PRELOG);
1394#endif
1395
1396#if defined(__x86_64__)
1397 kprintf("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1398#endif
1399 /*
1400 * DB_HALT (halt_in_debugger) can be requested on startup, we shouldn't generate
1401 * a coredump/paniclog for this type of debugger entry. If KDP isn't configured,
1402 * we'll just spin in kdp_raise_exception.
1403 */
1404 if (debugger_current_op == DBOP_DEBUGGER && halt_in_debugger) {
1405 kdp_raise_exception(exception, code, subcode, state);
1406 if (debugger_safe_to_return && !debugger_is_panic) {
1407 return;
1408 }
1409 }
1410
1411#ifdef CONFIG_KCOV
1412 /* Try not to break core dump path by sanitizer. */
1413 kcov_panic_disable();
1414#endif
1415
1416 if ((debugger_current_op == DBOP_PANIC) ||
1417 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1418 /*
1419 * Attempt to notify listeners once and only once that we've started
1420 * panicking. Only do this for Debugger() calls if we're treating
1421 * Debugger() calls like panic().
1422 */
1423 uint32_t panic_details = 0;
1424 /* if this is a force-reset panic then capture a log and reboot immediately. */
1425 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1426 panic_details |= kPanicDetailsForcePowerOff;
1427 }
1428 PEHaltRestartInternal(kPEPanicBegin, panic_details);
1429
1430 /*
1431 * Set the begin pointer in the panic log structure. We key off of this
1432 * static variable rather than contents from the panic header itself in case someone
1433 * has stomped over the panic_info structure. Also initializes the header magic.
1434 */
1435 static boolean_t began_writing_paniclog = FALSE;
1436 if (!began_writing_paniclog) {
1437 PE_init_panicheader();
1438 began_writing_paniclog = TRUE;
1439 }
1440
1441 if (CPUDEBUGGERCOUNT > 1) {
1442 /*
1443 * we are in a nested panic. Record the nested bit in panic flags and do some housekeeping
1444 */
1445 PE_update_panicheader_nestedpanic();
1446 paniclog_flush();
1447 }
1448 }
1449
1450 /*
1451 * Write panic string if this was a panic.
1452 *
1453 * TODO: Consider moving to SavePanicInfo as this is part of the panic log.
1454 */
1455 if (debugger_current_op == DBOP_PANIC) {
1456 paniclog_append_noflush("panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1457 if (debugger_panic_str) {
1458#pragma clang diagnostic push
1459#pragma clang diagnostic ignored "-Wformat-nonliteral"
1460#pragma clang diagnostic ignored "-Wformat"
1461 _doprnt(debugger_panic_str, debugger_panic_args, consdebug_putc, 0);
1462#pragma clang diagnostic pop
1463 }
1464 paniclog_append_noflush("\n");
1465 }
1466#if defined(__x86_64__)
1467 else if (((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1468 paniclog_append_noflush("Debugger called: <%s>\n", debugger_message ? debugger_message : "");
1469 }
1470
1471 /*
1472 * Debugger() is treated like panic() on embedded -- for example we use it for WDT
1473 * panics (so we need to write a paniclog). On desktop Debugger() is used in the
1474 * conventional sense.
1475 */
1476 if (debugger_current_op == DBOP_PANIC || ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic))
1477#endif /* __x86_64__ */
1478 {
1479 kdp_callouts(KDP_EVENT_PANICLOG);
1480
1481 /*
1482 * Write paniclog and panic stackshot (if supported)
1483 * TODO: Need to clear panic log when return from debugger
1484 * hooked up for embedded
1485 */
1486 SavePanicInfo(debugger_message, debugger_panic_data, debugger_panic_options, debugger_panic_initiator);
1487
1488#if DEVELOPMENT || DEBUG
1489 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTLOG);
1490#endif
1491
1492 /* DEBUGGER_OPTION_PANICLOGANDREBOOT is used for two finger resets on embedded so we get a paniclog */
1493 if (debugger_panic_options & DEBUGGER_OPTION_PANICLOGANDREBOOT) {
1494 PEHaltRestart(kPEPanicDiagnosticsDone);
1495 PEHaltRestart(kPEPanicRestartCPUNoCallouts);
1496 }
1497 }
1498
1499#if CONFIG_KDP_INTERACTIVE_DEBUGGING
1500 /*
1501 * If reboot on panic is enabled and the caller of panic indicated that we should skip
1502 * local coredumps, don't try to write these and instead go straight to reboot. This
1503 * allows us to persist any data that's stored in the panic log.
1504 */
1505 if ((debugger_panic_options & DEBUGGER_OPTION_SKIP_LOCAL_COREDUMP) &&
1506 (debug_boot_arg & DB_REBOOT_POST_CORE)) {
1507 PEHaltRestart(kPEPanicDiagnosticsDone);
1508 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1509 }
1510
1511 /*
1512 * Consider generating a local corefile if the infrastructure is configured
1513 * and we haven't disabled on-device coredumps.
1514 */
1515 if (on_device_corefile_enabled()) {
1516#if CONFIG_SPTM
1517 /* We want to skip taking a local core dump if this is a panic from SPTM/TXM/cL4. */
1518 extern uint8_t sptm_supports_local_coredump;
1519 bool sptm_interrupted = false;
1520 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
1521 (void)sptm_get_cpu_state(sptm_pcpu->sptm_cpu_id, CPUSTATE_SPTM_INTERRUPTED, &sptm_interrupted);
1522#endif
1523 if (!kdp_has_polled_corefile()) {
1524 if (debug_boot_arg & (DB_KERN_DUMP_ON_PANIC | DB_KERN_DUMP_ON_NMI)) {
1525 paniclog_append_noflush("skipping local kernel core because core file could not be opened prior to panic (mode : 0x%x, error : 0x%x)\n",
1526 kdp_polled_corefile_mode(), kdp_polled_corefile_error());
1527#if defined(__arm64__)
1528 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1529 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1530 }
1531 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1532 paniclog_flush();
1533#else /* defined(__arm64__) */
1534 if (panic_info->mph_panic_log_offset != 0) {
1535 if (kdp_polled_corefile_mode() == kIOPolledCoreFileModeUnlinked) {
1536 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREFILE_UNLINKED;
1537 }
1538 panic_info->mph_panic_flags |= MACOS_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1539 paniclog_flush();
1540 }
1541#endif /* defined(__arm64__) */
1542 }
1543 }
1544#if XNU_MONITOR
1545 else if (pmap_get_cpu_data()->ppl_state != PPL_STATE_KERNEL) {
1546 paniclog_append_noflush("skipping local kernel core because the PPL is not in KERNEL state\n");
1547 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1548 paniclog_flush();
1549 }
1550#elif CONFIG_SPTM
1551 else if (!sptm_supports_local_coredump) {
1552 paniclog_append_noflush("skipping local kernel core because the SPTM is in PANIC state and can't support core dump generation\n");
1553 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1554 paniclog_flush();
1555 } else if (sptm_interrupted) {
1556 paniclog_append_noflush("skipping local kernel core because the SPTM is in INTERRUPTED state and can't support core dump generation\n");
1557 panic_info->eph_panic_flags |= EMBEDDED_PANIC_HEADER_FLAG_COREDUMP_FAILED;
1558 paniclog_flush();
1559 }
1560#endif /* XNU_MONITOR */
1561 else {
1562 int ret = -1;
1563
1564#if defined (__x86_64__)
1565 /* On x86 we don't do a coredump on Debugger unless the DB_KERN_DUMP_ON_NMI boot-arg is specified. */
1566 if (debugger_current_op != DBOP_DEBUGGER || (debug_boot_arg & DB_KERN_DUMP_ON_NMI))
1567#endif
1568 {
1569 /*
1570 * Doing an on-device coredump leaves the disk driver in a state
1571 * that can not be resumed.
1572 */
1573 debugger_safe_to_return = FALSE;
1574 begin_panic_transfer();
1575 ret = kern_dump(KERN_DUMP_DISK);
1576 abort_panic_transfer();
1577
1578#if DEVELOPMENT || DEBUG
1579 INJECT_NESTED_PANIC_IF_REQUESTED(PANIC_TEST_CASE_RECURPANIC_POSTCORE);
1580#endif
1581 }
1582
1583 /*
1584 * If DB_REBOOT_POST_CORE is set, then reboot if coredump is sucessfully saved
1585 * or if option to ignore failures is set.
1586 */
1587 if ((debug_boot_arg & DB_REBOOT_POST_CORE) &&
1588 ((ret == 0) || (debugger_panic_options & DEBUGGER_OPTION_ATTEMPTCOREDUMPANDREBOOT))) {
1589 PEHaltRestart(kPEPanicDiagnosticsDone);
1590 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1591 }
1592 }
1593 }
1594
1595 if (debugger_current_op == DBOP_PANIC ||
1596 ((debugger_current_op == DBOP_DEBUGGER) && debugger_is_panic)) {
1597 PEHaltRestart(kPEPanicDiagnosticsDone);
1598 }
1599
1600 if (debug_boot_arg & DB_REBOOT_ALWAYS) {
1601 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1602 }
1603
1604 /* If KDP is configured, try to trap to the debugger */
1605#if defined(__arm64__)
1606 if (kdp_explicitly_requested && (current_debugger != NO_CUR_DB)) {
1607#else
1608 if (current_debugger != NO_CUR_DB) {
1609#endif
1610 kdp_raise_exception(exception, code, subcode, state);
1611 /*
1612 * Only return if we entered via Debugger and it's safe to return
1613 * (we halted the other cores successfully, this isn't a nested panic, etc)
1614 */
1615 if (debugger_current_op == DBOP_DEBUGGER &&
1616 debugger_safe_to_return &&
1617 kernel_debugger_entry_count == 1 &&
1618 !debugger_is_panic) {
1619 return;
1620 }
1621 }
1622
1623#if defined(__arm64__)
1624 if (PE_i_can_has_debugger(NULL) && panicDebugging) {
1625 /*
1626 * Print panic string at the end of serial output
1627 * to make panic more obvious when someone connects a debugger
1628 */
1629 if (debugger_panic_str) {
1630 panic_debugger_log("Original panic string:\n");
1631 panic_debugger_log("panic(cpu %u caller 0x%lx): ", (unsigned) cpu_number(), debugger_panic_caller);
1632#pragma clang diagnostic push
1633#pragma clang diagnostic ignored "-Wformat-nonliteral"
1634#pragma clang diagnostic ignored "-Wformat"
1635 _doprnt(debugger_panic_str, debugger_panic_args, consdebug_putc, 0);
1636#pragma clang diagnostic pop
1637 panic_debugger_log("\n");
1638 }
1639
1640 /* If panic debugging is configured and we're on a dev fused device, spin for astris to connect */
1641 panic_spin_shmcon();
1642 }
1643#endif /* defined(__arm64__) */
1644
1645#else /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1646
1647 PEHaltRestart(kPEPanicDiagnosticsDone);
1648
1649#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
1650
1651 if (!panicDebugging) {
1652 kdp_machine_reboot_type(kPEPanicRestartCPU, debugger_panic_options);
1653 }
1654
1655 paniclog_append_noflush("\nPlease go to https://panic.apple.com to report this panic\n");
1656 panic_spin_forever();
1657}
1658
1659#if SCHED_HYGIENE_DEBUG
1660uint64_t debugger_trap_timestamps[9];
1661# define DEBUGGER_TRAP_TIMESTAMP(i) debugger_trap_timestamps[i] = mach_absolute_time();
1662#else
1663# define DEBUGGER_TRAP_TIMESTAMP(i)
1664#endif /* SCHED_HYGIENE_DEBUG */
1665
1666void
1667handle_debugger_trap(unsigned int exception, unsigned int code, unsigned int subcode, void *state)
1668{
1669 unsigned int initial_not_in_kdp = not_in_kdp;
1670 kern_return_t ret = KERN_SUCCESS;
1671 debugger_op db_prev_op = debugger_current_op;
1672
1673 if (!DebuggerLock()) {
1674 /*
1675 * We lost the race to be the first to panic.
1676 * Return here so that we will enter the panic stop
1677 * infinite loop and take the debugger IPI from the
1678 * first CPU that got the debugger lock.
1679 */
1680 return;
1681 }
1682
1683 DEBUGGER_TRAP_TIMESTAMP(0);
1684
1685 ret = DebuggerHaltOtherCores(CPUDEBUGGERSYNC, (CPUDEBUGGEROP == DBOP_STACKSHOT));
1686
1687 DEBUGGER_TRAP_TIMESTAMP(1);
1688
1689#if SCHED_HYGIENE_DEBUG
1690 if (serialmode & SERIALMODE_OUTPUT) {
1691 ml_spin_debug_reset(current_thread());
1692 }
1693#endif /* SCHED_HYGIENE_DEBUG */
1694 if (ret != KERN_SUCCESS) {
1695 CPUDEBUGGERRET = ret;
1696 DebuggerUnlock();
1697 return;
1698 }
1699
1700 /* Update the global panic/debugger nested entry level */
1701 kernel_debugger_entry_count = CPUDEBUGGERCOUNT;
1702 if (kernel_debugger_entry_count > 0) {
1703 console_suspend();
1704 }
1705
1706 /*
1707 * TODO: Should we do anything special for nested panics here? i.e. if we've trapped more than twice
1708 * should we call into the debugger if it's configured and then reboot if the panic log has been written?
1709 */
1710
1711 if (CPUDEBUGGEROP == DBOP_NONE) {
1712 /* If there was no debugger context setup, we trapped due to a software breakpoint */
1713 debugger_current_op = DBOP_BREAKPOINT;
1714 } else {
1715 /* Not safe to return from a nested panic/debugger call */
1716 if (debugger_current_op == DBOP_PANIC ||
1717 debugger_current_op == DBOP_DEBUGGER) {
1718 debugger_safe_to_return = FALSE;
1719 }
1720
1721 debugger_current_op = CPUDEBUGGEROP;
1722
1723 /* Only overwrite the panic message if there is none already - save the data from the first call */
1724 if (debugger_panic_str == NULL) {
1725 debugger_panic_str = CPUPANICSTR;
1726 debugger_panic_args = CPUPANICARGS;
1727 debugger_panic_data = CPUPANICDATAPTR;
1728 debugger_message = CPUDEBUGGERMSG;
1729 debugger_panic_caller = CPUPANICCALLER;
1730 debugger_panic_initiator = CPUPANICINITIATOR;
1731 }
1732
1733 debugger_panic_options = CPUPANICOPTS;
1734 }
1735
1736 /*
1737 * Clear the op from the processor debugger context so we can handle
1738 * breakpoints in the debugger
1739 */
1740 CPUDEBUGGEROP = DBOP_NONE;
1741
1742 DEBUGGER_TRAP_TIMESTAMP(2);
1743
1744 kdp_callouts(KDP_EVENT_ENTER);
1745 not_in_kdp = 0;
1746
1747 DEBUGGER_TRAP_TIMESTAMP(3);
1748
1749#if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1750 shmem_mark_as_busy();
1751#endif
1752
1753 if (debugger_current_op == DBOP_BREAKPOINT) {
1754 kdp_raise_exception(exception, code, subcode, state);
1755 } else if (debugger_current_op == DBOP_STACKSHOT) {
1756 CPUDEBUGGERRET = do_stackshot(NULL);
1757#if PGO
1758 } else if (debugger_current_op == DBOP_RESET_PGO_COUNTERS) {
1759 CPUDEBUGGERRET = do_pgo_reset_counters();
1760#endif
1761 } else {
1762 /* note: this is the panic path... */
1763 if (!debug_fatal_panic_begin()) {
1764 /*
1765 * This CPU lost the race to be the first to panic. Re-enable
1766 * interrupts and dead loop here awaiting the debugger xcall from
1767 * the CPU that first panicked.
1768 */
1769 ml_set_interrupts_enabled(TRUE);
1770 panic_stop();
1771 }
1772#if defined(__arm64__) && (DEBUG || DEVELOPMENT)
1773 if (!PE_arm_debug_and_trace_initialized()) {
1774 paniclog_append_noflush("kernel panicked before debug and trace infrastructure initialized!\n"
1775 "spinning forever...\n");
1776 panic_spin_forever();
1777 }
1778#endif
1779 debugger_collect_diagnostics(exception, code, subcode, state);
1780 }
1781
1782#if defined(__arm64__) && CONFIG_KDP_INTERACTIVE_DEBUGGING
1783 shmem_unmark_as_busy();
1784#endif
1785
1786 DEBUGGER_TRAP_TIMESTAMP(4);
1787
1788 not_in_kdp = initial_not_in_kdp;
1789 kdp_callouts(KDP_EVENT_EXIT);
1790
1791 DEBUGGER_TRAP_TIMESTAMP(5);
1792
1793 if (debugger_current_op != DBOP_BREAKPOINT) {
1794 debugger_panic_str = NULL;
1795 debugger_panic_args = NULL;
1796 debugger_panic_data = NULL;
1797 debugger_panic_options = 0;
1798 debugger_message = NULL;
1799 }
1800
1801 /* Restore the previous debugger state */
1802 debugger_current_op = db_prev_op;
1803
1804 DEBUGGER_TRAP_TIMESTAMP(6);
1805
1806 DebuggerResumeOtherCores();
1807
1808 DEBUGGER_TRAP_TIMESTAMP(7);
1809
1810 DebuggerUnlock();
1811
1812 DEBUGGER_TRAP_TIMESTAMP(8);
1813
1814 return;
1815}
1816
1817__attribute__((noinline, not_tail_called))
1818void
1819log(__unused int level, char *fmt, ...)
1820{
1821 void *caller = __builtin_return_address(0);
1822 va_list listp;
1823 va_list listp2;
1824
1825
1826#ifdef lint
1827 level++;
1828#endif /* lint */
1829#ifdef MACH_BSD
1830 va_start(listp, fmt);
1831 va_copy(listp2, listp);
1832
1833 disable_preemption();
1834 _doprnt(fmt, &listp, cons_putc_locked, 0);
1835 enable_preemption();
1836
1837 va_end(listp);
1838
1839#pragma clang diagnostic push
1840#pragma clang diagnostic ignored "-Wformat-nonliteral"
1841#pragma clang diagnostic ignored "-Wformat"
1842 os_log_with_args(OS_LOG_DEFAULT, OS_LOG_TYPE_DEFAULT, fmt, listp2, caller);
1843#pragma clang diagnostic pop
1844 va_end(listp2);
1845#endif
1846}
1847
1848/*
1849 * Per <rdar://problem/24974766>, skip appending log messages to
1850 * the new logging infrastructure in contexts where safety is
1851 * uncertain. These contexts include:
1852 * - When we're in the debugger
1853 * - We're in a panic
1854 * - Interrupts are disabled
1855 * - Or Pre-emption is disabled
1856 * In all the above cases, it is potentially unsafe to log messages.
1857 */
1858
1859boolean_t
1860oslog_is_safe(void)
1861{
1862 return kernel_debugger_entry_count == 0 &&
1863 not_in_kdp == 1 &&
1864 get_preemption_level() == 0 &&
1865 ml_get_interrupts_enabled() == TRUE;
1866}
1867
1868boolean_t
1869debug_mode_active(void)
1870{
1871 return (0 != kernel_debugger_entry_count != 0) || (0 == not_in_kdp);
1872}
1873
1874void
1875debug_putc(char c)
1876{
1877 if ((debug_buf_size != 0) &&
1878 ((debug_buf_ptr - debug_buf_base) < (int)debug_buf_size) &&
1879 (!is_debug_ptr_in_ext_paniclog())) {
1880 *debug_buf_ptr = c;
1881 debug_buf_ptr++;
1882 }
1883}
1884
1885#if defined (__x86_64__)
1886struct pasc {
1887 unsigned a: 7;
1888 unsigned b: 7;
1889 unsigned c: 7;
1890 unsigned d: 7;
1891 unsigned e: 7;
1892 unsigned f: 7;
1893 unsigned g: 7;
1894 unsigned h: 7;
1895} __attribute__((packed));
1896
1897typedef struct pasc pasc_t;
1898
1899/*
1900 * In-place packing routines -- inefficient, but they're called at most once.
1901 * Assumes "buflen" is a multiple of 8. Used for compressing paniclogs on x86.
1902 */
1903int
1904packA(char *inbuf, uint32_t length, uint32_t buflen)
1905{
1906 unsigned int i, j = 0;
1907 pasc_t pack;
1908
1909 length = MIN(((length + 7) & ~7), buflen);
1910
1911 for (i = 0; i < length; i += 8) {
1912 pack.a = inbuf[i];
1913 pack.b = inbuf[i + 1];
1914 pack.c = inbuf[i + 2];
1915 pack.d = inbuf[i + 3];
1916 pack.e = inbuf[i + 4];
1917 pack.f = inbuf[i + 5];
1918 pack.g = inbuf[i + 6];
1919 pack.h = inbuf[i + 7];
1920 bcopy((char *) &pack, inbuf + j, 7);
1921 j += 7;
1922 }
1923 return j;
1924}
1925
1926void
1927unpackA(char *inbuf, uint32_t length)
1928{
1929 pasc_t packs;
1930 unsigned i = 0;
1931 length = (length * 8) / 7;
1932
1933 while (i < length) {
1934 packs = *(pasc_t *)&inbuf[i];
1935 bcopy(&inbuf[i + 7], &inbuf[i + 8], MAX(0, (int) (length - i - 8)));
1936 inbuf[i++] = packs.a;
1937 inbuf[i++] = packs.b;
1938 inbuf[i++] = packs.c;
1939 inbuf[i++] = packs.d;
1940 inbuf[i++] = packs.e;
1941 inbuf[i++] = packs.f;
1942 inbuf[i++] = packs.g;
1943 inbuf[i++] = packs.h;
1944 }
1945}
1946#endif /* defined (__x86_64__) */
1947
1948extern char *proc_name_address(void *);
1949extern char *proc_longname_address(void *);
1950
1951__private_extern__ void
1952panic_display_process_name(void)
1953{
1954 proc_name_t proc_name = {};
1955 struct proc *cbsd_info = NULL;
1956 task_t ctask = NULL;
1957 vm_size_t size;
1958
1959 if (!panic_get_thread_proc_task(current_thread(), &ctask, &cbsd_info)) {
1960 goto out;
1961 }
1962
1963 if (cbsd_info == NULL) {
1964 goto out;
1965 }
1966
1967 size = ml_nofault_copy((vm_offset_t)proc_longname_address(cbsd_info),
1968 (vm_offset_t)&proc_name, sizeof(proc_name));
1969
1970 if (size == 0 || proc_name[0] == '\0') {
1971 size = ml_nofault_copy((vm_offset_t)proc_name_address(cbsd_info),
1972 (vm_offset_t)&proc_name,
1973 MIN(sizeof(command_t), sizeof(proc_name)));
1974 if (size > 0) {
1975 proc_name[size - 1] = '\0';
1976 }
1977 }
1978
1979out:
1980 proc_name[sizeof(proc_name) - 1] = '\0';
1981 paniclog_append_noflush("\nProcess name corresponding to current thread (%p): %s\n",
1982 current_thread(), proc_name[0] != '\0' ? proc_name : "Unknown");
1983}
1984
1985unsigned
1986panic_active(void)
1987{
1988 return debugger_current_op == DBOP_PANIC ||
1989 (debugger_current_op == DBOP_DEBUGGER && debugger_is_panic);
1990}
1991
1992void
1993populate_model_name(char *model_string)
1994{
1995 strlcpy(model_name, model_string, sizeof(model_name));
1996}
1997
1998void
1999panic_display_model_name(void)
2000{
2001 char tmp_model_name[sizeof(model_name)];
2002
2003 if (ml_nofault_copy((vm_offset_t) &model_name, (vm_offset_t) &tmp_model_name, sizeof(model_name)) != sizeof(model_name)) {
2004 return;
2005 }
2006
2007 tmp_model_name[sizeof(tmp_model_name) - 1] = '\0';
2008
2009 if (tmp_model_name[0] != 0) {
2010 paniclog_append_noflush("System model name: %s\n", tmp_model_name);
2011 }
2012}
2013
2014void
2015panic_display_kernel_uuid(void)
2016{
2017 char tmp_kernel_uuid[sizeof(kernel_uuid_string)];
2018
2019 if (ml_nofault_copy((vm_offset_t) &kernel_uuid_string, (vm_offset_t) &tmp_kernel_uuid, sizeof(kernel_uuid_string)) != sizeof(kernel_uuid_string)) {
2020 return;
2021 }
2022
2023 if (tmp_kernel_uuid[0] != '\0') {
2024 paniclog_append_noflush("Kernel UUID: %s\n", tmp_kernel_uuid);
2025 }
2026}
2027
2028#if CONFIG_SPTM
2029static void
2030panic_display_component_uuid(char const *component_name, void *component_address)
2031{
2032 uuid_t *component_uuid;
2033 unsigned long component_uuid_len = 0;
2034 uuid_string_t component_uuid_string;
2035
2036 component_uuid = getuuidfromheader((kernel_mach_header_t *)component_address, &component_uuid_len);
2037
2038 if (component_uuid != NULL && component_uuid_len == sizeof(uuid_t)) {
2039 uuid_unparse_upper(*component_uuid, component_uuid_string);
2040 paniclog_append_noflush("%s UUID: %s\n", component_name, component_uuid_string);
2041 }
2042}
2043#endif /* CONFIG_SPTM */
2044
2045void
2046panic_display_kernel_aslr(void)
2047{
2048#if CONFIG_SPTM
2049 {
2050 struct debug_header const *dh = SPTMArgs->debug_header;
2051
2052 paniclog_append_noflush("Debug Header address: %p\n", dh);
2053
2054 if (dh != NULL) {
2055 void *component_address;
2056
2057 paniclog_append_noflush("Debug Header entry count: %d\n", dh->count);
2058
2059 switch (dh->count) {
2060 default: // 3 or more
2061 component_address = dh->image[DEBUG_HEADER_ENTRY_TXM];
2062 paniclog_append_noflush("TXM load address: %p\n", component_address);
2063
2064 panic_display_component_uuid("TXM", component_address);
2065 OS_FALLTHROUGH;
2066 case 2:
2067 component_address = dh->image[DEBUG_HEADER_ENTRY_XNU];
2068 paniclog_append_noflush("Debug Header kernelcache load address: %p\n", component_address);
2069
2070 panic_display_component_uuid("Debug Header kernelcache", component_address);
2071 OS_FALLTHROUGH;
2072 case 1:
2073 component_address = dh->image[DEBUG_HEADER_ENTRY_SPTM];
2074 paniclog_append_noflush("SPTM load address: %p\n", component_address);
2075
2076 panic_display_component_uuid("SPTM", component_address);
2077 OS_FALLTHROUGH;
2078 case 0:
2079 ; // nothing to print
2080 }
2081 }
2082 }
2083#endif /* CONFIG_SPTM */
2084
2085 kc_format_t kc_format;
2086
2087 PE_get_primary_kc_format(&kc_format);
2088
2089 if (kc_format == KCFormatFileset) {
2090 void *kch = PE_get_kc_header(KCKindPrimary);
2091 paniclog_append_noflush("KernelCache slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
2092 paniclog_append_noflush("KernelCache base: %p\n", (void*) kch);
2093 paniclog_append_noflush("Kernel slide: 0x%016lx\n", vm_kernel_stext - (unsigned long)kch + vm_kernel_slide);
2094 paniclog_append_noflush("Kernel text base: %p\n", (void *) vm_kernel_stext);
2095#if defined(__arm64__)
2096 extern vm_offset_t segTEXTEXECB;
2097 paniclog_append_noflush("Kernel text exec slide: 0x%016lx\n", (unsigned long)segTEXTEXECB - (unsigned long)kch + vm_kernel_slide);
2098 paniclog_append_noflush("Kernel text exec base: 0x%016lx\n", (unsigned long)segTEXTEXECB);
2099#endif /* defined(__arm64__) */
2100 } else if (vm_kernel_slide) {
2101 paniclog_append_noflush("Kernel slide: 0x%016lx\n", (unsigned long) vm_kernel_slide);
2102 paniclog_append_noflush("Kernel text base: %p\n", (void *)vm_kernel_stext);
2103 } else {
2104 paniclog_append_noflush("Kernel text base: %p\n", (void *)vm_kernel_stext);
2105 }
2106}
2107
2108void
2109panic_display_hibb(void)
2110{
2111#if defined(__i386__) || defined (__x86_64__)
2112 paniclog_append_noflush("__HIB text base: %p\n", (void *) vm_hib_base);
2113#endif
2114}
2115
2116#if CONFIG_ECC_LOGGING
2117__private_extern__ void
2118panic_display_ecc_errors(void)
2119{
2120 uint32_t count = ecc_log_get_correction_count();
2121
2122 if (count > 0) {
2123 paniclog_append_noflush("ECC Corrections:%u\n", count);
2124 }
2125}
2126#endif /* CONFIG_ECC_LOGGING */
2127
2128#if CONFIG_FREEZE
2129extern bool freezer_incore_cseg_acct;
2130extern int32_t c_segment_pages_compressed_incore;
2131#endif
2132
2133extern uint32_t c_segment_pages_compressed;
2134extern uint32_t c_segment_count;
2135extern uint32_t c_segments_limit;
2136extern uint32_t c_segment_pages_compressed_limit;
2137extern uint32_t c_segment_pages_compressed_nearing_limit;
2138extern uint32_t c_segments_nearing_limit;
2139extern int vm_num_swap_files;
2140
2141void
2142panic_display_compressor_stats(void)
2143{
2144 int isswaplow = vm_swap_low_on_space();
2145#if CONFIG_FREEZE
2146 uint32_t incore_seg_count;
2147 uint32_t incore_compressed_pages;
2148 if (freezer_incore_cseg_acct) {
2149 incore_seg_count = c_segment_count - c_swappedout_count - c_swappedout_sparse_count;
2150 incore_compressed_pages = c_segment_pages_compressed_incore;
2151 } else {
2152 incore_seg_count = c_segment_count;
2153 incore_compressed_pages = c_segment_pages_compressed;
2154 }
2155
2156 paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2157 (incore_compressed_pages * 100) / c_segment_pages_compressed_limit,
2158 (incore_compressed_pages > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2159 (incore_seg_count * 100) / c_segments_limit,
2160 (incore_seg_count > c_segments_nearing_limit) ? "BAD":"OK",
2161 vm_num_swap_files,
2162 isswaplow ? "LOW":"OK");
2163#else /* CONFIG_FREEZE */
2164 paniclog_append_noflush("Compressor Info: %u%% of compressed pages limit (%s) and %u%% of segments limit (%s) with %d swapfiles and %s swap space\n",
2165 (c_segment_pages_compressed * 100) / c_segment_pages_compressed_limit,
2166 (c_segment_pages_compressed > c_segment_pages_compressed_nearing_limit) ? "BAD":"OK",
2167 (c_segment_count * 100) / c_segments_limit,
2168 (c_segment_count > c_segments_nearing_limit) ? "BAD":"OK",
2169 vm_num_swap_files,
2170 isswaplow ? "LOW":"OK");
2171#endif /* CONFIG_FREEZE */
2172}
2173
2174#if !CONFIG_TELEMETRY
2175int
2176telemetry_gather(user_addr_t buffer __unused, uint32_t *length __unused, bool mark __unused)
2177{
2178 return KERN_NOT_SUPPORTED;
2179}
2180#endif
2181
2182#include <machine/machine_cpu.h>
2183
2184TUNABLE(uint32_t, kern_feature_overrides, "validation_disables", 0);
2185
2186__startup_func
2187static void
2188kern_feature_override_init(void)
2189{
2190 /*
2191 * update kern_feature_override based on the serverperfmode=1 boot-arg
2192 * being present, but do not look at the device-tree setting on purpose.
2193 *
2194 * scale_setup() will update serverperfmode=1 based on the DT later.
2195 */
2196
2197 if (serverperfmode) {
2198 kern_feature_overrides |= KF_SERVER_PERF_MODE_OVRD;
2199 }
2200}
2201STARTUP(TUNABLES, STARTUP_RANK_LAST, kern_feature_override_init);
2202
2203#if MACH_ASSERT
2204STATIC_IF_KEY_DEFINE_TRUE(mach_assert);
2205#endif
2206
2207#if SCHED_HYGIENE_DEBUG
2208STATIC_IF_KEY_DEFINE_TRUE(sched_debug_pmc);
2209STATIC_IF_KEY_DEFINE_TRUE(sched_debug_preemption_disable);
2210STATIC_IF_KEY_DEFINE_TRUE(sched_debug_interrupt_disable);
2211#endif /* SCHED_HYGIENE_DEBUG */
2212
2213__static_if_init_func
2214static void
2215kern_feature_override_apply(const char *args)
2216{
2217 uint64_t kf_ovrd;
2218
2219 /*
2220 * Compute the value of kern_feature_override like it will look like
2221 * after kern_feature_override_init().
2222 */
2223 kf_ovrd = static_if_boot_arg_uint64(args, "validation_disables", 0);
2224 if (static_if_boot_arg_uint64(args, "serverperfmode", 0)) {
2225 kf_ovrd |= KF_SERVER_PERF_MODE_OVRD;
2226 }
2227
2228#if DEBUG_RW
2229 lck_rw_assert_init(args, kf_ovrd);
2230#endif /* DEBUG_RW */
2231#if MACH_ASSERT
2232 if (kf_ovrd & KF_MACH_ASSERT_OVRD) {
2233 static_if_key_disable(mach_assert);
2234 }
2235#endif /* MACH_ASSERT */
2236#if SCHED_HYGIENE_DEBUG
2237 if ((int64_t)static_if_boot_arg_uint64(args, "wdt", 0) != -1) {
2238 if (kf_ovrd & KF_SCHED_HYGIENE_DEBUG_PMC_OVRD) {
2239 static_if_key_disable(sched_debug_pmc);
2240 }
2241 if (kf_ovrd & KF_PREEMPTION_DISABLED_DEBUG_OVRD) {
2242 static_if_key_disable(sched_debug_preemption_disable);
2243 if (kf_ovrd & KF_INTERRUPT_MASKED_DEBUG_OVRD) {
2244 static_if_key_disable(sched_debug_interrupt_disable);
2245 }
2246 }
2247 }
2248#endif /* SCHED_HYGIENE_DEBUG */
2249}
2250STATIC_IF_INIT(kern_feature_override_apply);
2251
2252boolean_t
2253kern_feature_override(uint32_t fmask)
2254{
2255 return (kern_feature_overrides & fmask) == fmask;
2256}
2257
2258#if !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING
2259static boolean_t
2260device_corefile_valid_on_ephemeral(void)
2261{
2262#ifdef CONFIG_KDP_COREDUMP_ENCRYPTION
2263 DTEntry node;
2264 const uint32_t *value = NULL;
2265 unsigned int size = 0;
2266 if (kSuccess != SecureDTLookupEntry(NULL, "/product", &node)) {
2267 return TRUE;
2268 }
2269 if (kSuccess != SecureDTGetProperty(node, "ephemeral-data-mode", (void const **) &value, &size)) {
2270 return TRUE;
2271 }
2272
2273 if (size != sizeof(uint32_t)) {
2274 return TRUE;
2275 }
2276
2277 if ((*value) && (kern_dump_should_enforce_encryption() == true)) {
2278 return FALSE;
2279 }
2280#endif /* ifdef CONFIG_KDP_COREDUMP_ENCRYPTION */
2281
2282 return TRUE;
2283}
2284#endif /* !XNU_TARGET_OS_OSX & CONFIG_KDP_INTERACTIVE_DEBUGGING */
2285
2286boolean_t
2287on_device_corefile_enabled(void)
2288{
2289 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2290#if CONFIG_KDP_INTERACTIVE_DEBUGGING
2291 if (debug_boot_arg == 0) {
2292 return FALSE;
2293 }
2294 if (debug_boot_arg & DB_DISABLE_LOCAL_CORE) {
2295 return FALSE;
2296 }
2297#if !XNU_TARGET_OS_OSX
2298 if (device_corefile_valid_on_ephemeral() == FALSE) {
2299 return FALSE;
2300 }
2301 /*
2302 * outside of macOS, if there's a debug boot-arg set and local
2303 * cores aren't explicitly disabled, we always write a corefile.
2304 */
2305 return TRUE;
2306#else /* !XNU_TARGET_OS_OSX */
2307 /*
2308 * on macOS, if corefiles on panic are requested and local cores
2309 * aren't disabled we write a local core.
2310 */
2311 if (debug_boot_arg & (DB_KERN_DUMP_ON_NMI | DB_KERN_DUMP_ON_PANIC)) {
2312 return TRUE;
2313 }
2314#endif /* !XNU_TARGET_OS_OSX */
2315#endif /* CONFIG_KDP_INTERACTIVE_DEBUGGING */
2316 return FALSE;
2317}
2318
2319boolean_t
2320panic_stackshot_to_disk_enabled(void)
2321{
2322 assert(startup_phase >= STARTUP_SUB_TUNABLES);
2323#if defined(__x86_64__)
2324 if (PEGetCoprocessorVersion() < kCoprocessorVersion2) {
2325 /* Only enabled on pre-Gibraltar machines where it hasn't been disabled explicitly */
2326 if ((debug_boot_arg != 0) && (debug_boot_arg & DB_DISABLE_STACKSHOT_TO_DISK)) {
2327 return FALSE;
2328 }
2329
2330 return TRUE;
2331 }
2332#endif
2333 return FALSE;
2334}
2335
2336const char *
2337sysctl_debug_get_preoslog(size_t *size)
2338{
2339 int result = 0;
2340 void *preoslog_pa = NULL;
2341 int preoslog_size = 0;
2342
2343 result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
2344 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2345 kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2346 *size = 0;
2347 return NULL;
2348 }
2349
2350 /*
2351 * Beware:
2352 * On release builds, we would need to call IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size) to free the preoslog buffer.
2353 * On Development & Debug builds, we retain the buffer so it can be extracted from coredumps.
2354 */
2355 *size = preoslog_size;
2356 return (char *)(ml_static_ptovirt((vm_offset_t)(preoslog_pa)));
2357}
2358
2359void
2360sysctl_debug_free_preoslog(void)
2361{
2362#if RELEASE
2363 int result = 0;
2364 void *preoslog_pa = NULL;
2365 int preoslog_size = 0;
2366
2367 result = IODTGetLoaderInfo("preoslog", &preoslog_pa, &preoslog_size);
2368 if (result || preoslog_pa == NULL || preoslog_size == 0) {
2369 kprintf("Couldn't obtain preoslog region: result = %d, preoslog_pa = %p, preoslog_size = %d\n", result, preoslog_pa, preoslog_size);
2370 return;
2371 }
2372
2373 IODTFreeLoaderInfo("preoslog", preoslog_pa, preoslog_size);
2374#else
2375 /* On Development & Debug builds, we retain the buffer so it can be extracted from coredumps. */
2376#endif // RELEASE
2377}
2378
2379#if HAS_UPSI_FAILURE_INJECTION
2380uint64_t xnu_upsi_injection_stage = 0;
2381uint64_t xnu_upsi_injection_action = 0;
2382
2383__attribute__((optnone)) static void
2384SPINNING_FOREVER(void)
2385{
2386 // Decided to disable optimizations on this function instead of using a
2387 // volatile bool for the deadloop.
2388 // This simplifies the process of using the deadloop as an LLDB attach point
2389 bool loop = true;
2390
2391 while (loop) {
2392 }
2393 return;
2394}
2395
2396void
2397check_for_failure_injection(failure_injection_stage_t current_stage)
2398{
2399 // Can't call this function with the default initialization for xnu_upsi_injection_stage
2400 assert(current_stage != 0);
2401
2402 // Check condition to inject a panic/stall/hang
2403 if (current_stage != xnu_upsi_injection_stage) {
2404 return;
2405 }
2406
2407 // Do the requested action
2408 switch (xnu_upsi_injection_action) {
2409 case INJECTION_ACTION_PANIC:
2410 panic("Test panic at stage 0x%llx", current_stage);
2411 case INJECTION_ACTION_WATCHDOG_TIMEOUT:
2412 case INJECTION_ACTION_DEADLOOP:
2413 SPINNING_FOREVER();
2414 break;
2415 default:
2416 break;
2417 }
2418}
2419#endif // HAS_UPSI_FAILURE_INJECTION
2420
2421#define AWL_HV_ENTRY_FLAG (0x1)
2422
2423static inline void
2424awl_set_scratch_reg_hv_bit(void)
2425{
2426#if defined(__arm64__)
2427#define WATCHDOG_DIAG0 "S3_5_c15_c2_6"
2428 uint64_t awl_diag0 = __builtin_arm_rsr64(WATCHDOG_DIAG0);
2429 awl_diag0 |= AWL_HV_ENTRY_FLAG;
2430 __builtin_arm_wsr64(WATCHDOG_DIAG0, awl_diag0);
2431#endif // defined(__arm64__)
2432}
2433
2434void
2435awl_mark_hv_entry(void)
2436{
2437 if (__probable(*PERCPU_GET(hv_entry_detected) || !awl_scratch_reg_supported)) {
2438 return;
2439 }
2440 *PERCPU_GET(hv_entry_detected) = true;
2441
2442 awl_set_scratch_reg_hv_bit();
2443}
2444
2445/*
2446 * Awl WatchdogDiag0 is not restored by hardware when coming out of reset,
2447 * so restore it manually.
2448 */
2449static bool
2450awl_pm_state_change_cbk(void *param __unused, enum cpu_event event, unsigned int cpu_or_cluster __unused)
2451{
2452 if (event == CPU_BOOTED) {
2453 if (*PERCPU_GET(hv_entry_detected)) {
2454 awl_set_scratch_reg_hv_bit();
2455 }
2456 }
2457
2458 return true;
2459}
2460
2461/*
2462 * Identifies and sets a flag if AWL Scratch0/1 exists in the system, subscribes
2463 * for a callback to restore register after hibernation
2464 */
2465__startup_func
2466static void
2467set_awl_scratch_exists_flag_and_subscribe_for_pm(void)
2468{
2469 DTEntry base = NULL;
2470
2471 if (SecureDTLookupEntry(NULL, "/arm-io/wdt", &base) != kSuccess) {
2472 return;
2473 }
2474 const uint8_t *data = NULL;
2475 unsigned int data_size = sizeof(uint8_t);
2476
2477 if (base != NULL && SecureDTGetProperty(base, "awl-scratch-supported", (const void **)&data, &data_size) == kSuccess) {
2478 for (unsigned int i = 0; i < data_size; i++) {
2479 if (data[i] != 0) {
2480 awl_scratch_reg_supported = true;
2481 cpu_event_register_callback(awl_pm_state_change_cbk, NULL);
2482 break;
2483 }
2484 }
2485 }
2486}
2487STARTUP(EARLY_BOOT, STARTUP_RANK_MIDDLE, set_awl_scratch_exists_flag_and_subscribe_for_pm);
2488
2489/**
2490 * Signal that the system is going down for a panic. Returns true if it is safe to
2491 * proceed with the panic flow, false if we should re-enable interrupts and spin
2492 * to allow another CPU to proceed with its panic flow.
2493 *
2494 * This function is idempotent when called from the same CPU; in the normal
2495 * panic case it is invoked twice, since it needs to be invoked in the case
2496 * where we enter the panic flow outside of panic() from DebuggerWithContext().
2497 */
2498static inline boolean_t
2499debug_fatal_panic_begin(void)
2500{
2501#if CONFIG_SPTM
2502 /*
2503 * Since we're going down, initiate panic lockdown.
2504 *
2505 * Whether or not this call to panic lockdown can be subverted is murky.
2506 * This doesn't really matter, however, because any security critical panics
2507 * events will have already initiated lockdown from the exception vector
2508 * before calling panic. Thus, lockdown from panic itself is fine as merely
2509 * a "best effort".
2510 */
2511#if DEVELOPMENT || DEBUG
2512 panic_lockdown_record_debug_data();
2513#endif /* DEVELOPMENT || DEBUG */
2514 sptm_xnu_panic_begin();
2515
2516 pmap_sptm_percpu_data_t *sptm_pcpu = PERCPU_GET(pmap_sptm_percpu);
2517 uint16_t sptm_cpu_id = sptm_pcpu->sptm_cpu_id;
2518 uint64_t sptm_panicking_cpu_id;
2519
2520 if (sptm_get_panicking_cpu_id(&sptm_panicking_cpu_id) == LIBSPTM_SUCCESS &&
2521 sptm_panicking_cpu_id != sptm_cpu_id) {
2522 return false;
2523 }
2524#endif /* CONFIG_SPTM */
2525 return true;
2526}