osfmk/arm64/dbgwrap.c
1/*
2 * Copyright (c) 2017 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29#include <arm/cpu_data_internal.h>
30#include <arm/dbgwrap.h>
31#include <arm64/proc_reg.h>
32#include <machine/atomic.h>
33#include <pexpert/arm64/board_config.h>
34
35#define DBGWRAP_REG_OFFSET 0
36#define DBGWRAP_DBGHALT (1ULL << 31)
37#define DBGWRAP_DBGACK (1ULL << 28)
38
39#define EDDTRRX_REG_OFFSET 0x80
40#define EDITR_REG_OFFSET 0x84
41#define EDSCR_REG_OFFSET 0x88
42#define EDSCR_TXFULL (1ULL << 29)
43#define EDSCR_ITE (1ULL << 24)
44#define EDSCR_MA (1ULL << 20)
45#define EDSCR_ERR (1ULL << 6)
46#define EDDTRTX_REG_OFFSET 0x8C
47#define EDRCR_REG_OFFSET 0x90
48#define EDRCR_CSE (1ULL << 2)
49#define EDPRSR_REG_OFFSET 0x314
50#define EDPRSR_OSLK (1ULL << 5)
51
52#define MAX_EDITR_RETRIES 16
53
54/* Older SoCs require 32-bit accesses for DBGWRAP;
55 * newer ones require 64-bit accesses. */
56#ifdef HAS_32BIT_DBGWRAP
57typedef uint32_t dbgwrap_reg_t;
58#else
59typedef uint64_t dbgwrap_reg_t;
60#endif
61
62#if DEVELOPMENT || DEBUG
63#define MAX_STUFFED_INSTRS 64
64uint32_t stuffed_instrs[MAX_STUFFED_INSTRS];
65volatile uint32_t stuffed_instr_count = 0;
66#endif
67
68static volatile uint32_t halt_from_cpu = (uint32_t)-1;
69
70boolean_t
71ml_dbgwrap_cpu_is_halted(int cpu_index)
72{
73 cpu_data_t *cdp = cpu_datap(cpu_index);
74 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) {
75 return FALSE;
76 }
77
78 return (*(volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET) & DBGWRAP_DBGACK) != 0;
79}
80
81dbgwrap_status_t
82ml_dbgwrap_wait_cpu_halted(int cpu_index, uint64_t timeout_ns)
83{
84 cpu_data_t *cdp = cpu_datap(cpu_index);
85 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0)) {
86 return DBGWRAP_ERR_UNSUPPORTED;
87 }
88
89 volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET);
90
91 uint64_t interval;
92 nanoseconds_to_absolutetime(timeout_ns, &interval);
93 uint64_t deadline = mach_absolute_time() + interval;
94 while (!(*dbgWrapReg & DBGWRAP_DBGACK)) {
95 if (mach_absolute_time() > deadline) {
96 return DBGWRAP_ERR_HALT_TIMEOUT;
97 }
98 }
99
100 return DBGWRAP_SUCCESS;
101}
102
103dbgwrap_status_t
104ml_dbgwrap_halt_cpu(int cpu_index, uint64_t timeout_ns)
105{
106 cpu_data_t *cdp = cpu_datap(cpu_index);
107 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_UTT] == 0) || (cdp->coresight_base[CORESIGHT_ED] == 0)) {
108 return DBGWRAP_ERR_UNSUPPORTED;
109 }
110
111 /* Only one cpu is allowed to initiate the halt sequence, to prevent cpus from cross-halting
112 * each other. The first cpu to request a halt may then halt any and all other cpus besides itself. */
113 int curcpu = cpu_number();
114 if (cpu_index == curcpu) {
115 return DBGWRAP_ERR_SELF_HALT;
116 }
117
118 if (!os_atomic_cmpxchg(&halt_from_cpu, (uint32_t)-1, (unsigned int)curcpu, acq_rel) &&
119 (halt_from_cpu != (uint32_t)curcpu)) {
120 return DBGWRAP_ERR_INPROGRESS;
121 }
122
123 /* Accessing EDPRSR or DBGWRAP when cluster is powered off would result in LLC bus error. Hence, check
124 * the cluster power status to make sure these PIO registers are accessible.*/
125 if (!PE_cpu_power_check_kdp(cpu_index)) {
126 return DBGWRAP_WARN_CPU_OFFLINE;
127 }
128
129 /* Ensure memory-mapped coresight registers can be written */
130 *((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR)) = ARM_DBG_LOCK_ACCESS_KEY;
131
132 /* A core that is not fully powered (e.g. idling in wfi) can still be halted; the dbgwrap
133 * register and certain coresight registers such EDPRSR are in the always-on domain and their
134 * values are retained over ACC power down. And the OS lock defaults to being set but we clear
135 * it first thing when CPU is up, so use that to detect the offline state of individual CPU. */
136 if (*((volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDPRSR_REG_OFFSET)) & EDPRSR_OSLK) {
137 return DBGWRAP_WARN_CPU_OFFLINE;
138 }
139
140 volatile dbgwrap_reg_t *dbgWrapReg = (volatile dbgwrap_reg_t *)(cdp->coresight_base[CORESIGHT_UTT] + DBGWRAP_REG_OFFSET);
141
142 if (ml_dbgwrap_cpu_is_halted(cpu_index)) {
143 return DBGWRAP_WARN_ALREADY_HALTED;
144 }
145
146
147 /* Clear all other writable bits besides dbgHalt; none of the power-down or reset bits must be set. */
148 *dbgWrapReg = DBGWRAP_DBGHALT;
149
150
151 if (timeout_ns != 0) {
152 dbgwrap_status_t stat = ml_dbgwrap_wait_cpu_halted(cpu_index, timeout_ns);
153 return stat;
154 } else {
155 return DBGWRAP_SUCCESS;
156 }
157}
158
159static void
160ml_dbgwrap_stuff_instr(cpu_data_t *cdp, uint32_t instr, uint64_t timeout_ns, dbgwrap_status_t *status)
161{
162 if (*status < 0) {
163 return;
164 }
165
166 volatile uint32_t *editr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDITR_REG_OFFSET);
167 volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET);
168 volatile uint32_t *edrcr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDRCR_REG_OFFSET);
169
170 int retries = 0;
171
172 uint64_t interval;
173 nanoseconds_to_absolutetime(timeout_ns, &interval);
174 uint64_t deadline = mach_absolute_time() + interval;
175
176#if DEVELOPMENT || DEBUG
177 uint32_t stuffed_instr_index = os_atomic_inc(&stuffed_instr_count, relaxed);
178 stuffed_instrs[(stuffed_instr_index - 1) % MAX_STUFFED_INSTRS] = instr;
179#endif
180
181 do {
182 *editr = instr;
183 volatile uint32_t edscr_val;
184 while (!((edscr_val = *edscr) & EDSCR_ITE)) {
185 if (mach_absolute_time() > deadline) {
186 *status = DBGWRAP_ERR_INSTR_TIMEOUT;
187 return;
188 }
189 if (edscr_val & EDSCR_ERR) {
190 break;
191 }
192 }
193 if (edscr_val & EDSCR_ERR) {
194 /* If memory access mode was enable by a debugger, clear it.
195 * This will cause ERR to be set on any attempt to use EDITR. */
196 if (edscr_val & EDSCR_MA) {
197 *edscr = edscr_val & ~EDSCR_MA;
198 }
199 *edrcr = EDRCR_CSE;
200 ++retries;
201 } else {
202 break;
203 }
204 } while (retries < MAX_EDITR_RETRIES);
205
206 if (retries >= MAX_EDITR_RETRIES) {
207 *status = DBGWRAP_ERR_INSTR_ERROR;
208 return;
209 }
210}
211
212static uint64_t
213ml_dbgwrap_read_dtr(cpu_data_t *cdp, uint64_t timeout_ns, dbgwrap_status_t *status)
214{
215 if (*status < 0) {
216 return 0;
217 }
218
219 uint64_t interval;
220 nanoseconds_to_absolutetime(timeout_ns, &interval);
221 uint64_t deadline = mach_absolute_time() + interval;
222
223 /* Per armv8 debug spec, writes to DBGDTR_EL0 on target cpu will set EDSCR.TXFull,
224 * with bits 63:32 available in EDDTRRX and bits 31:0 availabe in EDDTRTX. */
225 volatile uint32_t *edscr = (volatile uint32_t *)(cdp->coresight_base[CORESIGHT_ED] + EDSCR_REG_OFFSET);
226
227 while (!(*edscr & EDSCR_TXFULL)) {
228 if (*edscr & EDSCR_ERR) {
229 *status = DBGWRAP_ERR_INSTR_ERROR;
230 return 0;
231 }
232 if (mach_absolute_time() > deadline) {
233 *status = DBGWRAP_ERR_INSTR_TIMEOUT;
234 return 0;
235 }
236 }
237
238 uint32_t dtrrx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRRX_REG_OFFSET));
239 uint32_t dtrtx = *((volatile uint32_t*)(cdp->coresight_base[CORESIGHT_ED] + EDDTRTX_REG_OFFSET));
240
241 return ((uint64_t)dtrrx << 32) | dtrtx;
242}
243
244dbgwrap_status_t
245ml_dbgwrap_halt_cpu_with_state(int cpu_index, uint64_t timeout_ns, dbgwrap_thread_state_t *state)
246{
247 cpu_data_t *cdp = cpu_datap(cpu_index);
248 if ((cdp == NULL) || (cdp->coresight_base[CORESIGHT_ED] == 0)) {
249 return DBGWRAP_ERR_UNSUPPORTED;
250 }
251
252 dbgwrap_status_t status = ml_dbgwrap_halt_cpu(cpu_index, timeout_ns);
253
254 if (status == DBGWRAP_WARN_CPU_OFFLINE) {
255 bzero(state, sizeof(*state));
256 return status;
257 }
258
259 uint32_t instr;
260
261 for (unsigned int i = 0; i < (sizeof(state->x) / sizeof(state->x[0])); ++i) {
262 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | i; // msr DBGDTR0, x<i>
263 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
264 state->x[i] = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
265 }
266
267 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 29; // msr DBGDTR0, fp
268 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
269 state->fp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
270
271 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 30; // msr DBGDTR0, lr
272 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
273 state->lr = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
274
275 /* Stack pointer (x31) can't be used as a register operand for msr; register 31 is treated as xzr
276 * rather than sp when used as the transfer operand there. Instead, load sp into a GPR
277 * we've already saved off and then store that register in the DTR. I've chosen x18
278 * as the temporary GPR since it's reserved by the arm64 ABI and unused by xnu, so overwriting
279 * it poses the least risk of causing trouble for external debuggers. */
280
281 instr = (0x91U << 24) | (31 << 5) | 18; // mov x18, sp
282 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
283 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
284 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
285 state->sp = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
286
287 /* reading PC (e.g. through adr) is undefined in debug state. Instead use DLR_EL0,
288 * which contains PC at time of entry into debug state.*/
289
290 instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | (1 << 5) | 18; // mrs x18, DLR_EL0
291 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
292 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
293 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
294 state->pc = ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
295
296 /* reading CPSR is undefined in debug state. Instead use DSPSR_EL0,
297 * which contains CPSR at time of entry into debug state.*/
298 instr = (0xD53U << 20) | (1 << 19) | (3 << 16) | (4 << 12) | (5 << 8) | 18; // mrs x18, DSPSR_EL0
299 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
300 instr = (0xD51U << 20) | (2 << 19) | (3 << 16) | (4 << 8) | 18; // msr DBGDTR0, x18
301 ml_dbgwrap_stuff_instr(cdp, instr, timeout_ns, &status);
302 state->cpsr = (uint32_t)ml_dbgwrap_read_dtr(cdp, timeout_ns, &status);
303
304 return status;
305}