Implementation sources

osfmk/arm64/start.s

xnu source @ f6217f8 2025-10-16 · 876 lines · view on GitHub
1/* 2 * Copyright (c) 2007-2022 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28#include "assym.s" 29#include <arm64/asm.h> 30#include <arm64/proc_reg.h> 31#include <arm64/machine_machdep.h> 32#include <arm64/proc_reg.h> 33#include <pexpert/arm64/board_config.h> 34#include <mach_assert.h> 35#include <machine/asm.h> 36#include <arm64/tunables/tunables.s> 37#include <arm64/exception_asm.h> 38 39#if __ARM_KERNEL_PROTECT__ 40#include <arm/pmap.h> 41#endif /* __ARM_KERNEL_PROTECT__ */ 42 43 44 45.macro MSR_VBAR_EL1_X0 46#if defined(KERNEL_INTEGRITY_KTRR) 47 mov x1, lr 48 bl EXT(pinst_set_vbar) 49 mov lr, x1 50#else 51 msr VBAR_EL1, x0 52#endif 53.endmacro 54 55.macro MSR_TCR_EL1_X1 56#if defined(KERNEL_INTEGRITY_KTRR) 57 mov x0, x1 58 mov x1, lr 59 bl EXT(pinst_set_tcr) 60 mov lr, x1 61#else 62 msr TCR_EL1, x1 63#endif 64.endmacro 65 66.macro MSR_TTBR1_EL1_X0 67#if defined(KERNEL_INTEGRITY_KTRR) 68 mov x1, lr 69 bl EXT(pinst_set_ttbr1) 70 mov lr, x1 71#else 72 msr TTBR1_EL1, x0 73#endif 74.endmacro 75 76.macro MSR_SCTLR_EL1_X0 77#if defined(KERNEL_INTEGRITY_KTRR) 78 mov x1, lr 79 80 // This may abort, do so on SP1 81 bl EXT(pinst_spsel_1) 82 83 bl EXT(pinst_set_sctlr) 84 msr SPSel, #0 // Back to SP0 85 mov lr, x1 86#else 87 msr SCTLR_EL1, x0 88#endif /* defined(KERNEL_INTEGRITY_KTRR) */ 89.endmacro 90 91/* 92 * Checks the reset handler for global and CPU-specific reset-assist functions, 93 * then jumps to the reset handler with boot args and cpu data. This is copied 94 * to the first physical page during CPU bootstrap (see cpu.c). 95 * 96 * Variables: 97 * x19 - Reset handler data pointer 98 * x20 - Boot args pointer 99 * x21 - CPU data pointer 100 */ 101 .text 102 .align 12 103 .globl EXT(LowResetVectorBase) 104LEXT(LowResetVectorBase) 105 /* 106 * On reset, both RVBAR_EL1 and VBAR_EL1 point here. SPSel.SP is 1, 107 * so on reset the CPU will jump to offset 0x0 and on exceptions 108 * the CPU will jump to offset 0x200, 0x280, 0x300, or 0x380. 109 * In order for both the reset vector and exception vectors to 110 * coexist in the same space, the reset code is moved to the end 111 * of the exception vector area. 112 */ 113 b EXT(reset_vector) 114 115 /* EL1 SP1: These vectors trap errors during early startup on non-boot CPUs. */ 116 .align 9 117 b . 118 .align 7 119 b . 120 .align 7 121 b . 122 .align 7 123 b . 124 125 .align 7 126 .globl EXT(reset_vector) 127LEXT(reset_vector) 128 // Preserve x0 for start_first_cpu, if called 129 // Unlock the core for debugging 130 msr OSLAR_EL1, xzr 131 msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts 132 133#if !(defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR)) 134 // Set low reset vector before attempting any loads 135 adrp x0, EXT(LowExceptionVectorBase)@page 136 add x0, x0, EXT(LowExceptionVectorBase)@pageoff 137 msr VBAR_EL1, x0 138#endif 139 140 141 142 // Process reset handlers 143 adrp x19, EXT(ResetHandlerData)@page // Get address of the reset handler data 144 add x19, x19, EXT(ResetHandlerData)@pageoff 145 mrs x15, MPIDR_EL1 // Load MPIDR to get CPU number 146#if HAS_CLUSTER 147 and x0, x15, #0xFFFF // CPU number in Affinity0, cluster ID in Affinity1 148#else 149 and x0, x15, #0xFF // CPU number is in MPIDR Affinity Level 0 150#endif 151 ldr x1, [x19, CPU_DATA_ENTRIES] // Load start of data entries 152 add x3, x1, MAX_CPUS * 16 // end addr of data entries = start + (16 * MAX_CPUS) 153Lcheck_cpu_data_entry: 154 ldr x21, [x1, CPU_DATA_PADDR] // Load physical CPU data address 155 cbz x21, Lnext_cpu_data_entry 156 ldr w2, [x21, CPU_PHYS_ID] // Load ccc cpu phys id 157 cmp x0, x2 // Compare cpu data phys cpu and MPIDR_EL1 phys cpu 158 b.eq Lfound_cpu_data_entry // Branch if match 159Lnext_cpu_data_entry: 160 add x1, x1, #16 // Increment to the next cpu data entry 161 cmp x1, x3 162 b.eq Lskip_cpu_reset_handler // Not found 163 b Lcheck_cpu_data_entry // loop 164Lfound_cpu_data_entry: 165 166#ifdef APPLEEVEREST 167 /* 168 * On H15, we need to configure PIO-only tunables and to apply 169 * PIO lockdown as early as possible. 170 */ 171 SET_PIO_ONLY_REGISTERS x21, x2, x3, x4, x5, x6 172#endif /* APPLEEVEREST */ 173 174 adrp x20, EXT(const_boot_args)@page 175 add x20, x20, EXT(const_boot_args)@pageoff 176 ldr x0, [x21, CPU_RESET_HANDLER] // Call CPU reset handler 177 cbz x0, Lskip_cpu_reset_handler 178 179 // Validate that our handler is one of the two expected handlers 180 adrp x2, EXT(resume_idle_cpu)@page 181 add x2, x2, EXT(resume_idle_cpu)@pageoff 182 cmp x0, x2 183 beq 1f 184 adrp x2, EXT(start_cpu)@page 185 add x2, x2, EXT(start_cpu)@pageoff 186 cmp x0, x2 187 bne Lskip_cpu_reset_handler 1881: 189 190#if HAS_BP_RET 191 bl EXT(set_bp_ret) 192#endif 193 194#if __ARM_KERNEL_PROTECT__ && defined(KERNEL_INTEGRITY_KTRR) 195 /* 196 * Populate TPIDR_EL1 (in case the CPU takes an exception while 197 * turning on the MMU). 198 */ 199 ldr x13, [x21, CPU_ACTIVE_THREAD] 200 msr TPIDR_EL1, x13 201#endif /* __ARM_KERNEL_PROTECT__ */ 202 203 blr x0 204Lskip_cpu_reset_handler: 205 b . // Hang if the handler is NULL or returns 206 207 .align 3 208 .global EXT(LowResetVectorEnd) 209LEXT(LowResetVectorEnd) 210 .global EXT(SleepToken) 211#if WITH_CLASSIC_S2R 212LEXT(SleepToken) 213 .space (stSize_NUM),0 214#endif 215 216 .section __DATA_CONST,__const 217 .align 3 218 .globl EXT(ResetHandlerData) 219LEXT(ResetHandlerData) 220 .space (rhdSize_NUM),0 // (filled with 0s) 221 .text 222 223 224/* 225 * __start trampoline is located at a position relative to LowResetVectorBase 226 * so that iBoot can compute the reset vector position to set IORVBAR using 227 * only the kernel entry point. Reset vector = (__start & ~0xfff) 228 */ 229 .align 3 230 .globl EXT(_start) 231LEXT(_start) 232 ARM64_PROLOG 233 b EXT(start_first_cpu) 234 235 236/* 237 * Provides an early-boot exception vector so that the processor will spin 238 * and preserve exception information (e.g., ELR_EL1) when early CPU bootstrap 239 * code triggers an exception. This is copied to the second physical page 240 * during CPU bootstrap (see cpu.c). 241 */ 242 .align 12, 0 243 .global EXT(LowExceptionVectorBase) 244LEXT(LowExceptionVectorBase) 245 /* EL1 SP 0 */ 246 b . 247 .align 7 248 b . 249 .align 7 250 b . 251 .align 7 252 b . 253 /* EL1 SP1 */ 254 .align 7 255 b . 256 .align 7 257 b . 258 .align 7 259 b . 260 .align 7 261 b . 262 /* EL0 64 */ 263 .align 7 264 b . 265 .align 7 266 b . 267 .align 7 268 b . 269 .align 7 270 b . 271 /* EL0 32 */ 272 .align 7 273 b . 274 .align 7 275 b . 276 .align 7 277 b . 278 .align 7 279 b . 280 .align 12, 0 281 282#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) 283/* 284 * Provide a global symbol so that we can narrow the V=P mapping to cover 285 * this page during arm_vm_init. 286 */ 287.align ARM_PGSHIFT 288.globl EXT(bootstrap_instructions) 289LEXT(bootstrap_instructions) 290 291#endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) */ 292 .align 2 293 .globl EXT(resume_idle_cpu) 294LEXT(resume_idle_cpu) 295 adrp lr, EXT(arm_init_idle_cpu)@page 296 add lr, lr, EXT(arm_init_idle_cpu)@pageoff 297 b start_cpu 298 299 .align 2 300 .globl EXT(start_cpu) 301LEXT(start_cpu) 302 adrp lr, EXT(arm_init_cpu)@page 303 add lr, lr, EXT(arm_init_cpu)@pageoff 304 b start_cpu 305 306 .align 2 307start_cpu: 308#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) 309 // This is done right away in reset vector for pre-KTRR devices 310 // Set low reset vector now that we are in the KTRR-free zone 311 adrp x0, EXT(LowExceptionVectorBase)@page 312 add x0, x0, EXT(LowExceptionVectorBase)@pageoff 313 MSR_VBAR_EL1_X0 314#endif /* defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) */ 315 316 // x20 set to BootArgs phys address 317 // x21 set to cpu data phys address 318 319 // Get the kernel memory parameters from the boot args 320 ldr x22, [x20, BA_VIRT_BASE] // Get the kernel virt base 321 ldr x23, [x20, BA_PHYS_BASE] // Get the kernel phys base 322 ldr x24, [x20, BA_MEM_SIZE] // Get the physical memory size 323 adrp x25, EXT(bootstrap_pagetables)@page // Get the start of the page tables 324 ldr x26, [x20, BA_BOOT_FLAGS] // Get the kernel boot flags 325 326 327 // Set TPIDR_EL0 with cached CPU info 328 ldr x0, [x21, CPU_TPIDR_EL0] 329 msr TPIDR_EL0, x0 330 331 // Set TPIDRRO_EL0 to 0 332 msr TPIDRRO_EL0, xzr 333 334 335 // Set the exception stack pointer 336 ldr x0, [x21, CPU_EXCEPSTACK_TOP] 337 338 339 // Set SP_EL1 to exception stack 340#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) 341 mov x1, lr 342 bl EXT(pinst_spsel_1) 343 mov lr, x1 344#else 345 msr SPSel, #1 346#endif 347 mov sp, x0 348 349 // Set the interrupt stack pointer 350 ldr x0, [x21, CPU_INTSTACK_TOP] 351 msr SPSel, #0 352 mov sp, x0 353 354 // Convert lr to KVA 355 add lr, lr, x22 356 sub lr, lr, x23 357 358 b common_start 359 360/* 361 * create_l1_table_entry 362 * 363 * Given a virtual address, creates a table entry in an L1 translation table 364 * to point to an L2 translation table. 365 * arg0 - Virtual address 366 * arg1 - L1 table address 367 * arg2 - L2 table address 368 * arg3 - Scratch register 369 * arg4 - Scratch register 370 * arg5 - Scratch register 371 */ 372.macro create_l1_table_entry 373 and $3, $0, #(ARM_PTE_T1_REGION_MASK(TCR_EL1_BOOT)) 374 lsr $3, $3, #(ARM_TT_L1_SHIFT) // Get index in L1 table for L2 table 375 lsl $3, $3, #(TTE_SHIFT) // Convert index into pointer offset 376 add $3, $1, $3 // Get L1 entry pointer 377 mov $4, #(ARM_TTE_BOOT_TABLE) // Get L1 table entry template 378 and $5, $2, #(ARM_TTE_TABLE_MASK) // Get address bits of L2 table 379 orr $5, $4, $5 // Create table entry for L2 table 380 str $5, [$3] // Write entry to L1 table 381.endmacro 382 383/* 384 * create_l2_block_entries 385 * 386 * Given base virtual and physical addresses, creates consecutive block entries 387 * in an L2 translation table. 388 * arg0 - Virtual address 389 * arg1 - Physical address 390 * arg2 - L2 table address 391 * arg3 - Number of entries 392 * arg4 - Scratch register 393 * arg5 - Scratch register 394 * arg6 - Scratch register 395 * arg7 - Scratch register 396 */ 397.macro create_l2_block_entries 398 and $4, $0, #(ARM_TT_L2_INDEX_MASK) 399 lsr $4, $4, #(ARM_TTE_BLOCK_L2_SHIFT) // Get index in L2 table for block entry 400 lsl $4, $4, #(TTE_SHIFT) // Convert index into pointer offset 401 add $4, $2, $4 // Get L2 entry pointer 402 mov $5, #(ARM_TTE_BOOT_BLOCK_LOWER) // Get L2 block entry template 403 orr $5, $5, #(ARM_TTE_BOOT_BLOCK_UPPER) 404 and $6, $1, #(ARM_TTE_BLOCK_L2_MASK) // Get address bits of block mapping 405 orr $6, $5, $6 406 mov $5, $3 407 mov $7, #(ARM_TT_L2_SIZE) 4081: 409 str $6, [$4], #(1 << TTE_SHIFT) // Write entry to L2 table and advance 410 add $6, $6, $7 // Increment the output address 411 subs $5, $5, #1 // Decrement the number of entries 412 b.ne 1b 413.endmacro 414 415/* 416 * arg0 - virtual start address 417 * arg1 - physical start address 418 * arg2 - number of entries to map 419 * arg3 - L1 table address 420 * arg4 - free space pointer 421 * arg5 - scratch (entries mapped per loop) 422 * arg6 - scratch 423 * arg7 - scratch 424 * arg8 - scratch 425 * arg9 - scratch 426 */ 427.macro create_bootstrap_mapping 428 /* calculate entries left in this page */ 429 and $5, $0, #(ARM_TT_L2_INDEX_MASK) 430 lsr $5, $5, #(ARM_TT_L2_SHIFT) 431 mov $6, #(TTE_PGENTRIES) 432 sub $5, $6, $5 433 434 /* allocate an L2 table */ 4353: add $4, $4, PGBYTES 436 437 /* create_l1_table_entry(virt_base, L1 table, L2 table, scratch1, scratch2, scratch3) */ 438 create_l1_table_entry $0, $3, $4, $6, $7, $8 439 440 /* determine how many entries to map this loop - the smaller of entries 441 * remaining in page and total entries left */ 442 cmp $2, $5 443 csel $5, $2, $5, lt 444 445 /* create_l2_block_entries(virt_base, phys_base, L2 table, num_ents, scratch1, scratch2, scratch3) */ 446 create_l2_block_entries $0, $1, $4, $5, $6, $7, $8, $9 447 448 /* subtract entries just mapped and bail out if we're done */ 449 subs $2, $2, $5 450 beq 2f 451 452 /* entries left to map - advance base pointers */ 453 add $0, $0, $5, lsl #(ARM_TT_L2_SHIFT) 454 add $1, $1, $5, lsl #(ARM_TT_L2_SHIFT) 455 456 mov $5, #(TTE_PGENTRIES) /* subsequent loops map (up to) a whole L2 page */ 457 b 3b 4582: 459.endmacro 460 461/* 462 * _start_first_cpu 463 * Cold boot init routine. Called from __start 464 * x0 - Boot args 465 */ 466 .align 2 467 .globl EXT(start_first_cpu) 468LEXT(start_first_cpu) 469 470 // Unlock the core for debugging 471 msr OSLAR_EL1, xzr 472 msr DAIFSet, #(DAIFSC_ALL) // Disable all interrupts 473 474 mov x20, x0 475 mov x21, #0 476 477 // Set low reset vector before attempting any loads 478 adrp x0, EXT(LowExceptionVectorBase)@page 479 add x0, x0, EXT(LowExceptionVectorBase)@pageoff 480 MSR_VBAR_EL1_X0 481 482 483 // Get the kernel memory parameters from the boot args 484 ldr x22, [x20, BA_VIRT_BASE] // Get the kernel virt base 485 ldr x23, [x20, BA_PHYS_BASE] // Get the kernel phys base 486 ldr x24, [x20, BA_MEM_SIZE] // Get the physical memory size 487 adrp x25, EXT(bootstrap_pagetables)@page // Get the start of the page tables 488 ldr x26, [x20, BA_BOOT_FLAGS] // Get the kernel boot flags 489 490 // Clear the registers that will be used to store the userspace thread pointer and CPU number. 491 // We may not actually be booting from ordinal CPU 0, so this register will be updated 492 // in ml_parse_cpu_topology(), which happens later in bootstrap. 493 msr TPIDRRO_EL0, xzr 494 msr TPIDR_EL0, xzr 495 496 // Set up exception stack pointer 497 adrp x0, EXT(excepstack_top)@page // Load top of exception stack 498 add x0, x0, EXT(excepstack_top)@pageoff 499 add x0, x0, x22 // Convert to KVA 500 sub x0, x0, x23 501 502 // Set SP_EL1 to exception stack 503#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) 504 bl EXT(pinst_spsel_1) 505#else 506 msr SPSel, #1 507#endif 508 509 mov sp, x0 510 511 // Set up interrupt stack pointer 512 adrp x0, EXT(intstack_top)@page // Load top of irq stack 513 add x0, x0, EXT(intstack_top)@pageoff 514 add x0, x0, x22 // Convert to KVA 515 sub x0, x0, x23 516 msr SPSel, #0 // Set SP_EL0 to interrupt stack 517 mov sp, x0 518 519 // Load address to the C init routine into link register 520 adrp lr, EXT(arm_init)@page 521 add lr, lr, EXT(arm_init)@pageoff 522 add lr, lr, x22 // Convert to KVA 523 sub lr, lr, x23 524 525 /* 526 * Set up the bootstrap page tables with a single block entry for the V=P 527 * mapping, a single block entry for the trampolined kernel address (KVA), 528 * and all else invalid. This requires four pages: 529 * Page 1 - V=P L1 table 530 * Page 2 - V=P L2 table 531 * Page 3 - KVA L1 table 532 * Page 4 - KVA L2 table 533 */ 534 535 // Invalidate all entries in the bootstrap page tables 536 mov x0, #(ARM_TTE_EMPTY) // Load invalid entry template 537 mov x1, x25 // Start at V=P pagetable root 538 mov x2, #(TTE_PGENTRIES) // Load number of entries per page 539 lsl x2, x2, #2 // Shift by 2 for num entries on 4 pages 540 541Linvalidate_bootstrap: // do { 542 str x0, [x1], #(1 << TTE_SHIFT) // Invalidate and advance 543 subs x2, x2, #1 // entries-- 544 b.ne Linvalidate_bootstrap // } while (entries != 0) 545 546 /* 547 * In order to reclaim memory on targets where TZ0 (or some other entity) 548 * must be located at the base of memory, iBoot may set the virtual and 549 * physical base addresses to immediately follow whatever lies at the 550 * base of physical memory. 551 * 552 * If the base address belongs to TZ0, it may be dangerous for xnu to map 553 * it (as it may be prefetched, despite being technically inaccessible). 554 * In order to avoid this issue while keeping the mapping code simple, we 555 * may continue to use block mappings, but we will only map the kernelcache 556 * mach header to the end of memory. 557 * 558 * Given that iBoot guarantees that the unslid kernelcache base address 559 * will begin on an L2 boundary, this should prevent us from accidentally 560 * mapping TZ0. 561 */ 562 adrp x0, EXT(_mh_execute_header)@page // address of kernel mach header 563 add x0, x0, EXT(_mh_execute_header)@pageoff 564 ldr w1, [x0, #0x18] // load mach_header->flags 565 tbz w1, #0x1f, Lkernelcache_base_found // if MH_DYLIB_IN_CACHE unset, base is kernel mach header 566 ldr w1, [x0, #0x20] // load first segment cmd (offset sizeof(kernel_mach_header_t)) 567 cmp w1, #0x19 // must be LC_SEGMENT_64 568 bne . 569 ldr x1, [x0, #0x38] // load first segment vmaddr 570 sub x1, x0, x1 // compute slide 571 MOV64 x0, VM_KERNEL_LINK_ADDRESS 572 add x0, x0, x1 // base is kernel link address + slide 573 574Lkernelcache_base_found: 575 /* 576 * Adjust physical and virtual base addresses to account for physical 577 * memory preceeding xnu Mach-O header 578 * x22 - Kernel virtual base 579 * x23 - Kernel physical base 580 * x24 - Physical memory size 581 */ 582 sub x18, x0, x23 583 sub x24, x24, x18 584 add x22, x22, x18 585 add x23, x23, x18 586 587 /* 588 * x0 - V=P virtual cursor 589 * x4 - V=P physical cursor 590 * x14 - KVA virtual cursor 591 * x15 - KVA physical cursor 592 */ 593 mov x4, x0 594 mov x14, x22 595 mov x15, x23 596 597 /* 598 * Allocate L1 tables 599 * x1 - V=P L1 page 600 * x3 - KVA L1 page 601 * x2 - free mem pointer from which we allocate a variable number of L2 602 * pages. The maximum number of bootstrap page table pages is limited to 603 * BOOTSTRAP_TABLE_SIZE. For a 2G 4k page device, assuming the worst-case 604 * slide, we need 1xL1 and up to 3xL2 pages (1GB mapped per L1 entry), so 605 * 8 total pages for V=P and KVA. 606 */ 607 mov x1, x25 608 add x3, x1, PGBYTES 609 mov x2, x3 610 611 /* 612 * Setup the V=P bootstrap mapping 613 * x5 - total number of L2 entries to allocate 614 */ 615 lsr x5, x24, #(ARM_TT_L2_SHIFT) 616 /* create_bootstrap_mapping(vbase, pbase, num_ents, L1 table, freeptr) */ 617 create_bootstrap_mapping x0, x4, x5, x1, x2, x6, x10, x11, x12, x13 618 619 /* Setup the KVA bootstrap mapping */ 620 lsr x5, x24, #(ARM_TT_L2_SHIFT) 621 create_bootstrap_mapping x14, x15, x5, x3, x2, x9, x10, x11, x12, x13 622 623 /* Ensure TTEs are visible */ 624 dsb ish 625 626 627 b common_start 628 629/* 630 * Begin common CPU initialization 631 * 632 * Regster state: 633 * x20 - PA of boot args 634 * x21 - zero on cold boot, PA of cpu data on warm reset 635 * x22 - Kernel virtual base 636 * x23 - Kernel physical base 637 * x25 - PA of the V=P pagetable root 638 * lr - KVA of C init routine 639 * sp - SP_EL0 selected 640 * 641 * SP_EL0 - KVA of CPU's interrupt stack 642 * SP_EL1 - KVA of CPU's exception stack 643 * TPIDRRO_EL0 - CPU number 644 */ 645common_start: 646 647#if HAS_NEX_PG 648 mov x19, lr 649 bl EXT(set_nex_pg) 650 mov lr, x19 651#endif 652 653 // Set the translation control register. 654 MOV64 x1, TCR_EL1_BOOT 655 MSR_TCR_EL1_X1 656 657 658 /* Set up translation table base registers. 659 * TTBR0 - V=P table @ top of kernel 660 * TTBR1 - KVA table @ top of kernel + 1 page 661 */ 662#if defined(KERNEL_INTEGRITY_KTRR) || defined(KERNEL_INTEGRITY_CTRR) || defined(KERNEL_INTEGRITY_PV_CTRR) 663 /* Note that for KTRR configurations, the V=P map will be modified by 664 * arm_vm_init.c. 665 */ 666#endif 667 and x0, x25, #(TTBR_BADDR_MASK) 668 mov x19, lr 669 bl EXT(set_mmu_ttb) 670 mov lr, x19 671 add x0, x25, PGBYTES 672 and x0, x0, #(TTBR_BADDR_MASK) 673 MSR_TTBR1_EL1_X0 674 675 // Set up MAIR attr0 for normal memory, attr1 for device memory 676 mov x0, xzr 677 mov x1, #(MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITEBACK)) 678 orr x0, x0, x1 679 mov x1, #(MAIR_WRITETHRU << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITETHRU)) 680 orr x0, x0, x1 681 mov x1, #(MAIR_WRITECOMB << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITECOMB)) 682 orr x0, x0, x1 683 mov x1, #(MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_RESERVED)) 684 orr x0, x0, x1 685 mov x1, #(MAIR_POSTED_COMBINED_REORDERED << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED_COMBINED_REORDERED)) 686 orr x0, x0, x1 687 mov x1, #(MAIR_DISABLE << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE)) 688 orr x0, x0, x1 689#if HAS_FEAT_XS 690 mov x1, #(MAIR_DISABLE_XS << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE_XS)) 691 orr x0, x0, x1 692 mov x1, #(MAIR_POSTED_COMBINED_REORDERED_XS << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED_COMBINED_REORDERED_XS)) 693 orr x0, x0, x1 694#else 695 mov x1, #(MAIR_POSTED << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED)) 696 orr x0, x0, x1 697 mov x1, #(MAIR_POSTED_REORDERED << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_POSTED_REORDERED)) 698 orr x0, x0, x1 699#endif /* HAS_FEAT_XS */ 700 msr MAIR_EL1, x0 701 isb 702 tlbi vmalle1 703 dsb ish 704 705 706#ifndef __ARM_IC_NOALIAS_ICACHE__ 707 /* Invalidate the TLB and icache on systems that do not guarantee that the 708 * caches are invalidated on reset. 709 */ 710 tlbi vmalle1 711 ic iallu 712#endif 713 714 /* If x21 is not 0, then this is either the start_cpu path or 715 * the resume_idle_cpu path. cpu_ttep should already be 716 * populated, so just switch to the kernel_pmap now. 717 */ 718 719 cbz x21, 1f 720 adrp x0, EXT(cpu_ttep)@page 721 add x0, x0, EXT(cpu_ttep)@pageoff 722 ldr x0, [x0] 723 MSR_TTBR1_EL1_X0 7241: 725 726 // Set up the exception vectors 727#if __ARM_KERNEL_PROTECT__ 728 /* If this is not the first reset of the boot CPU, the alternate mapping 729 * for the exception vectors will be set up, so use it. Otherwise, we 730 * should use the mapping located in the kernelcache mapping. 731 */ 732 MOV64 x0, ARM_KERNEL_PROTECT_EXCEPTION_START 733 734 cbnz x21, 1f 735#endif /* __ARM_KERNEL_PROTECT__ */ 736 adrp x0, EXT(ExceptionVectorsBase)@page // Load exception vectors base address 737 add x0, x0, EXT(ExceptionVectorsBase)@pageoff 738 add x0, x0, x22 // Convert exception vector address to KVA 739 sub x0, x0, x23 7401: 741 MSR_VBAR_EL1_X0 742 743#if HAS_APPLE_PAC 744 PAC_INIT_KEY_STATE tmp=x0, tmp2=x1 745#endif /* HAS_APPLE_PAC */ 746 747 // Enable caches, MMU, ROP and JOP 748 MOV64 x0, SCTLR_EL1_DEFAULT 749 MSR_SCTLR_EL1_X0 750 isb sy 751 752#if !VMAPPLE 753 MOV64 x1, SCTLR_EL1_DEFAULT 754 cmp x0, x1 755 bne . 756#endif /* !VMAPPLE */ 757 758#if (!CONFIG_KERNEL_INTEGRITY || (CONFIG_KERNEL_INTEGRITY && !defined(KERNEL_INTEGRITY_WT))) 759 /* Watchtower 760 * 761 * If we have a Watchtower monitor it will setup CPACR_EL1 for us, touching 762 * it here would trap to EL3. 763 */ 764 765 // Enable NEON 766 mov x0, #(CPACR_FPEN_ENABLE) 767 msr CPACR_EL1, x0 768#endif 769 770 // Clear thread pointer 771 msr TPIDR_EL1, xzr // Set thread register 772 773 774#if defined(APPLE_ARM64_ARCH_FAMILY) 775 mrs x12, MDSCR_EL1 776 orr x12, x12, MDSCR_TDCC 777 msr MDSCR_EL1, x12 778 // Initialization common to all non-virtual Apple targets 779#endif // APPLE_ARM64_ARCH_FAMILY 780 781 // Read MIDR before start of per-SoC tunables 782 mrs x12, MIDR_EL1 783 784 APPLY_TUNABLES x12, x13, x14 785 786#if HAS_CLUSTER && !NO_CPU_OVRD 787 // Unmask external IRQs if we're restarting from non-retention WFI 788 mrs x9, CPU_OVRD 789 and x9, x9, #(~(ARM64_REG_CYC_OVRD_irq_mask | ARM64_REG_CYC_OVRD_fiq_mask)) 790 msr CPU_OVRD, x9 791#endif 792 793 // If x21 != 0, we're doing a warm reset, so we need to trampoline to the kernel pmap. 794 cbnz x21, Ltrampoline 795 796 // Set KVA of boot args as first arg 797 add x0, x20, x22 798 sub x0, x0, x23 799 800#if KASAN 801 mov x20, x0 802 mov x21, lr 803 804 // x0: boot args 805 // x1: KVA page table phys base 806 mrs x1, TTBR1_EL1 807 bl EXT(kasan_bootstrap) 808 809 mov x0, x20 810 mov lr, x21 811#endif 812 813 // Return to arm_init() 814 ret 815 816Ltrampoline: 817 // Load VA of the trampoline 818 adrp x0, arm_init_tramp@page 819 add x0, x0, arm_init_tramp@pageoff 820 add x0, x0, x22 821 sub x0, x0, x23 822 823 // Branch to the trampoline 824 br x0 825 826/* 827 * V=P to KVA trampoline. 828 * x0 - KVA of cpu data pointer 829 */ 830 .text 831 .align 2 832arm_init_tramp: 833 ARM64_JUMP_TARGET 834 /* On a warm boot, the full kernel translation table is initialized in 835 * addition to the bootstrap tables. The layout is as follows: 836 * 837 * +--Top of Memory--+ 838 * ... 839 * | | 840 * | Primary Kernel | 841 * | Trans. Table | 842 * | | 843 * +--Top + 5 pages--+ 844 * | | 845 * | Invalid Table | 846 * | | 847 * +--Top + 4 pages--+ 848 * | | 849 * | KVA Table | 850 * | | 851 * +--Top + 2 pages--+ 852 * | | 853 * | V=P Table | 854 * | | 855 * +--Top of Kernel--+ 856 * | | 857 * | Kernel Mach-O | 858 * | | 859 * ... 860 * +---Kernel Base---+ 861 */ 862 863 864 mov x19, lr 865 // Convert CPU data PA to VA and set as first argument 866 mov x0, x21 867 bl EXT(phystokv) 868 869 mov lr, x19 870 871 /* Return to arm_init() */ 872 ret 873 874//#include "globals_asm.h" 875 876/* vim: set ts=4: */