osfmk/vm/vm_user.c
1/*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28/*
29 * @OSF_COPYRIGHT@
30 */
31/*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989,1988 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56/*
57 */
58/*
59 * File: vm/vm_user.c
60 * Author: Avadis Tevanian, Jr., Michael Wayne Young
61 *
62 * User-exported virtual memory functions.
63 */
64
65/*
66 * There are three implementations of the "XXX_allocate" functionality in
67 * the kernel: mach_vm_allocate (for any task on the platform), vm_allocate
68 * (for a task with the same address space size, especially the current task),
69 * and vm32_vm_allocate (for the specific case of a 32-bit task). vm_allocate
70 * in the kernel should only be used on the kernel_task. vm32_vm_allocate only
71 * makes sense on platforms where a user task can either be 32 or 64, or the kernel
72 * task can be 32 or 64. mach_vm_allocate makes sense everywhere, and is preferred
73 * for new code.
74 *
75 * The entrypoints into the kernel are more complex. All platforms support a
76 * mach_vm_allocate-style API (subsystem 4800) which operates with the largest
77 * size types for the platform. On platforms that only support U32/K32,
78 * subsystem 4800 is all you need. On platforms that support both U32 and U64,
79 * subsystem 3800 is used disambiguate the size of parameters, and they will
80 * always be 32-bit and call into the vm32_vm_allocate APIs. On non-U32/K32 platforms,
81 * the MIG glue should never call into vm_allocate directly, because the calling
82 * task and kernel_task are unlikely to use the same size parameters
83 *
84 * New VM call implementations should be added here and to mach_vm.defs
85 * (subsystem 4800), and use mach_vm_* "wide" types.
86 */
87
88#include <debug.h>
89
90#include <mach/boolean.h>
91#include <mach/kern_return.h>
92#include <mach/mach_types.h> /* to get vm_address_t */
93#include <mach/memory_object.h>
94#include <mach/std_types.h> /* to get pointer_t */
95#include <mach/upl.h>
96#include <mach/vm_attributes.h>
97#include <mach/vm_param.h>
98#include <mach/vm_statistics.h>
99#include <mach/mach_syscalls.h>
100#include <mach/sdt.h>
101#include <mach/memory_entry.h>
102
103#include <mach/host_priv_server.h>
104#include <mach/mach_vm_server.h>
105#include <mach/memory_entry_server.h>
106#include <mach/vm_map_server.h>
107
108#include <kern/host.h>
109#include <kern/kalloc.h>
110#include <kern/task.h>
111#include <kern/misc_protos.h>
112#include <vm/vm_fault.h>
113#include <vm/vm_map_internal.h>
114#include <vm/vm_object_xnu.h>
115#include <vm/vm_kern.h>
116#include <vm/vm_page_internal.h>
117#include <vm/memory_object_internal.h>
118#include <vm/vm_pageout_internal.h>
119#include <vm/vm_protos.h>
120#include <vm/vm_purgeable_internal.h>
121#include <vm/vm_memory_entry_xnu.h>
122#include <vm/vm_kern_internal.h>
123#include <vm/vm_iokit.h>
124#include <vm/vm_sanitize_internal.h>
125#if CONFIG_DEFERRED_RECLAIM
126#include <vm/vm_reclaim_internal.h>
127#endif /* CONFIG_DEFERRED_RECLAIM */
128#include <vm/vm_init_xnu.h>
129
130#include <san/kasan.h>
131
132#include <libkern/OSDebug.h>
133#include <IOKit/IOBSD.h>
134#include <sys/kdebug_triage.h>
135
136#include <sys/code_signing.h> /* for is_address_space_debugged */
137
138/*
139 * mach_vm_allocate allocates "zero fill" memory in the specfied
140 * map.
141 */
142kern_return_t
143mach_vm_allocate_external(
144 vm_map_t map,
145 mach_vm_offset_ut *addr,
146 mach_vm_size_ut size,
147 int flags)
148{
149 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
150
151 /* filter out any kernel-only flags */
152 if (flags & ~VM_FLAGS_USER_ALLOCATE) {
153 ktriage_record(thread_tid(current_thread()),
154 KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM,
155 KDBG_TRIAGE_RESERVED,
156 KDBG_TRIAGE_VM_ALLOCATE_KERNEL_BADFLAGS_ERROR),
157 KERN_INVALID_ARGUMENT /* arg */);
158 return KERN_INVALID_ARGUMENT;
159 }
160
161 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
162
163 return mach_vm_allocate_kernel(map, addr, size, vmk_flags);
164}
165
166/*
167 * vm_allocate
168 * Legacy routine that allocates "zero fill" memory in the specfied
169 * map (which is limited to the same size as the kernel).
170 */
171kern_return_t
172vm_allocate_external(
173 vm_map_t map,
174 vm_offset_ut *addr,
175 vm_size_ut size,
176 int flags)
177{
178 return mach_vm_allocate_external(map, addr, size, flags);
179}
180
181static __attribute__((always_inline, warn_unused_result))
182kern_return_t
183mach_vm_deallocate_sanitize(
184 vm_map_t map,
185 mach_vm_offset_ut start_u,
186 mach_vm_size_ut size_u,
187 mach_vm_offset_t *start,
188 mach_vm_offset_t *end,
189 mach_vm_size_t *size)
190{
191 vm_sanitize_flags_t flags = VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS;
192
193
194 return vm_sanitize_addr_size(start_u, size_u,
195 VM_SANITIZE_CALLER_VM_DEALLOCATE, map, flags,
196 start, end, size);
197}
198
199/*
200 * mach_vm_deallocate -
201 * deallocates the specified range of addresses in the
202 * specified address map.
203 */
204kern_return_t
205mach_vm_deallocate(
206 vm_map_t map,
207 mach_vm_offset_ut start_u,
208 mach_vm_size_ut size_u)
209{
210 mach_vm_offset_t start, end;
211 mach_vm_size_t size;
212 kern_return_t kr;
213
214 if (map == VM_MAP_NULL) {
215 return KERN_INVALID_ARGUMENT;
216 }
217
218 kr = mach_vm_deallocate_sanitize(map,
219 start_u,
220 size_u,
221 &start,
222 &end,
223 &size);
224 if (__improbable(kr != KERN_SUCCESS)) {
225 return vm_sanitize_get_kr(kr);
226 }
227
228 return vm_map_remove_guard(map, start, end,
229 VM_MAP_REMOVE_NO_FLAGS,
230 KMEM_GUARD_NONE).kmr_return;
231}
232
233/*
234 * vm_deallocate -
235 * deallocates the specified range of addresses in the
236 * specified address map (limited to addresses the same
237 * size as the kernel).
238 */
239kern_return_t
240vm_deallocate(
241 vm_map_t map,
242 vm_offset_ut start,
243 vm_size_ut size)
244{
245 return mach_vm_deallocate(map, start, size);
246}
247
248/*
249 * mach_vm_inherit -
250 * Sets the inheritance of the specified range in the
251 * specified map.
252 */
253kern_return_t
254mach_vm_inherit(
255 vm_map_t map,
256 mach_vm_offset_ut start_u,
257 mach_vm_size_ut size_u,
258 vm_inherit_ut new_inheritance_u)
259{
260 if (map == VM_MAP_NULL) {
261 return KERN_INVALID_ARGUMENT;
262 }
263
264 if (VM_SANITIZE_UNSAFE_IS_ZERO(size_u)) {
265 return KERN_SUCCESS;
266 }
267
268 return vm_map_inherit(map,
269 start_u,
270 vm_sanitize_compute_ut_end(start_u, size_u),
271 new_inheritance_u);
272}
273
274/*
275 * vm_inherit -
276 * Sets the inheritance of the specified range in the
277 * specified map (range limited to addresses
278 */
279kern_return_t
280vm_inherit(
281 vm_map_t map,
282 vm_offset_ut start_u,
283 vm_size_ut size_u,
284 vm_inherit_ut new_inheritance_u)
285{
286 return mach_vm_inherit(map, start_u, size_u, new_inheritance_u);
287}
288
289/*
290 * mach_vm_protect -
291 * Sets the protection of the specified range in the
292 * specified map.
293 */
294
295kern_return_t
296mach_vm_protect(
297 vm_map_t map,
298 mach_vm_address_ut start_u,
299 mach_vm_size_ut size_u,
300 boolean_t set_maximum,
301 vm_prot_ut new_protection_u)
302{
303 if (map == VM_MAP_NULL) {
304 return KERN_INVALID_ARGUMENT;
305 }
306
307 if (VM_SANITIZE_UNSAFE_IS_ZERO(size_u)) {
308 return KERN_SUCCESS;
309 }
310
311 return vm_map_protect(map,
312 start_u,
313 vm_sanitize_compute_ut_end(start_u, size_u),
314 set_maximum,
315 new_protection_u);
316}
317
318/*
319 * vm_protect -
320 * Sets the protection of the specified range in the
321 * specified map. Addressability of the range limited
322 * to the same size as the kernel.
323 */
324
325kern_return_t
326vm_protect(
327 vm_map_t map,
328 vm_offset_ut start_u,
329 vm_size_ut size_u,
330 boolean_t set_maximum,
331 vm_prot_ut new_protection_u)
332{
333 return mach_vm_protect(map, start_u, size_u, set_maximum, new_protection_u);
334}
335
336/*
337 * mach_vm_machine_attributes -
338 * Handle machine-specific attributes for a mapping, such
339 * as cachability, migrability, etc.
340 */
341kern_return_t
342mach_vm_machine_attribute(
343 vm_map_t map,
344 mach_vm_address_ut addr_u,
345 mach_vm_size_ut size_u,
346 vm_machine_attribute_t attribute,
347 vm_machine_attribute_val_t *value) /* IN/OUT */
348{
349 if (map == VM_MAP_NULL) {
350 return KERN_INVALID_ARGUMENT;
351 }
352
353 if (VM_SANITIZE_UNSAFE_IS_ZERO(size_u)) {
354 return KERN_SUCCESS;
355 }
356
357 return vm_map_machine_attribute(map,
358 addr_u,
359 vm_sanitize_compute_ut_end(addr_u, size_u),
360 attribute,
361 value);
362}
363
364/*
365 * vm_machine_attribute -
366 * Handle machine-specific attributes for a mapping, such
367 * as cachability, migrability, etc. Limited addressability
368 * (same range limits as for the native kernel map).
369 */
370kern_return_t
371vm_machine_attribute(
372 vm_map_t map,
373 vm_address_ut addr_u,
374 vm_size_ut size_u,
375 vm_machine_attribute_t attribute,
376 vm_machine_attribute_val_t *value) /* IN/OUT */
377{
378 return mach_vm_machine_attribute(map, addr_u, size_u, attribute, value);
379}
380
381/*
382 * mach_vm_read -
383 * Read/copy a range from one address space and return it to the caller.
384 *
385 * It is assumed that the address for the returned memory is selected by
386 * the IPC implementation as part of receiving the reply to this call.
387 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
388 * that gets returned.
389 *
390 * JMM - because of mach_msg_type_number_t, this call is limited to a
391 * single 4GB region at this time.
392 *
393 */
394kern_return_t
395mach_vm_read(
396 vm_map_t map,
397 mach_vm_address_ut addr,
398 mach_vm_size_ut size,
399 pointer_ut *data,
400 mach_msg_type_number_t *data_size)
401{
402 kern_return_t error;
403 vm_map_copy_t ipc_address;
404
405 if (map == VM_MAP_NULL) {
406 return KERN_INVALID_ARGUMENT;
407 }
408
409 /*
410 * mach_msg_type_number_t is a signed int,
411 * make sure we do not overflow it.
412 */
413 if (!VM_SANITIZE_UNSAFE_FITS(size, mach_msg_type_number_t)) {
414 return KERN_INVALID_ARGUMENT;
415 }
416
417 error = vm_map_copyin(map, addr, size, FALSE, &ipc_address);
418
419 if (KERN_SUCCESS == error) {
420 VM_SANITIZE_UT_SET(*data, (pointer_t) ipc_address);
421 /* On success we know size was validated by vm_map_copyin. */
422 *data_size =
423 (mach_msg_type_number_t)VM_SANITIZE_UNSAFE_UNWRAP(size);
424 }
425 return error;
426}
427
428/*
429 * vm_read -
430 * Read/copy a range from one address space and return it to the caller.
431 * Limited addressability (same range limits as for the native kernel map).
432 *
433 * It is assumed that the address for the returned memory is selected by
434 * the IPC implementation as part of receiving the reply to this call.
435 * If IPC isn't used, the caller must deal with the vm_map_copy_t object
436 * that gets returned.
437 */
438kern_return_t
439vm_read(
440 vm_map_t map,
441 vm_address_ut addr,
442 vm_size_ut size,
443 pointer_ut *data,
444 mach_msg_type_number_t *data_size)
445{
446 return mach_vm_read(map, addr, size, data, data_size);
447}
448
449/*
450 * mach_vm_read_list -
451 * Read/copy a list of address ranges from specified map.
452 *
453 * MIG does not know how to deal with a returned array of
454 * vm_map_copy_t structures, so we have to do the copyout
455 * manually here.
456 */
457kern_return_t
458mach_vm_read_list(
459 vm_map_t map,
460 mach_vm_read_entry_t data_list,
461 natural_t count)
462{
463 mach_msg_type_number_t i;
464 kern_return_t error;
465 vm_map_copy_t copy;
466
467 if (map == VM_MAP_NULL ||
468 count > VM_MAP_ENTRY_MAX) {
469 return KERN_INVALID_ARGUMENT;
470 }
471
472 error = KERN_SUCCESS;
473 for (i = 0; i < count; i++) {
474 vm_map_address_t map_addr;
475 vm_map_size_t map_size;
476
477 map_addr = (vm_map_address_t)(data_list[i].address);
478 map_size = (vm_map_size_t)(data_list[i].size);
479
480 if (map_size != 0) {
481 error = vm_map_copyin(map,
482 map_addr,
483 map_size,
484 FALSE, /* src_destroy */
485 ©);
486 if (KERN_SUCCESS == error) {
487 error = vm_map_copyout(
488 current_task()->map,
489 &map_addr,
490 copy);
491 if (KERN_SUCCESS == error) {
492 data_list[i].address = map_addr;
493 continue;
494 }
495 vm_map_copy_discard(copy);
496 }
497 }
498 data_list[i].address = (mach_vm_address_t)0;
499 data_list[i].size = (mach_vm_size_t)0;
500 }
501 return error;
502}
503
504/*
505 * vm_read_list -
506 * Read/copy a list of address ranges from specified map.
507 *
508 * MIG does not know how to deal with a returned array of
509 * vm_map_copy_t structures, so we have to do the copyout
510 * manually here.
511 *
512 * The source and destination ranges are limited to those
513 * that can be described with a vm_address_t (i.e. same
514 * size map as the kernel).
515 *
516 * JMM - If the result of the copyout is an address range
517 * that cannot be described with a vm_address_t (i.e. the
518 * caller had a larger address space but used this call
519 * anyway), it will result in a truncated address being
520 * returned (and a likely confused caller).
521 */
522
523kern_return_t
524vm_read_list(
525 vm_map_t map,
526 vm_read_entry_t data_list,
527 natural_t count)
528{
529 mach_msg_type_number_t i;
530 kern_return_t error;
531 vm_map_copy_t copy;
532
533 if (map == VM_MAP_NULL ||
534 count > VM_MAP_ENTRY_MAX) {
535 return KERN_INVALID_ARGUMENT;
536 }
537
538 error = KERN_SUCCESS;
539 for (i = 0; i < count; i++) {
540 vm_map_address_t map_addr;
541 vm_map_size_t map_size;
542
543 map_addr = (vm_map_address_t)(data_list[i].address);
544 map_size = (vm_map_size_t)(data_list[i].size);
545
546 if (map_size != 0) {
547 error = vm_map_copyin(map,
548 map_addr,
549 map_size,
550 FALSE, /* src_destroy */
551 ©);
552 if (KERN_SUCCESS == error) {
553 error = vm_map_copyout(current_task()->map,
554 &map_addr,
555 copy);
556 if (KERN_SUCCESS == error) {
557 data_list[i].address =
558 CAST_DOWN(vm_offset_t, map_addr);
559 continue;
560 }
561 vm_map_copy_discard(copy);
562 }
563 }
564 data_list[i].address = (mach_vm_address_t)0;
565 data_list[i].size = (mach_vm_size_t)0;
566 }
567 return error;
568}
569
570/*
571 * mach_vm_read_overwrite -
572 * Overwrite a range of the current map with data from the specified
573 * map/address range.
574 *
575 * In making an assumption that the current thread is local, it is
576 * no longer cluster-safe without a fully supportive local proxy
577 * thread/task (but we don't support cluster's anymore so this is moot).
578 */
579
580kern_return_t
581mach_vm_read_overwrite(
582 vm_map_t map,
583 mach_vm_address_ut address,
584 mach_vm_size_ut size,
585 mach_vm_address_ut data,
586 mach_vm_size_ut *data_size)
587{
588 kern_return_t error;
589 vm_map_copy_t copy;
590
591 if (map == VM_MAP_NULL) {
592 return KERN_INVALID_ARGUMENT;
593 }
594
595 error = vm_map_copyin(map, address, size, FALSE, ©);
596
597 if (KERN_SUCCESS == error) {
598 if (copy) {
599 assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
600 }
601
602 error = vm_map_copy_overwrite(current_thread()->map,
603 data,
604 copy,
605 size,
606 FALSE);
607 if (KERN_SUCCESS == error) {
608 *data_size = size;
609 return error;
610 }
611 vm_map_copy_discard(copy);
612 }
613 return error;
614}
615
616/*
617 * vm_read_overwrite -
618 * Overwrite a range of the current map with data from the specified
619 * map/address range.
620 *
621 * This routine adds the additional limitation that the source and
622 * destination ranges must be describable with vm_address_t values
623 * (i.e. the same size address spaces as the kernel, or at least the
624 * the ranges are in that first portion of the respective address
625 * spaces).
626 */
627
628kern_return_t
629vm_read_overwrite(
630 vm_map_t map,
631 vm_address_ut address,
632 vm_size_ut size,
633 vm_address_ut data,
634 vm_size_ut *data_size)
635{
636 return mach_vm_read_overwrite(map, address, size, data, data_size);
637}
638
639/*
640 * mach_vm_update_pointers_with_remote_tags -
641 */
642
643kern_return_t
644mach_vm_update_pointers_with_remote_tags(
645 __unused vm_map_t map,
646 __unused mach_vm_offset_list_t in_pointer_list,
647 __unused mach_msg_type_number_t in_pointer_listCnt,
648 __unused mach_vm_offset_list_t out_pointer_list,
649 __unused mach_msg_type_number_t *out_pointer_listCnt)
650{
651 if (!in_pointer_list
652 || !out_pointer_list
653 || in_pointer_listCnt >= 512
654 /* The length of the output pointer list must match the input pointer list */
655 || !out_pointer_listCnt
656 || *out_pointer_listCnt != in_pointer_listCnt
657 ) {
658 return KERN_INVALID_ARGUMENT;
659 }
660
661 if (!map || !map->pmap) {
662 return KERN_INVALID_ARGUMENT;
663 }
664
665 return KERN_FAILURE;
666}
667
668/*
669 * mach_vm_write -
670 * Overwrite the specified address range with the data provided
671 * (from the current map).
672 */
673kern_return_t
674mach_vm_write(
675 vm_map_t map,
676 mach_vm_address_ut address,
677 pointer_ut data_u,
678 mach_msg_type_number_t size)
679{
680 if (map == VM_MAP_NULL) {
681 return KERN_INVALID_ARGUMENT;
682 }
683
684 /*
685 * data is created by the kernel's MIG server from a userspace buffer,
686 * so it is safe to unwrap.
687 */
688 vm_map_copy_t data = (vm_map_copy_t) VM_SANITIZE_UNSAFE_UNWRAP(data_u);
689
690 return vm_map_copy_overwrite(map,
691 address,
692 data,
693 size,
694 FALSE /* interruptible XXX */);
695}
696
697/*
698 * vm_write -
699 * Overwrite the specified address range with the data provided
700 * (from the current map).
701 *
702 * The addressability of the range of addresses to overwrite is
703 * limited bu the use of a vm_address_t (same size as kernel map).
704 * Either the target map is also small, or the range is in the
705 * low addresses within it.
706 */
707kern_return_t
708vm_write(
709 vm_map_t map,
710 vm_address_ut address,
711 pointer_ut data,
712 mach_msg_type_number_t size)
713{
714 return mach_vm_write(map, address, data, size);
715}
716
717/*
718 * mach_vm_copy -
719 * Overwrite one range of the specified map with the contents of
720 * another range within that same map (i.e. both address ranges
721 * are "over there").
722 */
723kern_return_t
724mach_vm_copy(
725 vm_map_t map,
726 mach_vm_address_ut source_address,
727 mach_vm_size_ut size,
728 mach_vm_address_ut dest_address)
729{
730 vm_map_copy_t copy;
731 kern_return_t kr;
732
733 if (map == VM_MAP_NULL) {
734 return KERN_INVALID_ARGUMENT;
735 }
736
737 kr = vm_map_copyin(map, source_address, size, FALSE, ©);
738
739 if (KERN_SUCCESS == kr) {
740 if (copy) {
741 assert(VM_SANITIZE_UNSAFE_IS_EQUAL(size, copy->size));
742 }
743
744 kr = vm_map_copy_overwrite(map,
745 dest_address,
746 copy,
747 size,
748 FALSE);
749
750 if (KERN_SUCCESS != kr) {
751 vm_map_copy_discard(copy);
752 }
753 }
754 return kr;
755}
756
757kern_return_t
758vm_copy(
759 vm_map_t map,
760 vm_address_ut source_address,
761 vm_size_ut size,
762 vm_address_ut dest_address)
763{
764 return mach_vm_copy(map, source_address, size, dest_address);
765}
766
767/*
768 * mach_vm_map -
769 * Map some range of an object into an address space.
770 *
771 * The object can be one of several types of objects:
772 * NULL - anonymous memory
773 * a named entry - a range within another address space
774 * or a range within a memory object
775 * a whole memory object
776 *
777 */
778kern_return_t
779mach_vm_map_external(
780 vm_map_t target_map,
781 mach_vm_offset_ut *address,
782 mach_vm_size_ut initial_size,
783 mach_vm_offset_ut mask,
784 int flags,
785 ipc_port_t port,
786 memory_object_offset_ut offset,
787 boolean_t copy,
788 vm_prot_ut cur_protection,
789 vm_prot_ut max_protection,
790 vm_inherit_ut inheritance)
791{
792 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
793
794 /* filter out any kernel-only flags */
795 if (flags & ~VM_FLAGS_USER_MAP) {
796 return KERN_INVALID_ARGUMENT;
797 }
798
799 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
800 /* range_id is set by mach_vm_map_kernel */
801 return mach_vm_map_kernel(target_map, address, initial_size, mask,
802 vmk_flags, port, offset, copy,
803 cur_protection, max_protection,
804 inheritance);
805}
806
807/* legacy interface */
808__attribute__((always_inline))
809kern_return_t
810vm_map_64_external(
811 vm_map_t target_map,
812 vm_offset_ut *address,
813 vm_size_ut size,
814 vm_offset_ut mask,
815 int flags,
816 ipc_port_t port,
817 memory_object_offset_ut offset,
818 boolean_t copy,
819 vm_prot_ut cur_protection,
820 vm_prot_ut max_protection,
821 vm_inherit_ut inheritance)
822{
823 return mach_vm_map_external(target_map, address,
824 size, mask, flags, port, offset, copy,
825 cur_protection, max_protection, inheritance);
826}
827
828/* temporary, until world build */
829__attribute__((always_inline))
830kern_return_t
831vm_map_external(
832 vm_map_t target_map,
833 vm_offset_ut *address,
834 vm_size_ut size,
835 vm_offset_ut mask,
836 int flags,
837 ipc_port_t port,
838 vm_offset_ut offset,
839 boolean_t copy,
840 vm_prot_ut cur_protection,
841 vm_prot_ut max_protection,
842 vm_inherit_ut inheritance)
843{
844 return mach_vm_map_external(target_map, address,
845 size, mask, flags, port, offset, copy,
846 cur_protection, max_protection, inheritance);
847}
848
849static __attribute__((always_inline, warn_unused_result))
850kern_return_t
851mach_vm_remap_new_external_sanitize(
852 vm_map_t target_map,
853 vm_prot_ut cur_protection_u,
854 vm_prot_ut max_protection_u,
855 vm_prot_t *cur_protection,
856 vm_prot_t *max_protection)
857{
858 return vm_sanitize_cur_and_max_prots(cur_protection_u, max_protection_u,
859 VM_SANITIZE_CALLER_VM_MAP_REMAP, target_map,
860 cur_protection, max_protection);
861}
862
863/*
864 * mach_vm_remap_new -
865 * Behaves like mach_vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
866 * and {cur,max}_protection are in/out.
867 */
868kern_return_t
869mach_vm_remap_new_external(
870 vm_map_t target_map,
871 mach_vm_offset_ut *address,
872 mach_vm_size_ut size,
873 mach_vm_offset_ut mask,
874 int flags,
875 mach_port_t src_tport,
876 mach_vm_offset_ut memory_address,
877 boolean_t copy,
878 vm_prot_ut *cur_protection_u, /* IN/OUT */
879 vm_prot_ut *max_protection_u, /* IN/OUT */
880 vm_inherit_ut inheritance)
881{
882 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
883 vm_map_t src_map;
884 vm_prot_t cur_protection, max_protection;
885 kern_return_t kr;
886
887 if (target_map == VM_MAP_NULL) {
888 return KERN_INVALID_ARGUMENT;
889 }
890
891 /* filter out any kernel-only flags */
892 if (flags & ~VM_FLAGS_USER_REMAP) {
893 return KERN_INVALID_ARGUMENT;
894 }
895
896 vm_map_kernel_flags_set_vmflags(&vmk_flags,
897 flags | VM_FLAGS_RETURN_DATA_ADDR);
898
899 /*
900 * We don't need cur_protection here, but sanitizing it before
901 * enforcing W^X below matches historical error codes better.
902 */
903 kr = mach_vm_remap_new_external_sanitize(target_map,
904 *cur_protection_u,
905 *max_protection_u,
906 &cur_protection,
907 &max_protection);
908 if (__improbable(kr != KERN_SUCCESS)) {
909 return vm_sanitize_get_kr(kr);
910 }
911
912 if ((max_protection & (VM_PROT_WRITE | VM_PROT_EXECUTE)) ==
913 (VM_PROT_WRITE | VM_PROT_EXECUTE)) {
914 /*
915 * XXX FBDP TODO
916 * enforce target's "wx" policies
917 */
918 return KERN_PROTECTION_FAILURE;
919 }
920
921 if (copy || max_protection == VM_PROT_READ || max_protection == VM_PROT_NONE) {
922 src_map = convert_port_to_map_read(src_tport);
923 } else {
924 src_map = convert_port_to_map(src_tport);
925 }
926
927 /* range_id is set by vm_map_remap */
928 kr = vm_map_remap(target_map,
929 address,
930 size,
931 mask,
932 vmk_flags,
933 src_map,
934 memory_address,
935 copy,
936 cur_protection_u, /* IN/OUT */
937 max_protection_u, /* IN/OUT */
938 inheritance);
939
940 vm_map_deallocate(src_map);
941
942 if (kr == KERN_SUCCESS) {
943 ipc_port_release_send(src_tport); /* consume on success */
944 }
945 return kr;
946}
947
948/*
949 * mach_vm_remap -
950 * Remap a range of memory from one task into another,
951 * to another address range within the same task, or
952 * over top of itself (with altered permissions and/or
953 * as an in-place copy of itself).
954 */
955kern_return_t
956mach_vm_remap_external(
957 vm_map_t target_map,
958 mach_vm_offset_ut *address,
959 mach_vm_size_ut size,
960 mach_vm_offset_ut mask,
961 int flags,
962 vm_map_t src_map,
963 mach_vm_offset_ut memory_address,
964 boolean_t copy,
965 vm_prot_ut *cur_protection, /* OUT */
966 vm_prot_ut *max_protection, /* OUT */
967 vm_inherit_ut inheritance)
968{
969 vm_map_kernel_flags_t vmk_flags = VM_MAP_KERNEL_FLAGS_NONE;
970
971 /* filter out any kernel-only flags */
972 if (flags & ~VM_FLAGS_USER_REMAP) {
973 return KERN_INVALID_ARGUMENT;
974 }
975
976 vm_map_kernel_flags_set_vmflags(&vmk_flags, flags);
977
978 *cur_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
979 *max_protection = vm_sanitize_wrap_prot(VM_PROT_NONE);
980 vmk_flags.vmkf_remap_legacy_mode = true;
981
982 /* range_id is set by vm_map_remap */
983 return vm_map_remap(target_map,
984 address,
985 size,
986 mask,
987 vmk_flags,
988 src_map,
989 memory_address,
990 copy,
991 cur_protection,
992 max_protection,
993 inheritance);
994}
995
996/*
997 * vm_remap_new -
998 * Behaves like vm_remap, except that VM_FLAGS_RETURN_DATA_ADDR is always set
999 * and {cur,max}_protection are in/out.
1000 */
1001kern_return_t
1002vm_remap_new_external(
1003 vm_map_t target_map,
1004 vm_offset_ut *address,
1005 vm_size_ut size,
1006 vm_offset_ut mask,
1007 int flags,
1008 mach_port_t src_tport,
1009 vm_offset_ut memory_address,
1010 boolean_t copy,
1011 vm_prot_ut *cur_protection, /* IN/OUT */
1012 vm_prot_ut *max_protection, /* IN/OUT */
1013 vm_inherit_ut inheritance)
1014{
1015 return mach_vm_remap_new_external(target_map,
1016 address,
1017 size,
1018 mask,
1019 flags,
1020 src_tport,
1021 memory_address,
1022 copy,
1023 cur_protection, /* IN/OUT */
1024 max_protection, /* IN/OUT */
1025 inheritance);
1026}
1027
1028/*
1029 * vm_remap -
1030 * Remap a range of memory from one task into another,
1031 * to another address range within the same task, or
1032 * over top of itself (with altered permissions and/or
1033 * as an in-place copy of itself).
1034 *
1035 * The addressability of the source and target address
1036 * range is limited by the size of vm_address_t (in the
1037 * kernel context).
1038 */
1039kern_return_t
1040vm_remap_external(
1041 vm_map_t target_map,
1042 vm_offset_ut *address,
1043 vm_size_ut size,
1044 vm_offset_ut mask,
1045 int flags,
1046 vm_map_t src_map,
1047 vm_offset_ut memory_address,
1048 boolean_t copy,
1049 vm_prot_ut *cur_protection, /* OUT */
1050 vm_prot_ut *max_protection, /* OUT */
1051 vm_inherit_ut inheritance)
1052{
1053 return mach_vm_remap_external(target_map, address,
1054 size, mask, flags, src_map, memory_address, copy,
1055 cur_protection, max_protection, inheritance);
1056}
1057
1058/*
1059 * NOTE: these routine (and this file) will no longer require mach_host_server.h
1060 * when mach_vm_wire and vm_wire are changed to use ledgers.
1061 */
1062#include <mach/mach_host_server.h>
1063/*
1064 * mach_vm_wire
1065 * Specify that the range of the virtual address space
1066 * of the target task must not cause page faults for
1067 * the indicated accesses.
1068 *
1069 * [ To unwire the pages, specify VM_PROT_NONE. ]
1070 */
1071kern_return_t
1072mach_vm_wire_external(
1073 host_priv_t host_priv,
1074 vm_map_t map,
1075 mach_vm_address_ut start,
1076 mach_vm_size_ut size,
1077 vm_prot_ut access)
1078{
1079 kern_return_t rc;
1080 mach_vm_offset_ut end;
1081
1082 if (host_priv == HOST_PRIV_NULL) {
1083 return KERN_INVALID_HOST;
1084 }
1085
1086 if (map == VM_MAP_NULL) {
1087 return KERN_INVALID_TASK;
1088 }
1089
1090 end = vm_sanitize_compute_ut_end(start, size);
1091 if (VM_SANITIZE_UNSAFE_IS_ZERO(access)) {
1092 rc = vm_map_unwire_impl(map, start, end, true,
1093 VM_SANITIZE_CALLER_VM_UNWIRE_USER);
1094 } else {
1095 rc = vm_map_wire_impl(map, start, end, access,
1096 VM_KERN_MEMORY_MLOCK, true, NULL, VM_SANITIZE_CALLER_VM_WIRE_USER);
1097 }
1098
1099 return rc;
1100}
1101
1102/*
1103 * vm_wire -
1104 * Specify that the range of the virtual address space
1105 * of the target task must not cause page faults for
1106 * the indicated accesses.
1107 *
1108 * [ To unwire the pages, specify VM_PROT_NONE. ]
1109 */
1110kern_return_t
1111vm_wire(
1112 host_priv_t host_priv,
1113 vm_map_t map,
1114 vm_offset_ut start,
1115 vm_size_ut size,
1116 vm_prot_ut access)
1117{
1118 return mach_vm_wire_external(host_priv, map, start, size, access);
1119}
1120
1121/*
1122 * vm_msync
1123 *
1124 * Synchronises the memory range specified with its backing store
1125 * image by either flushing or cleaning the contents to the appropriate
1126 * memory manager.
1127 *
1128 * interpretation of sync_flags
1129 * VM_SYNC_INVALIDATE - discard pages, only return precious
1130 * pages to manager.
1131 *
1132 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1133 * - discard pages, write dirty or precious
1134 * pages back to memory manager.
1135 *
1136 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1137 * - write dirty or precious pages back to
1138 * the memory manager.
1139 *
1140 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1141 * is a hole in the region, and we would
1142 * have returned KERN_SUCCESS, return
1143 * KERN_INVALID_ADDRESS instead.
1144 *
1145 * RETURNS
1146 * KERN_INVALID_TASK Bad task parameter
1147 * KERN_INVALID_ARGUMENT both sync and async were specified.
1148 * KERN_SUCCESS The usual.
1149 * KERN_INVALID_ADDRESS There was a hole in the region.
1150 */
1151
1152kern_return_t
1153mach_vm_msync(
1154 vm_map_t map,
1155 mach_vm_address_ut address_u,
1156 mach_vm_size_ut size_u,
1157 vm_sync_t sync_flags)
1158{
1159 if (map == VM_MAP_NULL) {
1160 return KERN_INVALID_TASK;
1161 }
1162
1163 if (VM_SANITIZE_UNSAFE_IS_ZERO(size_u)) {
1164 return KERN_SUCCESS;
1165 }
1166
1167 return vm_map_msync(map, address_u, size_u, sync_flags);
1168}
1169
1170/*
1171 * vm_msync
1172 *
1173 * Synchronises the memory range specified with its backing store
1174 * image by either flushing or cleaning the contents to the appropriate
1175 * memory manager.
1176 *
1177 * interpretation of sync_flags
1178 * VM_SYNC_INVALIDATE - discard pages, only return precious
1179 * pages to manager.
1180 *
1181 * VM_SYNC_INVALIDATE & (VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS)
1182 * - discard pages, write dirty or precious
1183 * pages back to memory manager.
1184 *
1185 * VM_SYNC_SYNCHRONOUS | VM_SYNC_ASYNCHRONOUS
1186 * - write dirty or precious pages back to
1187 * the memory manager.
1188 *
1189 * VM_SYNC_CONTIGUOUS - does everything normally, but if there
1190 * is a hole in the region, and we would
1191 * have returned KERN_SUCCESS, return
1192 * KERN_INVALID_ADDRESS instead.
1193 *
1194 * The addressability of the range is limited to that which can
1195 * be described by a vm_address_t.
1196 *
1197 * RETURNS
1198 * KERN_INVALID_TASK Bad task parameter
1199 * KERN_INVALID_ARGUMENT both sync and async were specified.
1200 * KERN_SUCCESS The usual.
1201 * KERN_INVALID_ADDRESS There was a hole in the region.
1202 */
1203
1204kern_return_t
1205vm_msync(
1206 vm_map_t map,
1207 vm_address_ut address_u,
1208 vm_size_ut size_u,
1209 vm_sync_t sync_flags)
1210{
1211 return mach_vm_msync(map, address_u, size_u, sync_flags);
1212}
1213
1214
1215int
1216vm_toggle_entry_reuse(int toggle, int *old_value)
1217{
1218 vm_map_t map = current_map();
1219
1220 vmlp_api_start(VM_TOGGLE_ENTRY_REUSE);
1221
1222 assert(!map->is_nested_map);
1223 if (toggle == VM_TOGGLE_GETVALUE && old_value != NULL) {
1224 *old_value = map->disable_vmentry_reuse;
1225 } else if (toggle == VM_TOGGLE_SET) {
1226 vm_map_entry_t map_to_entry;
1227
1228 vm_map_lock(map);
1229 vm_map_disable_hole_optimization(map);
1230 map->disable_vmentry_reuse = TRUE;
1231 __IGNORE_WCASTALIGN(map_to_entry = vm_map_to_entry(map));
1232 if (map->first_free == map_to_entry) {
1233 map->highest_entry_end = vm_map_min(map);
1234 } else {
1235 map->highest_entry_end = map->first_free->vme_end;
1236 }
1237 vm_map_unlock(map);
1238 } else if (toggle == VM_TOGGLE_CLEAR) {
1239 vm_map_lock(map);
1240 map->disable_vmentry_reuse = FALSE;
1241 vm_map_unlock(map);
1242 } else {
1243 vmlp_api_end(VM_TOGGLE_ENTRY_REUSE, KERN_INVALID_ARGUMENT);
1244 return KERN_INVALID_ARGUMENT;
1245 }
1246
1247 vmlp_api_end(VM_TOGGLE_ENTRY_REUSE, KERN_SUCCESS);
1248 return KERN_SUCCESS;
1249}
1250
1251
1252static __attribute__((always_inline, warn_unused_result))
1253kern_return_t
1254mach_vm_behavior_set_sanitize(
1255 vm_map_t map,
1256 mach_vm_offset_ut start_u,
1257 mach_vm_size_ut size_u,
1258 vm_behavior_ut new_behavior_u,
1259 mach_vm_offset_t *start,
1260 mach_vm_offset_t *end,
1261 mach_vm_size_t *size,
1262 vm_behavior_t *new_behavior)
1263{
1264 mach_vm_offset_t align_mask;
1265 kern_return_t kr;
1266
1267 kr = vm_sanitize_behavior(new_behavior_u, VM_SANITIZE_CALLER_VM_BEHAVIOR_SET, new_behavior);
1268 if (__improbable(kr != KERN_SUCCESS)) {
1269 return kr;
1270 }
1271
1272 /* Choose alignment of addr/size based on the behavior being set. */
1273 switch (*new_behavior) {
1274 case VM_BEHAVIOR_REUSABLE:
1275 case VM_BEHAVIOR_REUSE:
1276 case VM_BEHAVIOR_CAN_REUSE:
1277 case VM_BEHAVIOR_ZERO:
1278 /*
1279 * Align to the hardware page size, to allow
1280 * malloc() to maximize the amount of re-usability,
1281 * even on systems with larger software page size.
1282 */
1283 align_mask = PAGE_MASK;
1284 break;
1285 default:
1286 align_mask = VM_MAP_PAGE_MASK(map);
1287 break;
1288 }
1289
1290 vm_sanitize_flags_t flags = VM_SANITIZE_FLAGS_SIZE_ZERO_SUCCEEDS;
1291
1292
1293 kr = vm_sanitize_addr_size(start_u, size_u, VM_SANITIZE_CALLER_VM_BEHAVIOR_SET,
1294 align_mask, map, flags, start, end, size);
1295 if (__improbable(kr != KERN_SUCCESS)) {
1296 return kr;
1297 }
1298
1299 return KERN_SUCCESS;
1300}
1301
1302/*
1303 * mach_vm_behavior_set
1304 *
1305 * Sets the paging behavior attribute for the specified range
1306 * in the specified map.
1307 *
1308 * This routine will fail with KERN_INVALID_ADDRESS if any address
1309 * in [start,start+size) is not a valid allocated memory region.
1310 */
1311kern_return_t
1312mach_vm_behavior_set(
1313 vm_map_t map,
1314 mach_vm_offset_ut start_u,
1315 mach_vm_size_ut size_u,
1316 vm_behavior_ut new_behavior_u)
1317{
1318 kern_return_t kr;
1319 mach_vm_offset_t start, end;
1320 mach_vm_size_t size;
1321 vm_behavior_t new_behavior;
1322
1323 if (map == VM_MAP_NULL) {
1324 return KERN_INVALID_ARGUMENT;
1325 }
1326
1327 kr = mach_vm_behavior_set_sanitize(map,
1328 start_u, size_u, new_behavior_u,
1329 &start, &end, &size, &new_behavior);
1330 if (__improbable(kr != KERN_SUCCESS)) {
1331 return vm_sanitize_get_kr(kr);
1332 }
1333
1334 return vm_map_behavior_set(map,
1335 start,
1336 end,
1337 new_behavior);
1338}
1339
1340/*
1341 * vm_behavior_set
1342 *
1343 * Sets the paging behavior attribute for the specified range
1344 * in the specified map.
1345 *
1346 * This routine will fail with KERN_INVALID_ADDRESS if any address
1347 * in [start,start+size) is not a valid allocated memory region.
1348 *
1349 * This routine is potentially limited in addressibility by the
1350 * use of vm_offset_t (if the map provided is larger than the
1351 * kernel's).
1352 */
1353kern_return_t
1354vm_behavior_set(
1355 vm_map_t map,
1356 vm_offset_ut start,
1357 vm_size_ut size,
1358 vm_behavior_ut new_behavior)
1359{
1360 return mach_vm_behavior_set(map,
1361 start,
1362 size,
1363 new_behavior);
1364}
1365
1366/*
1367 * mach_vm_region:
1368 *
1369 * User call to obtain information about a region in
1370 * a task's address map. Currently, only one flavor is
1371 * supported.
1372 *
1373 * XXX The reserved and behavior fields cannot be filled
1374 * in until the vm merge from the IK is completed, and
1375 * vm_reserve is implemented.
1376 *
1377 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1378 */
1379
1380kern_return_t
1381mach_vm_region(
1382 vm_map_t map,
1383 mach_vm_offset_ut *address_u, /* IN/OUT */
1384 mach_vm_size_ut *size_u, /* OUT */
1385 vm_region_flavor_t flavor, /* IN */
1386 vm_region_info_t info, /* OUT */
1387 mach_msg_type_number_t *count, /* IN/OUT */
1388 mach_port_t *object_name) /* OUT */
1389{
1390 if (VM_MAP_NULL == map) {
1391 return KERN_INVALID_ARGUMENT;
1392 }
1393
1394 /* legacy conversion */
1395 if (VM_REGION_BASIC_INFO == flavor) {
1396 flavor = VM_REGION_BASIC_INFO_64;
1397 }
1398
1399 return vm_map_region(map, address_u, size_u, flavor, info, count,
1400 object_name);
1401}
1402
1403static inline kern_return_t
1404vm_region_get_kern_return(
1405 kern_return_t kr,
1406 vm_offset_ut addr_u,
1407 vm_size_ut size_u)
1408{
1409 vm_offset_ut end_u = vm_sanitize_compute_ut_end(addr_u, size_u);
1410
1411 if (KERN_SUCCESS == kr && VM_SANITIZE_UNSAFE_UNWRAP(end_u) > VM_MAX_ADDRESS) {
1412 return KERN_INVALID_ADDRESS;
1413 }
1414 return kr;
1415}
1416
1417/*
1418 * vm_region_64 and vm_region:
1419 *
1420 * User call to obtain information about a region in
1421 * a task's address map. Currently, only one flavor is
1422 * supported.
1423 *
1424 * XXX The reserved and behavior fields cannot be filled
1425 * in until the vm merge from the IK is completed, and
1426 * vm_reserve is implemented.
1427 *
1428 * XXX Dependency: syscall_vm_region() also supports only one flavor.
1429 */
1430
1431kern_return_t
1432vm_region_64(
1433 vm_map_t map,
1434 vm_offset_ut *address_u, /* IN/OUT */
1435 vm_size_ut *size_u, /* OUT */
1436 vm_region_flavor_t flavor, /* IN */
1437 vm_region_info_t info, /* OUT */
1438 mach_msg_type_number_t *count, /* IN/OUT */
1439 mach_port_t *object_name) /* OUT */
1440{
1441 kern_return_t kr;
1442
1443 kr = mach_vm_region(map, address_u, size_u, flavor, info, count,
1444 object_name);
1445
1446 return vm_region_get_kern_return(kr, *address_u, *size_u);
1447}
1448
1449kern_return_t
1450vm_region(
1451 vm_map_t map,
1452 vm_address_ut *address_u, /* IN/OUT */
1453 vm_size_ut *size_u, /* OUT */
1454 vm_region_flavor_t flavor, /* IN */
1455 vm_region_info_t info, /* OUT */
1456 mach_msg_type_number_t *count, /* IN/OUT */
1457 mach_port_t *object_name) /* OUT */
1458{
1459 kern_return_t kr;
1460
1461 if (VM_MAP_NULL == map) {
1462 return KERN_INVALID_ARGUMENT;
1463 }
1464
1465 kr = vm_map_region(map, address_u, size_u, flavor, info, count,
1466 object_name);
1467
1468 return vm_region_get_kern_return(kr, *address_u, *size_u);
1469}
1470
1471/*
1472 * vm_region_recurse: A form of vm_region which follows the
1473 * submaps in a target map
1474 *
1475 */
1476kern_return_t
1477mach_vm_region_recurse(
1478 vm_map_t map,
1479 mach_vm_address_ut *address_u,
1480 mach_vm_size_ut *size_u,
1481 uint32_t *depth,
1482 vm_region_recurse_info_t info,
1483 mach_msg_type_number_t *infoCnt)
1484{
1485 if (VM_MAP_NULL == map) {
1486 return KERN_INVALID_ARGUMENT;
1487 }
1488
1489 return vm_map_region_recurse_64(map, address_u, size_u, depth,
1490 (vm_region_submap_info_64_t)info, infoCnt);
1491}
1492
1493/*
1494 * vm_region_recurse: A form of vm_region which follows the
1495 * submaps in a target map
1496 *
1497 */
1498kern_return_t
1499vm_region_recurse_64(
1500 vm_map_t map,
1501 vm_address_ut *address_u,
1502 vm_size_ut *size_u,
1503 uint32_t *depth,
1504 vm_region_recurse_info_64_t info,
1505 mach_msg_type_number_t *infoCnt)
1506{
1507 kern_return_t kr;
1508
1509 kr = mach_vm_region_recurse(map, address_u, size_u, depth,
1510 (vm_region_recurse_info_t)info, infoCnt);
1511
1512 return vm_region_get_kern_return(kr, *address_u, *size_u);
1513}
1514
1515kern_return_t
1516vm_region_recurse(
1517 vm_map_t map,
1518 vm_offset_ut *address_u, /* IN/OUT */
1519 vm_size_ut *size_u, /* OUT */
1520 natural_t *depth, /* IN/OUT */
1521 vm_region_recurse_info_t info32, /* IN/OUT */
1522 mach_msg_type_number_t *infoCnt) /* IN/OUT */
1523{
1524 vm_region_submap_info_data_64_t info64;
1525 vm_region_submap_info_t info;
1526 kern_return_t kr;
1527
1528 if (VM_MAP_NULL == map || *infoCnt < VM_REGION_SUBMAP_INFO_COUNT) {
1529 return KERN_INVALID_ARGUMENT;
1530 }
1531
1532 info = (vm_region_submap_info_t)info32;
1533 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT_64;
1534
1535 kr = vm_map_region_recurse_64(map, address_u, size_u,
1536 depth, &info64, infoCnt);
1537
1538 info->protection = info64.protection;
1539 info->max_protection = info64.max_protection;
1540 info->inheritance = info64.inheritance;
1541 info->offset = (uint32_t)info64.offset; /* trouble-maker */
1542 info->user_tag = info64.user_tag;
1543 info->pages_resident = info64.pages_resident;
1544 info->pages_shared_now_private = info64.pages_shared_now_private;
1545 info->pages_swapped_out = info64.pages_swapped_out;
1546 info->pages_dirtied = info64.pages_dirtied;
1547 info->ref_count = info64.ref_count;
1548 info->shadow_depth = info64.shadow_depth;
1549 info->external_pager = info64.external_pager;
1550 info->share_mode = info64.share_mode;
1551 info->is_submap = info64.is_submap;
1552 info->behavior = info64.behavior;
1553 info->object_id = info64.object_id;
1554 info->user_wired_count = info64.user_wired_count;
1555
1556 *infoCnt = VM_REGION_SUBMAP_INFO_COUNT;
1557
1558 return vm_region_get_kern_return(kr, *address_u, *size_u);
1559}
1560
1561kern_return_t
1562mach_vm_purgable_control(
1563 vm_map_t map,
1564 mach_vm_offset_ut address_u,
1565 vm_purgable_t control,
1566 int *state)
1567{
1568 if (VM_MAP_NULL == map) {
1569 return KERN_INVALID_ARGUMENT;
1570 }
1571
1572 switch (control) {
1573 case VM_PURGABLE_SET_STATE:
1574 case VM_PURGABLE_GET_STATE:
1575 case VM_PURGABLE_PURGE_ALL:
1576 break;
1577 case VM_PURGABLE_SET_STATE_FROM_KERNEL:
1578 default:
1579 /* not allowed from user-space */
1580 return KERN_INVALID_ARGUMENT;
1581 }
1582
1583 return vm_map_purgable_control(map, address_u, control, state);
1584}
1585
1586kern_return_t
1587mach_vm_purgable_control_external(
1588 mach_port_t target_tport,
1589 mach_vm_offset_ut address_u,
1590 vm_purgable_t control,
1591 int *state)
1592{
1593 vm_map_t map;
1594 kern_return_t kr;
1595
1596 if (control == VM_PURGABLE_GET_STATE) {
1597 map = convert_port_to_map_read(target_tport);
1598 } else {
1599 map = convert_port_to_map(target_tport);
1600 }
1601
1602 kr = mach_vm_purgable_control(map, address_u, control, state);
1603 vm_map_deallocate(map);
1604
1605 return kr;
1606}
1607
1608kern_return_t
1609vm_purgable_control_external(
1610 mach_port_t target_tport,
1611 vm_offset_ut address,
1612 vm_purgable_t control,
1613 int *state)
1614{
1615 return mach_vm_purgable_control_external(target_tport, address, control, state);
1616}
1617
1618
1619kern_return_t
1620mach_vm_page_query(
1621 vm_map_t map,
1622 mach_vm_offset_ut offset_u,
1623 int *disposition,
1624 int *ref_count)
1625{
1626 kern_return_t kr;
1627 vm_page_info_basic_data_t info;
1628 mach_msg_type_number_t count;
1629
1630 if (VM_MAP_NULL == map) {
1631 return KERN_INVALID_ARGUMENT;
1632 }
1633
1634 count = VM_PAGE_INFO_BASIC_COUNT;
1635 kr = vm_map_page_info(map, offset_u, VM_PAGE_INFO_BASIC,
1636 (vm_page_info_t) &info, &count);
1637 if (kr == KERN_SUCCESS) {
1638 *disposition = info.disposition;
1639 *ref_count = info.ref_count;
1640 } else {
1641 *disposition = 0;
1642 *ref_count = 0;
1643 }
1644
1645 return kr;
1646}
1647
1648kern_return_t
1649vm_map_page_query(
1650 vm_map_t map,
1651 vm_offset_ut offset,
1652 int *disposition,
1653 int *ref_count)
1654{
1655 return mach_vm_page_query(map, offset, disposition, ref_count);
1656}
1657
1658static __attribute__((always_inline, warn_unused_result))
1659kern_return_t
1660mach_vm_page_range_query_sanitize(
1661 mach_vm_offset_ut address_u,
1662 mach_vm_size_ut size_u,
1663 int effective_page_mask,
1664 mach_vm_address_ut dispositions_addr_u,
1665 mach_vm_size_ut dispositions_count_u,
1666 mach_vm_offset_t *start,
1667 mach_vm_size_t *size,
1668 mach_vm_address_t *dispositions_addr,
1669 mach_vm_size_t *disp_buf_req_size)
1670{
1671 mach_vm_offset_t end;
1672 mach_vm_size_t dispositions_count;
1673 mach_vm_address_t discard;
1674
1675 /*
1676 * There are no alignment requirements on
1677 * dispositions_addr_u/dispositions_count_u, those are derived into
1678 * inputs into copyout. So it is safe to unwrap them. We do want to
1679 * check that the range starting at dispositions_addr_u and ending
1680 * after dispositions_count_u integers is sound (i.e., doesn't wrap
1681 * around due to integer overflow).
1682 */
1683 *dispositions_addr = VM_SANITIZE_UNSAFE_UNWRAP(dispositions_addr_u);
1684 dispositions_count = VM_SANITIZE_UNSAFE_UNWRAP(dispositions_count_u);
1685 if (
1686 os_mul_overflow(
1687 dispositions_count,
1688 sizeof(int),
1689 disp_buf_req_size) ||
1690 os_add_overflow(
1691 *dispositions_addr,
1692 *disp_buf_req_size,
1693 &discard)) {
1694 return KERN_INVALID_ARGUMENT;
1695 }
1696
1697 return vm_sanitize_addr_size(address_u, size_u,
1698 VM_SANITIZE_CALLER_VM_MAP_PAGE_RANGE_QUERY,
1699 effective_page_mask,
1700 VM_SANITIZE_FLAGS_SIZE_ZERO_FALLTHROUGH, start,
1701 &end, size);
1702}
1703
1704kern_return_t
1705mach_vm_page_range_query(
1706 vm_map_t map,
1707 mach_vm_offset_ut address_u,
1708 mach_vm_size_ut size_u,
1709 mach_vm_address_ut dispositions_addr_u,
1710 mach_vm_size_ut *dispositions_count_u)
1711{
1712 kern_return_t kr;
1713 int num_pages = 0, i = 0;
1714 mach_vm_size_t curr_sz = 0, copy_sz = 0;
1715 mach_vm_size_t disp_buf_req_size = 0, disp_buf_total_size = 0;
1716 mach_msg_type_number_t count = 0;
1717 mach_vm_address_t dispositions_addr;
1718
1719 void *info = NULL;
1720 void *local_disp = NULL;
1721 vm_map_size_t info_size = 0, local_disp_size = 0;
1722 mach_vm_offset_t start = 0;
1723 vm_map_size_t size;
1724 int effective_page_shift, effective_page_size, effective_page_mask;
1725
1726 if (map == VM_MAP_NULL || dispositions_count_u == NULL) {
1727 return KERN_INVALID_ARGUMENT;
1728 }
1729
1730 effective_page_shift = vm_self_region_page_shift_safely(map);
1731 if (effective_page_shift == -1) {
1732 return KERN_INVALID_ARGUMENT;
1733 }
1734 effective_page_size = (1 << effective_page_shift);
1735 effective_page_mask = effective_page_size - 1;
1736
1737 kr = mach_vm_page_range_query_sanitize(address_u,
1738 size_u,
1739 effective_page_mask,
1740 dispositions_addr_u,
1741 *dispositions_count_u,
1742 &start,
1743 &size,
1744 &dispositions_addr,
1745 &disp_buf_req_size);
1746 if (__improbable(kr != KERN_SUCCESS)) {
1747 return vm_sanitize_get_kr(kr);
1748 }
1749
1750 if (disp_buf_req_size == 0 || size == 0) {
1751 return KERN_SUCCESS;
1752 }
1753
1754 /*
1755 * For large requests, we will go through them
1756 * MAX_PAGE_RANGE_QUERY chunk at a time.
1757 */
1758
1759 curr_sz = MIN(size, MAX_PAGE_RANGE_QUERY);
1760 num_pages = (int) (curr_sz >> effective_page_shift);
1761
1762 info_size = num_pages * sizeof(vm_page_info_basic_data_t);
1763 info = kalloc_data(info_size, Z_WAITOK);
1764
1765 local_disp_size = num_pages * sizeof(int);
1766 local_disp = kalloc_data(local_disp_size, Z_WAITOK);
1767
1768 if (info == NULL || local_disp == NULL) {
1769 kr = KERN_RESOURCE_SHORTAGE;
1770 goto out;
1771 }
1772
1773 while (size) {
1774 count = VM_PAGE_INFO_BASIC_COUNT;
1775 kr = vm_map_page_range_info_internal(
1776 map,
1777 start,
1778 vm_map_round_page(start + curr_sz, effective_page_mask),
1779 effective_page_shift,
1780 VM_PAGE_INFO_BASIC,
1781 (vm_page_info_t) info,
1782 &count);
1783
1784 assert(kr == KERN_SUCCESS);
1785
1786 for (i = 0; i < num_pages; i++) {
1787 ((int*)local_disp)[i] = ((vm_page_info_basic_t)info)[i].disposition;
1788 }
1789
1790 copy_sz = MIN(disp_buf_req_size, num_pages * sizeof(int) /* an int per page */);
1791 kr = copyout(local_disp, (mach_vm_address_t)dispositions_addr, copy_sz);
1792
1793 start += curr_sz;
1794 disp_buf_req_size -= copy_sz;
1795 disp_buf_total_size += copy_sz;
1796
1797 if (kr != 0) {
1798 break;
1799 }
1800
1801 if ((disp_buf_req_size == 0) || (curr_sz >= size)) {
1802 /*
1803 * We might have inspected the full range OR
1804 * more than it esp. if the user passed in
1805 * non-page aligned start/size and/or if we
1806 * descended into a submap. We are done here.
1807 */
1808
1809 size = 0;
1810 } else {
1811 dispositions_addr += copy_sz;
1812
1813 size -= curr_sz;
1814
1815 curr_sz = MIN(vm_map_round_page(size, effective_page_mask), MAX_PAGE_RANGE_QUERY);
1816 num_pages = (int)(curr_sz >> effective_page_shift);
1817 }
1818 }
1819
1820 VM_SANITIZE_UT_SET(
1821 *dispositions_count_u,
1822 disp_buf_total_size / sizeof(int));
1823
1824out:
1825 kfree_data(local_disp, local_disp_size);
1826 kfree_data(info, info_size);
1827 return kr;
1828}
1829
1830kern_return_t
1831mach_vm_page_info(
1832 vm_map_t map,
1833 mach_vm_address_ut address,
1834 vm_page_info_flavor_t flavor,
1835 vm_page_info_t info,
1836 mach_msg_type_number_t *count)
1837{
1838 kern_return_t kr;
1839
1840 if (map == VM_MAP_NULL) {
1841 return KERN_INVALID_ARGUMENT;
1842 }
1843
1844 kr = vm_map_page_info(map, address, flavor, info, count);
1845 return kr;
1846}
1847
1848/*
1849 * task_wire
1850 *
1851 * Set or clear the map's wiring_required flag. This flag, if set,
1852 * will cause all future virtual memory allocation to allocate
1853 * user wired memory. Unwiring pages wired down as a result of
1854 * this routine is done with the vm_wire interface.
1855 */
1856kern_return_t
1857task_wire(
1858 vm_map_t map,
1859 boolean_t must_wire __unused)
1860{
1861 if (map == VM_MAP_NULL) {
1862 return KERN_INVALID_ARGUMENT;
1863 }
1864
1865 return KERN_NOT_SUPPORTED;
1866}
1867
1868kern_return_t
1869vm_map_exec_lockdown(
1870 vm_map_t map)
1871{
1872 vmlp_api_start(VM_MAP_EXEC_LOCKDOWN);
1873
1874 if (map == VM_MAP_NULL) {
1875 vmlp_api_end(VM_MAP_EXEC_LOCKDOWN, KERN_INVALID_ARGUMENT);
1876 return KERN_INVALID_ARGUMENT;
1877 }
1878
1879 vm_map_lock(map);
1880 map->map_disallow_new_exec = TRUE;
1881 vm_map_unlock(map);
1882
1883 vmlp_api_end(VM_MAP_EXEC_LOCKDOWN, KERN_SUCCESS);
1884 return KERN_SUCCESS;
1885}
1886
1887#if XNU_PLATFORM_MacOSX
1888kern_return_t
1889vm_region_object_create(
1890 __unused vm_map_t target_map,
1891 __unused vm_size_t size,
1892 __unused ipc_port_t *object_handle)
1893{
1894 return KERN_NOT_SUPPORTED;
1895}
1896#endif /* XNU_PLATFORM_MacOSX */
1897
1898extern boolean_t proc_is_simulated(struct proc *p);
1899
1900kern_return_t
1901mach_vm_deferred_reclamation_buffer_allocate(
1902 task_t task,
1903 mach_vm_address_ut *address,
1904 uint64_t *sampling_period,
1905 uint32_t initial_capacity,
1906 uint32_t max_capacity)
1907{
1908#if CONFIG_DEFERRED_RECLAIM
1909 if (task != current_task()) {
1910 /* Remote buffer operations are not supported*/
1911 return KERN_INVALID_TASK;
1912 }
1913 struct proc *p = task_get_proc_raw(task);
1914 if (proc_is_simulated(p)) {
1915 return KERN_NOT_SUPPORTED;
1916 }
1917 return vm_deferred_reclamation_buffer_allocate_internal(task, address, sampling_period, initial_capacity, max_capacity);
1918#else
1919 (void) task;
1920 (void) address;
1921 (void) size;
1922 return KERN_NOT_SUPPORTED;
1923#endif /* CONFIG_DEFERRED_RECLAIM */
1924}
1925
1926kern_return_t
1927mach_vm_deferred_reclamation_buffer_flush(
1928 task_t task,
1929 uint32_t num_entries_to_reclaim,
1930 mach_vm_size_ut *bytes_reclaimed_out)
1931{
1932#if CONFIG_DEFERRED_RECLAIM
1933 kern_return_t kr;
1934 mach_vm_size_t bytes_reclaimed = 0;
1935 if (task != current_task()) {
1936 /* Remote buffer operations are not supported */
1937 return KERN_INVALID_TASK;
1938 }
1939 if (bytes_reclaimed_out == NULL) {
1940 return KERN_INVALID_ARGUMENT;
1941 }
1942 kr = vm_deferred_reclamation_buffer_flush_internal(task, num_entries_to_reclaim, &bytes_reclaimed);
1943 *bytes_reclaimed_out = vm_sanitize_wrap_size(bytes_reclaimed);
1944 return kr;
1945#else
1946 (void) task;
1947 (void) num_entries_to_reclaim;
1948 return KERN_NOT_SUPPORTED;
1949#endif /* CONFIG_DEFERRED_RECLAIM */
1950}
1951
1952kern_return_t
1953mach_vm_deferred_reclamation_buffer_resize(task_t task,
1954 uint32_t new_len,
1955 mach_vm_size_ut *bytes_reclaimed_out)
1956{
1957#if CONFIG_DEFERRED_RECLAIM
1958 mach_error_t err;
1959 mach_vm_size_t bytes_reclaimed = 0;
1960
1961 if (task != current_task()) {
1962 /* Remote buffer operations are not supported */
1963 return KERN_INVALID_TASK;
1964 }
1965 if (bytes_reclaimed_out == NULL) {
1966 return KERN_INVALID_ARGUMENT;
1967 }
1968
1969 err = vm_deferred_reclamation_buffer_resize_internal(task, new_len, &bytes_reclaimed);
1970 *bytes_reclaimed_out = vm_sanitize_wrap_size(bytes_reclaimed);
1971 return err;
1972#else
1973 (void) task;
1974 (void) size;
1975 return KERN_NOT_SUPPORTED;
1976#endif /* CONFIG_DEFERRED_RECLAIM */
1977}
1978
1979kern_return_t
1980mach_vm_deferred_reclamation_buffer_query(task_t task,
1981 mach_vm_address_ut *addr_out_ut,
1982 mach_vm_size_ut *size_out_ut)
1983{
1984#if CONFIG_DEFERRED_RECLAIM
1985 return vm_deferred_reclamation_buffer_query_internal(task, addr_out_ut, size_out_ut);
1986#else /* CONFIG_DEFERRED_RECLAIM */
1987 (void) task;
1988 (void) addr_out_ut;
1989 (void) size_out_ut;
1990 return KERN_NOT_SUPPORTED;
1991#endif /* !CONFIG_DEFERRED_RECLAIM */
1992}
1993
1994#if CONFIG_MAP_RANGES
1995
1996extern void qsort(void *a, size_t n, size_t es, int (*cmp)(const void *, const void *));
1997
1998static int
1999vm_map_user_range_cmp(const void *e1, const void *e2)
2000{
2001 const struct vm_map_user_range *r1 = e1;
2002 const struct vm_map_user_range *r2 = e2;
2003
2004 if (r1->vmur_min_address != r2->vmur_min_address) {
2005 return r1->vmur_min_address < r2->vmur_min_address ? -1 : 1;
2006 }
2007
2008 return 0;
2009}
2010
2011static int
2012mach_vm_range_recipe_v1_cmp(const void *e1, const void *e2)
2013{
2014 const mach_vm_range_recipe_v1_t *r1 = e1;
2015 const mach_vm_range_recipe_v1_t *r2 = e2;
2016
2017 if (r1->range.min_address != r2->range.min_address) {
2018 return r1->range.min_address < r2->range.min_address ? -1 : 1;
2019 }
2020
2021 return 0;
2022}
2023
2024static inline __result_use_check kern_return_t
2025mach_vm_range_create_v1_sanitize(
2026 vm_map_t map,
2027 mach_vm_range_recipe_v1_ut *recipe_u,
2028 uint32_t count,
2029 mach_vm_range_recipe_v1_t **recipe_p)
2030{
2031 kern_return_t kr;
2032
2033 for (size_t i = 0; i < count; i++) {
2034 vm_map_offset_t start, end;
2035 vm_map_size_t size;
2036 mach_vm_range_ut * range_u = &recipe_u[i].range_u;
2037 kr = vm_sanitize_addr_end(
2038 range_u->min_address_u,
2039 range_u->max_address_u,
2040 VM_SANITIZE_CALLER_MACH_VM_RANGE_CREATE,
2041 map,
2042 VM_SANITIZE_FLAGS_SIZE_ZERO_FAILS
2043 | VM_SANITIZE_FLAGS_CHECK_ALIGNED_START
2044 | VM_SANITIZE_FLAGS_CHECK_ALIGNED_SIZE,
2045 &start, &end, &size); // Ignore return values
2046 if (__improbable(kr != KERN_SUCCESS)) {
2047 return kr;
2048 }
2049 }
2050 /*
2051 * Sanitization only checked properties of recipe_u.
2052 * We can now see it through the lens of the safe type.
2053 * The cast is undefined behavior, but of the kind VM sanitization
2054 * relies on anyway, so we don't expect this to cause issues.
2055 */
2056 *recipe_p = (mach_vm_range_recipe_v1_t *)recipe_u;
2057
2058 return KERN_SUCCESS;
2059}
2060
2061/*!
2062 * @function mach_vm_range_create_v1()
2063 *
2064 * @brief
2065 * Handle the backend for mach_vm_range_create() for the
2066 * MACH_VM_RANGE_FLAVOR_V1 flavor.
2067 *
2068 * @description
2069 * This call allows to create "ranges" in the map of a task
2070 * that have special semantics/policies around placement of
2071 * new allocations (in the vm_map_locate_space() sense).
2072 *
2073 * @returns
2074 * - KERN_SUCCESS on success
2075 * - KERN_INVALID_ARGUMENT for incorrect arguments
2076 * - KERN_NO_SPACE if the maximum amount of ranges would be exceeded
2077 * - KERN_MEMORY_PRESENT if any of the requested ranges
2078 * overlaps with existing ranges or allocations in the map.
2079 */
2080static kern_return_t
2081mach_vm_range_create_v1(
2082 vm_map_t map,
2083 mach_vm_range_recipe_v1_ut *recipe_u,
2084 uint32_t new_count)
2085{
2086 mach_vm_range_recipe_v1_t *recipe;
2087 vm_map_user_range_t table;
2088 kern_return_t kr = KERN_SUCCESS;
2089 uint16_t count;
2090
2091 vmlp_api_start(MACH_VM_RANGE_CREATE_V1);
2092
2093 struct mach_vm_range void1 = {
2094 .min_address = map->default_range.max_address,
2095 .max_address = map->data_range.min_address,
2096 };
2097 struct mach_vm_range void2 = {
2098 .min_address = map->data_range.max_address,
2099#if XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT
2100 .max_address = MACH_VM_JUMBO_ADDRESS,
2101#else /* !XNU_TARGET_OS_IOS || !EXTENDED_USER_VA_SUPPORT */
2102 .max_address = vm_map_max(map),
2103#endif /* XNU_TARGET_OS_IOS && EXTENDED_USER_VA_SUPPORT */
2104 };
2105
2106 kr = mach_vm_range_create_v1_sanitize(map, recipe_u, new_count, &recipe);
2107 if (__improbable(kr != KERN_SUCCESS)) {
2108 kr = vm_sanitize_get_kr(kr);
2109 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, kr);
2110 return kr;
2111 }
2112
2113 qsort(recipe, new_count, sizeof(mach_vm_range_recipe_v1_t),
2114 mach_vm_range_recipe_v1_cmp);
2115
2116 /*
2117 * Step 1: Validate that the recipes have no intersections.
2118 */
2119
2120 for (size_t i = 0; i < new_count; i++) {
2121 mach_vm_range_t r = &recipe[i].range;
2122 mach_vm_size_t s;
2123
2124 if (recipe[i].flags) {
2125 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, KERN_INVALID_ARGUMENT);
2126 return KERN_INVALID_ARGUMENT;
2127 }
2128
2129 static_assert((int)UMEM_RANGE_ID_FIXED == MACH_VM_RANGE_FIXED);
2130 switch (recipe[i].range_tag) {
2131 case MACH_VM_RANGE_FIXED:
2132 break;
2133 default:
2134 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, KERN_INVALID_ARGUMENT);
2135 return KERN_INVALID_ARGUMENT;
2136 }
2137
2138 s = mach_vm_range_size(r);
2139 if (!mach_vm_range_contains(&void1, r->min_address, s) &&
2140 !mach_vm_range_contains(&void2, r->min_address, s)) {
2141 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, KERN_INVALID_ARGUMENT);
2142 return KERN_INVALID_ARGUMENT;
2143 }
2144
2145 if (i > 0 && recipe[i - 1].range.max_address >
2146 recipe[i].range.min_address) {
2147 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, KERN_INVALID_ARGUMENT);
2148 return KERN_INVALID_ARGUMENT;
2149 }
2150 }
2151
2152 vm_map_lock(map);
2153
2154 table = map->extra_ranges;
2155 count = map->extra_ranges_count;
2156
2157 if (count + new_count > VM_MAP_EXTRA_RANGES_MAX) {
2158 kr = KERN_NO_SPACE;
2159 goto out_unlock;
2160 }
2161
2162 /*
2163 * Step 2: Check that there is no intersection with existing ranges.
2164 */
2165
2166 for (size_t i = 0, j = 0; i < new_count && j < count;) {
2167 mach_vm_range_t r1 = &recipe[i].range;
2168 vm_map_user_range_t r2 = &table[j];
2169
2170 if (r1->max_address <= r2->vmur_min_address) {
2171 i++;
2172 } else if (r2->vmur_max_address <= r1->min_address) {
2173 j++;
2174 } else {
2175 kr = KERN_MEMORY_PRESENT;
2176 goto out_unlock;
2177 }
2178 }
2179
2180 /*
2181 * Step 3: commit the new ranges.
2182 */
2183
2184 static_assert(VM_MAP_EXTRA_RANGES_MAX * sizeof(struct vm_map_user_range) <=
2185 KALLOC_SAFE_ALLOC_SIZE);
2186
2187 table = krealloc_data(table,
2188 count * sizeof(struct vm_map_user_range),
2189 (count + new_count) * sizeof(struct vm_map_user_range),
2190 Z_ZERO | Z_WAITOK | Z_NOFAIL);
2191
2192 for (size_t i = 0; i < new_count; i++) {
2193 static_assert(MACH_VM_MAX_ADDRESS < (1ull << 56));
2194
2195 table[count + i] = (struct vm_map_user_range){
2196 .vmur_min_address = recipe[i].range.min_address,
2197 .vmur_max_address = recipe[i].range.max_address,
2198 .vmur_range_id = (vm_map_range_id_t)recipe[i].range_tag,
2199 };
2200 }
2201
2202 qsort(table, count + new_count,
2203 sizeof(struct vm_map_user_range), vm_map_user_range_cmp);
2204
2205 map->extra_ranges_count += new_count;
2206 map->extra_ranges = table;
2207
2208out_unlock:
2209 vm_map_unlock(map);
2210
2211 if (kr == KERN_SUCCESS) {
2212 for (size_t i = 0; i < new_count; i++) {
2213 vm_map_kernel_flags_t vmk_flags = {
2214 .vmf_fixed = true,
2215 .vmf_overwrite = true,
2216 .vmkf_overwrite_immutable = true,
2217 .vm_tag = recipe[i].vm_tag,
2218 };
2219 __assert_only kern_return_t kr2;
2220
2221 vmlp_range_event(map, recipe[i].range.min_address, recipe[i].range.max_address - recipe[i].range.min_address);
2222
2223 kr2 = vm_map_enter(map, &recipe[i].range.min_address,
2224 mach_vm_range_size(&recipe[i].range),
2225 0, vmk_flags, VM_OBJECT_NULL, 0, FALSE,
2226 VM_PROT_NONE, VM_PROT_ALL,
2227 VM_INHERIT_DEFAULT);
2228 assert(kr2 == KERN_SUCCESS);
2229 }
2230 }
2231 vmlp_api_end(MACH_VM_RANGE_CREATE_V1, kr);
2232 return kr;
2233}
2234
2235kern_return_t
2236mach_vm_range_create(
2237 vm_map_t map,
2238 mach_vm_range_flavor_t flavor,
2239 mach_vm_range_recipes_raw_t recipe,
2240 natural_t size)
2241{
2242 if (map != current_map()) {
2243 return KERN_INVALID_ARGUMENT;
2244 }
2245
2246 if (!map->uses_user_ranges) {
2247 return KERN_NOT_SUPPORTED;
2248 }
2249
2250 if (size == 0) {
2251 return KERN_SUCCESS;
2252 }
2253
2254 if (flavor == MACH_VM_RANGE_FLAVOR_V1) {
2255 mach_vm_range_recipe_v1_ut *array;
2256
2257 if (size % sizeof(mach_vm_range_recipe_v1_ut)) {
2258 return KERN_INVALID_ARGUMENT;
2259 }
2260
2261 size /= sizeof(mach_vm_range_recipe_v1_ut);
2262 if (size > VM_MAP_EXTRA_RANGES_MAX) {
2263 return KERN_NO_SPACE;
2264 }
2265
2266 array = (mach_vm_range_recipe_v1_ut *)recipe;
2267 return mach_vm_range_create_v1(map, array, size);
2268 }
2269
2270 return KERN_INVALID_ARGUMENT;
2271}
2272
2273#else /* !CONFIG_MAP_RANGES */
2274
2275kern_return_t
2276mach_vm_range_create(
2277 vm_map_t map,
2278 mach_vm_range_flavor_t flavor,
2279 mach_vm_range_recipes_raw_t recipe,
2280 natural_t size)
2281{
2282#pragma unused(map, flavor, recipe, size)
2283 return KERN_NOT_SUPPORTED;
2284}
2285
2286#endif /* !CONFIG_MAP_RANGES */
2287
2288/*
2289 * These symbols are looked up at runtime by vmware, VirtualBox,
2290 * despite not being exported in the symbol sets.
2291 */
2292
2293#if defined(__x86_64__)
2294
2295extern typeof(mach_vm_remap_external) mach_vm_remap;
2296extern typeof(mach_vm_map_external) mach_vm_map;
2297extern typeof(vm_map_external) vm_map;
2298
2299kern_return_t
2300mach_vm_map(
2301 vm_map_t target_map,
2302 mach_vm_offset_ut *address,
2303 mach_vm_size_ut initial_size,
2304 mach_vm_offset_ut mask,
2305 int flags,
2306 ipc_port_t port,
2307 memory_object_offset_ut offset,
2308 boolean_t copy,
2309 vm_prot_ut cur_protection,
2310 vm_prot_ut max_protection,
2311 vm_inherit_ut inheritance)
2312{
2313 return mach_vm_map_external(target_map, address, initial_size, mask, flags, port,
2314 offset, copy, cur_protection, max_protection, inheritance);
2315}
2316
2317kern_return_t
2318mach_vm_remap(
2319 vm_map_t target_map,
2320 mach_vm_offset_ut *address,
2321 mach_vm_size_ut size,
2322 mach_vm_offset_ut mask,
2323 int flags,
2324 vm_map_t src_map,
2325 mach_vm_offset_ut memory_address,
2326 boolean_t copy,
2327 vm_prot_ut *cur_protection, /* OUT */
2328 vm_prot_ut *max_protection, /* OUT */
2329 vm_inherit_ut inheritance)
2330{
2331 return mach_vm_remap_external(target_map, address, size, mask, flags, src_map, memory_address,
2332 copy, cur_protection, max_protection, inheritance);
2333}
2334
2335kern_return_t
2336vm_map(
2337 vm_map_t target_map,
2338 vm_offset_ut *address,
2339 vm_size_ut size,
2340 vm_offset_ut mask,
2341 int flags,
2342 ipc_port_t port,
2343 vm_offset_ut offset,
2344 boolean_t copy,
2345 vm_prot_ut cur_protection,
2346 vm_prot_ut max_protection,
2347 vm_inherit_ut inheritance)
2348{
2349 return mach_vm_map(target_map, address,
2350 size, mask, flags, port, offset, copy,
2351 cur_protection, max_protection, inheritance);
2352}
2353
2354#endif /* __x86_64__ */