Index: lib/libc/sys/mmap.2 =================================================================== --- lib/libc/sys/mmap.2 (revision 258445) +++ lib/libc/sys/mmap.2 (working copy) @@ -28,7 +28,7 @@ .\" @(#)mmap.2 8.4 (Berkeley) 5/11/95 .\" $FreeBSD$ .\" -.Dd March 18, 2012 +.Dd August 16, 2013 .Dt MMAP 2 .Os .Sh NAME @@ -97,7 +97,30 @@ argument by .Em or Ns 'ing the following values: -.Bl -tag -width MAP_HASSEMAPHORE +.Bl -tag -width MAP_PREFAULT_READ +.It Dv MAP_ALIGNED Ns Pq Fa n +Align the region on a requested boundary. +If a suitable region cannot be found, +.Fn mmap +will fail. +The +.Fa n +argument specifies the binary logarithm of the desired alignment. +.It Dv MAP_ALIGNED_SUPER +Align the region to maximize the potential use of large +.Pq Dq super +pages. +If a suitable region cannot be found, +.Fn mmap +will fail. +The system will choose a suitable page size based on the size of +mapping. +The page size used as well as the alignment of the region may both be +affected by properties of the file being mapped. +In particular, +the physical address of existing pages of a file may require a specific +alignment. +The region is not guaranteed to be aligned on any specific boundary. .It Dv MAP_ANON Map anonymous memory not associated with any specific file. The file descriptor used for creating @@ -274,6 +297,25 @@ the .Fa offset argument, a portable program must only use page-aligned values. +.Pp +Large page mappings require that the pages backing an object be +aligned in matching blocks in both the virtual address space and RAM. +The system will automatically attempt to use large page mappings when +mapping an object that is already backed by large pages in RAM by +aligning the mapping request in the virtual address space to match the +alignment of the large physical pages. +The system may also use large page mappings when mapping portions of an +object that are not yet backed by pages in RAM. +The +.Dv MAP_ALIGNED_SUPER +flag is an optimization that will align the mapping request to the +size of a large page similar to +.Dv MAP_ALIGNED , +except that the system will override this alignment if an object already +uses large pages so that the mapping will be consistent with the existing +large pages. +This flag is mostly useful for maximizing the use of large pages on the +first mapping of objects that do not yet have pages present in RAM. .Sh RETURN VALUES Upon successful completion, .Fn mmap @@ -325,6 +367,10 @@ argument was equal to zero. .It Bq Er EINVAL +.Dv MAP_ALIGNED +was specified and the desired alignment was either larger than the +virtual address size of the machine or smaller than a page. +.It Bq Er EINVAL .Dv MAP_ANON was specified and the .Fa fd @@ -356,7 +402,8 @@ .Xr msync 2 , .Xr munlock 2 , .Xr munmap 2 , -.Xr getpagesize 3 +.Xr getpagesize 3 , +.Xr getpagesizes 3 .Sh BUGS The .Fa len Index: lib/libc/sys =================================================================== --- lib/libc/sys (revision 258445) +++ lib/libc/sys (working copy) Property changes on: lib/libc/sys ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/lib/libc/sys:r254430 Index: lib/libc =================================================================== --- lib/libc (revision 258445) +++ lib/libc (working copy) Property changes on: lib/libc ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/lib/libc:r254430 Index: sys/dev/drm2/i915/i915_gem.c =================================================================== --- sys/dev/drm2/i915/i915_gem.c (revision 258445) +++ sys/dev/drm2/i915/i915_gem.c (working copy) @@ -1289,7 +1289,7 @@ vm_object_reference(obj->vm_obj); DRM_UNLOCK(dev); rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, - VMFS_ANY_SPACE, VM_PROT_READ | VM_PROT_WRITE, + VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, MAP_INHERIT_SHARE); if (rv != KERN_SUCCESS) { vm_object_deallocate(obj->vm_obj); Index: sys/dev =================================================================== --- sys/dev (revision 258445) +++ sys/dev (working copy) Property changes on: sys/dev ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev:r253471,253620,254430 Index: sys/kern/sysv_shm.c =================================================================== --- sys/kern/sysv_shm.c (revision 258445) +++ sys/kern/sysv_shm.c (working copy) @@ -413,7 +413,7 @@ vm_object_reference(shmseg->object); rv = vm_map_find(&p->p_vmspace->vm_map, shmseg->object, 0, &attach_va, size, (flags & MAP_FIXED) ? VMFS_NO_SPACE : - VMFS_ANY_SPACE, prot, prot, MAP_INHERIT_SHARE); + VMFS_OPTIMAL_SPACE, prot, prot, MAP_INHERIT_SHARE); if (rv != KERN_SUCCESS) { vm_object_deallocate(shmseg->object); error = ENOMEM; Index: sys/kern/uipc_shm.c =================================================================== --- sys/kern/uipc_shm.c (revision 258445) +++ sys/kern/uipc_shm.c (working copy) @@ -778,7 +778,7 @@ offset = trunc_page(offset); size = round_page(size + ofs); rv = vm_map_find(kernel_map, obj, offset, &kva, size, - VMFS_ALIGNED_SPACE, VM_PROT_READ | VM_PROT_WRITE, + VMFS_OPTIMAL_SPACE, VM_PROT_READ | VM_PROT_WRITE, VM_PROT_READ | VM_PROT_WRITE, 0); if (rv == KERN_SUCCESS) { rv = vm_map_wire(kernel_map, kva, kva + size, Index: sys/sys/mman.h =================================================================== --- sys/sys/mman.h (revision 258445) +++ sys/sys/mman.h (working copy) @@ -91,6 +91,17 @@ */ #define MAP_NOCORE 0x00020000 /* dont include these pages in a coredump */ #define MAP_PREFAULT_READ 0x00040000 /* prefault mapping for reading */ + +/* + * Request specific alignment (n == log2 of the desired alignment). + * + * MAP_ALIGNED_SUPER requests optimal superpage alignment, but does + * not enforce a specific alignment. + */ +#define MAP_ALIGNED(n) ((n) << MAP_ALIGNMENT_SHIFT) +#define MAP_ALIGNMENT_SHIFT 24 +#define MAP_ALIGNMENT_MASK MAP_ALIGNED(0xff) +#define MAP_ALIGNED_SUPER MAP_ALIGNED(1) /* align on a superpage */ #endif /* __BSD_VISIBLE */ #if __POSIX_VISIBLE >= 199309 Index: sys/sys =================================================================== --- sys/sys (revision 258445) +++ sys/sys (working copy) Property changes on: sys/sys ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/sys:r254430 Index: sys/vm/vm_kern.c =================================================================== --- sys/vm/vm_kern.c (revision 258445) +++ sys/vm/vm_kern.c (working copy) @@ -238,7 +238,7 @@ *min = vm_map_min(parent); ret = vm_map_find(parent, NULL, 0, min, size, superpage_align ? - VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, + VMFS_SUPER_SPACE : VMFS_ANY_SPACE, VM_PROT_ALL, VM_PROT_ALL, MAP_ACC_NO_CHARGE); if (ret != KERN_SUCCESS) panic("kmem_suballoc: bad status return of %d", ret); Index: sys/vm/vm_map.c =================================================================== --- sys/vm/vm_map.c (revision 258445) +++ sys/vm/vm_map.c (working copy) @@ -1444,19 +1444,34 @@ vm_size_t length, int find_space, vm_prot_t prot, vm_prot_t max, int cow) { - vm_offset_t start; + vm_offset_t alignment, initial_addr, start; int result; - start = *addr; + if (find_space == VMFS_OPTIMAL_SPACE && (object == NULL || + (object->flags & OBJ_COLORED) == 0)) + find_space = VMFS_ANY_SPACE; + if (find_space >> 8 != 0) { + KASSERT((find_space & 0xff) == 0, ("bad VMFS flags")); + alignment = (vm_offset_t)1 << (find_space >> 8); + } else + alignment = 0; + initial_addr = *addr; +again: + start = initial_addr; vm_map_lock(map); do { if (find_space != VMFS_NO_SPACE) { if (vm_map_findspace(map, start, length, addr)) { vm_map_unlock(map); + if (find_space == VMFS_OPTIMAL_SPACE) { + find_space = VMFS_ANY_SPACE; + goto again; + } return (KERN_NO_SPACE); } switch (find_space) { - case VMFS_ALIGNED_SPACE: + case VMFS_SUPER_SPACE: + case VMFS_OPTIMAL_SPACE: pmap_align_superpage(object, offset, addr, length); break; @@ -1465,7 +1480,13 @@ pmap_align_tlb(addr); break; #endif + case VMFS_ANY_SPACE: + break; default: + if ((*addr & (alignment - 1)) != 0) { + *addr &= ~(alignment - 1); + *addr += alignment; + } break; } @@ -1473,11 +1494,8 @@ } result = vm_map_insert(map, object, offset, start, start + length, prot, max, cow); - } while (result == KERN_NO_SPACE && (find_space == VMFS_ALIGNED_SPACE -#ifdef VMFS_TLB_ALIGNED_SPACE - || find_space == VMFS_TLB_ALIGNED_SPACE -#endif - )); + } while (result == KERN_NO_SPACE && find_space != VMFS_NO_SPACE && + find_space != VMFS_ANY_SPACE); vm_map_unlock(map); return (result); } Index: sys/vm/vm_map.h =================================================================== --- sys/vm/vm_map.h (revision 258445) +++ sys/vm/vm_map.h (working copy) @@ -339,14 +339,19 @@ #define VM_FAULT_READ_AHEAD_MAX min(atop(MAXPHYS) - 1, UINT8_MAX) /* - * The following "find_space" options are supported by vm_map_find() + * The following "find_space" options are supported by vm_map_find(). + * + * For VMFS_ALIGNED_SPACE, the desired alignment is specified to + * the macro argument as log base 2 of the desired alignment. */ #define VMFS_NO_SPACE 0 /* don't find; use the given range */ #define VMFS_ANY_SPACE 1 /* find a range with any alignment */ -#define VMFS_ALIGNED_SPACE 2 /* find a superpage-aligned range */ +#define VMFS_SUPER_SPACE 2 /* find a superpage-aligned range */ +#define VMFS_ALIGNED_SPACE(x) ((x) << 8) /* find a range with fixed alignment */ #if defined(__mips__) #define VMFS_TLB_ALIGNED_SPACE 3 /* find a TLB entry aligned range */ #endif +#define VMFS_OPTIMAL_SPACE 4 /* find a range with optimal alignment*/ /* * vm_map_wire and vm_map_unwire option flags Index: sys/vm/vm_mmap.c =================================================================== --- sys/vm/vm_mmap.c (revision 258445) +++ sys/vm/vm_mmap.c (working copy) @@ -201,7 +201,7 @@ vm_prot_t cap_maxprot, prot, maxprot; void *handle; objtype_t handle_type; - int flags, error; + int align, error, flags; off_t pos; struct vmspace *vms = td->td_proc->p_vmspace; cap_rights_t rights; @@ -251,6 +251,13 @@ size += pageoff; /* low end... */ size = (vm_size_t) round_page(size); /* hi end */ + /* Ensure alignment is at least a page and fits in a pointer. */ + align = flags & MAP_ALIGNMENT_MASK; + if (align != 0 && align != MAP_ALIGNED_SUPER && + (align >> MAP_ALIGNMENT_SHIFT >= sizeof(void *) * NBBY || + align >> MAP_ALIGNMENT_SHIFT < PAGE_SHIFT)) + return (EINVAL); + /* * Check for illegal addresses. Watch out for address wrap... Note * that VM_*_ADDRESS are not constants due to casts (argh). @@ -1486,7 +1493,7 @@ boolean_t fitit; vm_object_t object = NULL; struct thread *td = curthread; - int docow, error, rv; + int docow, error, findspace, rv; boolean_t writecounted; if (size == 0) @@ -1601,11 +1608,17 @@ if (flags & MAP_STACK) rv = vm_map_stack(map, *addr, size, prot, maxprot, docow | MAP_STACK_GROWS_DOWN); - else if (fitit) - rv = vm_map_find(map, object, foff, addr, size, - object != NULL && object->type == OBJT_DEVICE ? - VMFS_ALIGNED_SPACE : VMFS_ANY_SPACE, prot, maxprot, docow); - else + else if (fitit) { + if ((flags & MAP_ALIGNMENT_MASK) == MAP_ALIGNED_SUPER) + findspace = VMFS_SUPER_SPACE; + else if ((flags & MAP_ALIGNMENT_MASK) != 0) + findspace = VMFS_ALIGNED_SPACE(flags >> + MAP_ALIGNMENT_SHIFT); + else + findspace = VMFS_OPTIMAL_SPACE; + rv = vm_map_find(map, object, foff, addr, size, findspace, + prot, maxprot, docow); + } else rv = vm_map_fixed(map, object, foff, *addr, size, prot, maxprot, docow); Index: sys =================================================================== --- sys (revision 258445) +++ sys (working copy) Property changes on: sys ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys:r253471,253620,254430 Index: usr.bin/kdump/mksubr =================================================================== --- usr.bin/kdump/mksubr (revision 258445) +++ usr.bin/kdump/mksubr (working copy) @@ -370,7 +370,6 @@ auto_switch_type "madvisebehavname" "_?MADV_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h" auto_switch_type "minheritname" "INHERIT_[A-Z]+[[:space:]]+[0-9]+" "sys/mman.h" auto_or_type "mlockallname" "MCL_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mman.h" -auto_or_type "mmapflagsname" "MAP_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h" auto_or_type "mmapprotname" "PROT_[A-Z]+[[:space:]]+0x[0-9A-Fa-f]+" "sys/mman.h" auto_or_type "modename" "S_[A-Z]+[[:space:]]+[0-6]{7}" "sys/stat.h" auto_or_type "mountflagsname" "MNT_[A-Z]+[[:space:]]+0x[0-9]+" "sys/mount.h" @@ -448,6 +447,40 @@ /* * AUTO - Special * + * The MAP_ALIGNED flag requires special handling. + */ +void +mmapflagsname(int flags) +{ + int align; + int or = 0; + printf("%#x<", flags); +_EOF_ +egrep "^#[[:space:]]*define[[:space:]]+MAP_[A-Z_]+[[:space:]]+0x[0-9A-Fa-f]+[[:space:]]*" \ + $include_dir/sys/mman.h | grep -v MAP_ALIGNED | \ + awk '{ for (i = 1; i <= NF; i++) \ + if ($i ~ /define/) \ + break; \ + ++i; \ + printf "\tif (!((flags > 0) ^ ((%s) > 0)))\n\t\tif_print_or(flags, %s, or);\n", $i, $i }' +cat <<_EOF_ + align = flags & MAP_ALIGNMENT_MASK; + if (align != 0) { + if (align == MAP_ALIGNED_SUPER) + print_or("MAP_ALIGNED_SUPER", or); + else { + print_or("MAP_ALIGNED", or); + printf("(%d)", align >> MAP_ALIGNMENT_SHIFT); + } + } + printf(">"); + if (or == 0) + printf("%d", flags); +} + +/* + * AUTO - Special + * * The only reason this is not fully automated is due to the * grep -v RTP_PRIO statement. A better egrep line should * make this capable of being a auto_switch_type() function. Index: usr.bin/kdump =================================================================== --- usr.bin/kdump (revision 258445) +++ usr.bin/kdump (working copy) Property changes on: usr.bin/kdump ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/usr.bin/kdump:r254430 Index: usr.bin/truss/syscalls.c =================================================================== --- usr.bin/truss/syscalls.c (revision 258445) +++ usr.bin/truss/syscalls.c (working copy) @@ -296,7 +296,7 @@ X(MAP_SHARED) X(MAP_PRIVATE) X(MAP_FIXED) X(MAP_RENAME) X(MAP_NORESERVE) X(MAP_RESERVED0080) X(MAP_RESERVED0100) X(MAP_HASSEMAPHORE) X(MAP_STACK) X(MAP_NOSYNC) X(MAP_ANON) - X(MAP_NOCORE) XEND + X(MAP_NOCORE) X(MAP_PREFAULT_READ) XEND }; static struct xlat mprot_flags[] = { @@ -893,9 +893,41 @@ case Mprot: tmp = strdup(xlookup_bits(mprot_flags, args[sc->offset])); break; - case Mmapflags: - tmp = strdup(xlookup_bits(mmap_flags, args[sc->offset])); + case Mmapflags: { + char *base, *alignstr; + int align, flags; + + /* + * MAP_ALIGNED can't be handled by xlookup_bits(), so + * generate that string manually and prepend it to the + * string from xlookup_bits(). Have to be careful to + * avoid outputting MAP_ALIGNED|0 if MAP_ALIGNED is + * the only flag. + */ + flags = args[sc->offset] & ~MAP_ALIGNMENT_MASK; + align = args[sc->offset] & MAP_ALIGNMENT_MASK; + if (align != 0) { + if (align == MAP_ALIGNED_SUPER) + alignstr = strdup("MAP_ALIGNED_SUPER"); + else + asprintf(&alignstr, "MAP_ALIGNED(%d)", + align >> MAP_ALIGNMENT_SHIFT); + if (flags == 0) { + tmp = alignstr; + break; + } + } else + alignstr = NULL; + base = strdup(xlookup_bits(mmap_flags, flags)); + if (alignstr == NULL) { + tmp = base; + break; + } + asprintf(&tmp, "%s|%s", alignstr, base); + free(alignstr); + free(base); break; + } case Whence: tmp = strdup(xlookup(whence_arg, args[sc->offset])); break; Index: usr.bin/truss =================================================================== --- usr.bin/truss (revision 258445) +++ usr.bin/truss (working copy) Property changes on: usr.bin/truss ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/usr.bin/truss:r254430,254538