From 6ff72a3702854bfa2e2c1b01d1ceb42d78894716 Mon Sep 17 00:00:00 2001 From: Roger Pau Monne Date: Tue, 14 Apr 2015 14:07:08 +0200 Subject: [PATCH] xen: introduce a newbus function to allocate unused memory In order to map memory from other domains when running on Xen FreeBSD uses unused physical memory regions. Until now this memory has been allocated using bus_alloc_resource, but this is not completely safe as we can end up using unreclaimed MMIO or ACPI regions. Fix this by introducing a new newbus method that can be used by Xen drivers to request for unused memory regions. On amd64 we make sure this memory comes from regions above 4GB in order to prevent clashes with MMIO/ACPI regions. On i386 there's nothing we can do, so just fall back to the previous mechanism. Sponsored by: Citrix Systems R&D --- sys/conf/files | 1 + sys/dev/xen/blkback/blkback.c | 11 ++-- sys/dev/xen/grant_table/grant_table.c | 5 +- sys/dev/xen/netback/netback.c | 9 ++-- sys/dev/xen/privcmd/privcmd.c | 26 +++------- sys/x86/xen/xenpv.c | 84 +++++++++++++++++++++++++++++++ sys/xen/xen-os.h | 7 +++ sys/xen/xenmem/xenmem_if.m | 95 +++++++++++++++++++++++++++++++++++ 8 files changed, 202 insertions(+), 36 deletions(-) create mode 100644 sys/xen/xenmem/xenmem_if.m diff --git a/sys/conf/files b/sys/conf/files index 7ef7afe..7bb423d 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -4053,6 +4053,7 @@ xen/xenbus/xenbusb_if.m optional xen | xenhvm xen/xenbus/xenbusb.c optional xen | xenhvm xen/xenbus/xenbusb_front.c optional xen | xenhvm xen/xenbus/xenbusb_back.c optional xen | xenhvm +xen/xenmem/xenmem_if.m optional xen | xenhvm xdr/xdr.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_array.c optional krpc | nfslockd | nfscl | nfsd xdr/xdr_mbuf.c optional krpc | nfslockd | nfscl | nfsd diff --git a/sys/dev/xen/blkback/blkback.c b/sys/dev/xen/blkback/blkback.c index 1273961..719accc 100644 --- a/sys/dev/xen/blkback/blkback.c +++ b/sys/dev/xen/blkback/blkback.c @@ -2822,9 +2822,8 @@ xbb_free_communication_mem(struct xbb_softc *xbb) kva_free(xbb->kva, xbb->kva_size); #else if (xbb->pseudo_phys_res != NULL) { - bus_release_resource(xbb->dev, SYS_RES_MEMORY, - xbb->pseudo_phys_res_id, - xbb->pseudo_phys_res); + xenmem_free(xbb->dev, xbb->pseudo_phys_res_id, + xbb->pseudo_phys_res); xbb->pseudo_phys_res = NULL; } #endif @@ -3068,10 +3067,8 @@ xbb_alloc_communication_mem(struct xbb_softc *xbb) * via grant table operations. */ xbb->pseudo_phys_res_id = 0; - xbb->pseudo_phys_res = bus_alloc_resource(xbb->dev, SYS_RES_MEMORY, - &xbb->pseudo_phys_res_id, - 0, ~0, xbb->kva_size, - RF_ACTIVE); + xbb->pseudo_phys_res = xenmem_alloc(xbb->dev, &xbb->pseudo_phys_res_id, + xbb->kva_size); if (xbb->pseudo_phys_res == NULL) { xbb->kva = 0; return (ENOMEM); diff --git a/sys/dev/xen/grant_table/grant_table.c b/sys/dev/xen/grant_table/grant_table.c index 2511657..0d144e6 100644 --- a/sys/dev/xen/grant_table/grant_table.c +++ b/sys/dev/xen/grant_table/grant_table.c @@ -627,9 +627,8 @@ gnttab_resume(device_t dev) KASSERT(dev != NULL, ("No resume frames and no device provided")); - gnttab_pseudo_phys_res = bus_alloc_resource(dev, - SYS_RES_MEMORY, &gnttab_pseudo_phys_res_id, 0, ~0, - PAGE_SIZE * max_nr_gframes, RF_ACTIVE); + gnttab_pseudo_phys_res = xenmem_alloc(dev, + &gnttab_pseudo_phys_res_id, PAGE_SIZE * max_nr_gframes); if (gnttab_pseudo_phys_res == NULL) panic("Unable to reserve physical memory for gnttab"); resume_frames = rman_get_start(gnttab_pseudo_phys_res); diff --git a/sys/dev/xen/netback/netback.c b/sys/dev/xen/netback/netback.c index 63337ad..edf8803 100644 --- a/sys/dev/xen/netback/netback.c +++ b/sys/dev/xen/netback/netback.c @@ -630,8 +630,7 @@ xnb_free_communication_mem(struct xnb_softc *xnb) kva_free(xnb->kva, xnb->kva_size); #else if (xnb->pseudo_phys_res != NULL) { - bus_release_resource(xnb->dev, SYS_RES_MEMORY, - xnb->pseudo_phys_res_id, + xenmem_free(xnb->dev, xnb->pseudo_phys_res_id, xnb->pseudo_phys_res); xnb->pseudo_phys_res = NULL; } @@ -830,10 +829,8 @@ xnb_alloc_communication_mem(struct xnb_softc *xnb) * into this space. */ xnb->pseudo_phys_res_id = 0; - xnb->pseudo_phys_res = bus_alloc_resource(xnb->dev, SYS_RES_MEMORY, - &xnb->pseudo_phys_res_id, - 0, ~0, xnb->kva_size, - RF_ACTIVE); + xnb->pseudo_phys_res = xenmem_alloc(xnb->dev, &xnb->pseudo_phys_res_id, + xnb->kva_size); if (xnb->pseudo_phys_res == NULL) { xnb->kva = 0; return (ENOMEM); diff --git a/sys/dev/xen/privcmd/privcmd.c b/sys/dev/xen/privcmd/privcmd.c index 761fb03..0bf9585 100644 --- a/sys/dev/xen/privcmd/privcmd.c +++ b/sys/dev/xen/privcmd/privcmd.c @@ -141,11 +141,8 @@ retry: free(map->errs, M_PRIVCMD); } - vm_phys_fictitious_unreg_range(map->phys_base_addr, - map->phys_base_addr + map->size * PAGE_SIZE); - - error = bus_release_resource(privcmd_dev, SYS_RES_MEMORY, - map->pseudo_phys_res_id, map->pseudo_phys_res); + error = xenmem_free(privcmd_dev, map->pseudo_phys_res_id, + map->pseudo_phys_res); KASSERT(error == 0, ("Unable to release memory resource: %d", error)); free(map, M_PRIVCMD); @@ -196,36 +193,25 @@ privcmd_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, vm_object_t *object, int nprot) { struct privcmd_map *map; - int error; map = malloc(sizeof(*map), M_PRIVCMD, M_WAITOK | M_ZERO); map->size = OFF_TO_IDX(size); map->pseudo_phys_res_id = 0; - map->pseudo_phys_res = bus_alloc_resource(privcmd_dev, SYS_RES_MEMORY, - &map->pseudo_phys_res_id, 0, ~0, size, RF_ACTIVE); + map->pseudo_phys_res = xenmem_alloc(privcmd_dev, + &map->pseudo_phys_res_id, size); if (map->pseudo_phys_res == NULL) { free(map, M_PRIVCMD); return (ENOMEM); } map->phys_base_addr = rman_get_start(map->pseudo_phys_res); - - error = vm_phys_fictitious_reg_range(map->phys_base_addr, - map->phys_base_addr + size, VM_MEMATTR_DEFAULT); - if (error) { - bus_release_resource(privcmd_dev, SYS_RES_MEMORY, - map->pseudo_phys_res_id, map->pseudo_phys_res); - free(map, M_PRIVCMD); - return (error); - } - map->mem = cdev_pager_allocate(map, OBJT_MGTDEVICE, &privcmd_pg_ops, size, nprot, *offset, NULL); if (map->mem == NULL) { - bus_release_resource(privcmd_dev, SYS_RES_MEMORY, - map->pseudo_phys_res_id, map->pseudo_phys_res); + xenmem_free(privcmd_dev, map->pseudo_phys_res_id, + map->pseudo_phys_res); free(map, M_PRIVCMD); return (ENOMEM); } diff --git a/sys/x86/xen/xenpv.c b/sys/x86/xen/xenpv.c index bdda883..2e4a9fe 100644 --- a/sys/x86/xen/xenpv.c +++ b/sys/x86/xen/xenpv.c @@ -33,11 +33,33 @@ __FBSDID("$FreeBSD$"); #include #include #include +#include #include +#include +#include +#include +#include + #include #include +#include "xenmem_if.h" + +/* + * Allocate unused physical memory above 4GB in order to map memory + * from foreign domains. We use memory starting at 4GB in order to + * prevent clashes with MMIO/ACPI regions. + * + * Since this is not possible on i386 just use any available memory + * chunk and hope we don't clash with anything else. + */ +#ifdef __amd64__ +#define LOW_MEM_LIMIT 0x100000000ul +#else +#define LOW_MEM_LIMIT 0 +#endif + static devclass_t xenpv_devclass; static void @@ -85,6 +107,42 @@ xenpv_attach(device_t dev) return (0); } +static struct resource * +xenpv_alloc_physmem(device_t dev, device_t child, int *res_id, size_t size) +{ + struct resource *res; + vm_paddr_t phys_addr; + int error; + + res = bus_alloc_resource(child, SYS_RES_MEMORY, res_id, LOW_MEM_LIMIT, + ~0ul, size, RF_ACTIVE); + if (res == NULL) + return (NULL); + + phys_addr = rman_get_start(res); + error = vm_phys_fictitious_reg_range(phys_addr, phys_addr + size, + VM_MEMATTR_DEFAULT); + if (error) { + bus_release_resource(child, SYS_RES_MEMORY, *res_id, res); + return (NULL); + } + + return (res); +} + +static int +xenpv_free_physmem(device_t dev, device_t child, int res_id, struct resource *res) +{ + vm_paddr_t phys_addr; + size_t size; + + phys_addr = rman_get_start(res); + size = rman_get_size(res); + + vm_phys_fictitious_unreg_range(phys_addr, phys_addr + size); + return (bus_release_resource(child, SYS_RES_MEMORY, res_id, res)); +} + static device_method_t xenpv_methods[] = { /* Device interface */ DEVMETHOD(device_identify, xenpv_identify), @@ -100,6 +158,10 @@ static device_method_t xenpv_methods[] = { DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), + /* Interface to allocate memory for foreign mappings */ + DEVMETHOD(xenmem_alloc, xenpv_alloc_physmem), + DEVMETHOD(xenmem_free, xenpv_free_physmem), + DEVMETHOD_END }; @@ -110,3 +172,25 @@ static driver_t xenpv_driver = { }; DRIVER_MODULE(xenpv, nexus, xenpv_driver, xenpv_devclass, 0, 0); + +struct resource * +xenmem_alloc(device_t dev, int *res_id, size_t size) +{ + device_t parent; + + parent = device_get_parent(dev); + if (parent == NULL) + return (NULL); + return (XENMEM_ALLOC(parent, dev, res_id, size)); +} + +int +xenmem_free(device_t dev, int res_id, struct resource *res) +{ + device_t parent; + + parent = device_get_parent(dev); + if (parent == NULL) + return (ENXIO); + return (XENMEM_FREE(parent, dev, res_id, res)); +} diff --git a/sys/xen/xen-os.h b/sys/xen/xen-os.h index c982d55..7ceef4f 100644 --- a/sys/xen/xen-os.h +++ b/sys/xen/xen-os.h @@ -89,6 +89,13 @@ xen_initial_domain(void) (HYPERVISOR_start_info->flags & SIF_INITDOMAIN) != 0); } +/* + * Functions to allocate/free unused memory in order + * to map memory from other domains. + */ +struct resource *xenmem_alloc(device_t dev, int *res_id, size_t size); +int xenmem_free(device_t dev, int res_id, struct resource *res); + /* Debug/emergency function, prints directly to hypervisor console */ void xc_printf(const char *, ...) __printflike(1, 2); diff --git a/sys/xen/xenmem/xenmem_if.m b/sys/xen/xenmem/xenmem_if.m new file mode 100644 index 0000000..5cc2171 --- /dev/null +++ b/sys/xen/xenmem/xenmem_if.m @@ -0,0 +1,95 @@ +#- +# Copyright (c) 2015 Roger Pau Monné +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# $FreeBSD$ +# + +#include + +INTERFACE xenmem; + +# +# Default implementations of some methods. +# +CODE { + static struct resource * + xenmem_generic_alloc(device_t dev, device_t child, int *res_id, + size_t size) + { + device_t parent; + + parent = device_get_parent(dev); + if (parent == NULL) + return (NULL); + return (XENMEM_ALLOC(parent, child, res_id, size)); + } + + static int + xenmem_generic_free(device_t dev, device_t child, int res_id, + struct resource *res) + { + device_t parent; + + parent = device_get_parent(dev); + if (parent == NULL) + return (ENXIO); + return (XENMEM_FREE(parent, child, res_id, res)); + } +}; + +/** + * @brief Request for unused physical memory regions. + * + * @param _dev the device whose child was being probed. + * @param _child the child device which failed to probe. + * @param _res_id a pointer to the resource identifier. + * @param _size size of the required memory region. + * + * @returns the resource which was allocated or @c NULL if no + * resource could be allocated. + */ +METHOD struct resource * alloc { + device_t _dev; + device_t _child; + int *_res_id; + size_t _size; +} DEFAULT xenmem_generic_alloc; + +/** + * @brief Free physical memory regions. + * + * @param _dev the device whose child was being probed. + * @param _child the child device which failed to probe. + * @param _res_id the resource identifier. + * @param _res the resource. + * + * @returns 0 on success, otherwise an error code. + */ +METHOD int free { + device_t _dev; + device_t _child; + int _res_id; + struct resource *_res; +} DEFAULT xenmem_generic_free; -- 1.9.5 (Apple Git-50.3)