Index: sys/amd64/include/atomic.h =================================================================== --- sys/amd64/include/atomic.h (revision 261896) +++ sys/amd64/include/atomic.h (working copy) @@ -54,6 +54,7 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) @@ -60,6 +61,7 @@ * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -80,6 +82,8 @@ int atomic_cmpset_int(volatile u_int *dst, u_int e int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); +int atomic_testandset_int(volatile u_int *p, u_int v); +int atomic_testandset_long(volatile u_long *p, u_int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -211,6 +215,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Ir" (v & 0x1f) /* 2 */ + : "cc"); + return (res); +} + +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsq %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_long" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Jr" ((u_long)(v & 0x3f)) /* 2 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -338,10 +376,36 @@ atomic_readandclear_long(volatile u_long *addr) return (res); } +static __inline u_int +atomic_swap_int(volatile u_int *p, u_int v) +{ + + __asm __volatile( + " xchgl %1,%0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "+m" (*p)); /* 1 */ + return (v); +} + +static __inline u_long +atomic_swap_long(volatile u_long *p, u_long v) +{ + + __asm __volatile( + " xchgq %1,%0 ; " + "# atomic_swap_long" + : "+r" (v), /* 0 */ + "+m" (*p)); /* 1 */ + return (v); +} + #else /* !__GNUCLIKE_ASM */ u_int atomic_readandclear_int(volatile u_int *addr); u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *p, u_int v); +u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ @@ -435,8 +499,10 @@ u_long atomic_readandclear_long(volatile u_long *a #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on 64-bit quad words. */ #define atomic_set_64 atomic_set_long @@ -456,7 +522,9 @@ u_long atomic_readandclear_long(volatile u_long *a #define atomic_cmpset_64 atomic_cmpset_long #define atomic_cmpset_acq_64 atomic_cmpset_acq_long #define atomic_cmpset_rel_64 atomic_cmpset_rel_long +#define atomic_swap_64 atomic_swap_long #define atomic_readandclear_64 atomic_readandclear_long +#define atomic_testandset_64 atomic_testandset_long /* Operations on pointers. */ #define atomic_set_ptr atomic_set_long @@ -476,6 +544,7 @@ u_long atomic_readandclear_long(volatile u_long *a #define atomic_cmpset_ptr atomic_cmpset_long #define atomic_cmpset_acq_ptr atomic_cmpset_acq_long #define atomic_cmpset_rel_ptr atomic_cmpset_rel_long +#define atomic_swap_ptr atomic_swap_long #define atomic_readandclear_ptr atomic_readandclear_long #endif /* !WANT_FUNCTIONS */ Index: sys/dev/drm2/drmP.h =================================================================== --- sys/dev/drm2/drmP.h (revision 261896) +++ sys/dev/drm2/drmP.h (working copy) @@ -1,9 +1,15 @@ -/* drmP.h -- Private header for Direct Rendering Manager -*- linux-c -*- - * Created: Mon Jan 4 10:05:05 1999 by faith@precisioninsight.com +/** + * \file drmP.h + * Private header for Direct Rendering Manager + * + * \author Rickard E. (Rik) Faith + * \author Gareth Hughes */ -/*- + +/* * Copyright 1999 Precision Insight, Inc., Cedar Park, Texas. * Copyright 2000 VA Linux Systems, Inc., Sunnyvale, California. + * Copyright (c) 2009-2010, Code Aurora Forum. * All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a @@ -24,11 +30,6 @@ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. - * - * Authors: - * Rickard E. (Rik) Faith - * Gareth Hughes - * */ #include @@ -39,9 +40,6 @@ __FBSDID("$FreeBSD$"); #if defined(_KERNEL) || defined(__KERNEL__) -struct drm_device; -struct drm_file; - #include #include #include @@ -98,12 +96,19 @@ __FBSDID("$FreeBSD$"); #include #include +#include + #include #include #include #include + +struct drm_file; +struct drm_device; + +#include +#include #include -#include #include "opt_compat.h" #include "opt_drm.h" @@ -119,6 +124,10 @@ __FBSDID("$FreeBSD$"); #undef DRM_LINUX #define DRM_LINUX 0 +/***********************************************************************/ +/** \name DRM template customization defaults */ +/*@{*/ + /* driver capabilities and requirements mask */ #define DRIVER_USE_AGP 0x1 #define DRIVER_REQUIRE_AGP 0x2 @@ -134,8 +143,8 @@ __FBSDID("$FreeBSD$"); #define DRIVER_IRQ_VBL2 0x800 #define DRIVER_GEM 0x1000 #define DRIVER_MODESET 0x2000 -#define DRIVER_USE_PLATFORM_DEVICE 0x4000 -#define DRIVER_LOCKLESS_IRQ 0x8000 +#define DRIVER_PRIME 0x4000 +#define DRIVER_LOCKLESS_IRQ 0x8000 #define DRM_HASH_SIZE 16 /* Size of key hash table */ @@ -176,7 +185,10 @@ SYSCTL_DECL(_hw_drm); #define DRM_MAX_CTXBITMAP (PAGE_SIZE * 8) - /* Internal types and structures */ +/***********************************************************************/ +/** \name Internal types and structures */ +/*@{*/ + #define DRM_ARRAY_SIZE(x) (sizeof(x)/sizeof(x[0])) #define DRM_MIN(a,b) ((a)<(b)?(a):(b)) #define DRM_MAX(a,b) ((a)>(b)?(a):(b)) @@ -227,12 +239,6 @@ typedef void irqreturn_t; #define IRQ_HANDLED /* nothing */ #define IRQ_NONE /* nothing */ -#define unlikely(x) __builtin_expect(!!(x), 0) -#define likely(x) __builtin_expect(!!(x), 1) -#define container_of(ptr, type, member) ({ \ - __typeof( ((type *)0)->member ) *__mptr = (ptr); \ - (type *)( (char *)__mptr - offsetof(type,member) );}) - enum { DRM_IS_NOT_AGP, DRM_IS_AGP, @@ -254,16 +260,6 @@ enum { #define time_after_eq(a,b) ((long)(b) - (long)(a) <= 0) #define drm_msleep(x, msg) pause((msg), ((int64_t)(x)) * hz / 1000) -typedef vm_paddr_t dma_addr_t; -typedef uint64_t u64; -typedef uint32_t u32; -typedef uint16_t u16; -typedef uint8_t u8; -typedef int64_t s64; -typedef int32_t s32; -typedef int16_t s16; -typedef int8_t s8; - /* DRM_READMEMORYBARRIER() prevents reordering of reads. * DRM_WRITEMEMORYBARRIER() prevents reordering of writes. * DRM_MEMORYBARRIER() prevents reordering of reads and writes. @@ -311,16 +307,6 @@ enum { #define DRM_GET_USER_UNCHECKED(val, uaddr) \ ((val) = fuword32(uaddr), 0) -#define cpu_to_le32(x) htole32(x) -#define le32_to_cpu(x) le32toh(x) - -#define DRM_HZ hz -#define DRM_UDELAY(udelay) DELAY(udelay) -#define DRM_MDELAY(msecs) do { int loops = (msecs); \ - while (loops--) DELAY(1000); \ - } while (0) -#define DRM_TIME_SLICE (hz/20) /* Time slice for GLXContexts */ - #define DRM_GET_PRIV_SAREA(_dev, _ctx, _map) do { \ (_map) = (_dev)->context_sareas[_ctx]; \ } while(0) @@ -371,6 +357,18 @@ for ( ret = 0 ; !ret && !(condition) ; ) { \ __func__ , ##__VA_ARGS__); \ } while (0) +#define dev_err(dev, fmt, ...) \ + device_printf((dev), "error: " fmt, ## __VA_ARGS__) +#define dev_warn(dev, fmt, ...) \ + device_printf((dev), "warning: " fmt, ## __VA_ARGS__) +#define dev_info(dev, fmt, ...) \ + device_printf((dev), "info: " fmt, ## __VA_ARGS__) +#define dev_dbg(dev, fmt, ...) do { \ + if ((drm_debug_flag& DRM_DEBUGBITS_KMS) != 0) { \ + device_printf((dev), "debug: " fmt, ## __VA_ARGS__); \ + } \ +} while (0) + typedef struct drm_pci_id_list { int vendor; @@ -386,7 +384,7 @@ struct drm_msi_blacklist_entry }; #define DRM_AUTH 0x1 -#define DRM_MASTER 0x2 +#define DRM_MASTER 0x2 #define DRM_ROOT_ONLY 0x4 #define DRM_CONTROL_ALLOW 0x8 #define DRM_UNLOCKED 0x10 @@ -396,7 +394,9 @@ typedef struct drm_ioctl_desc { int (*func)(struct drm_device *dev, void *data, struct drm_file *file_priv); int flags; + unsigned int cmd_drv; } drm_ioctl_desc_t; + /** * Creates a driver or general drm_ioctl_desc array entry for the given * ioctl, for use by drm_ioctl(). @@ -404,6 +404,9 @@ typedef struct drm_ioctl_desc { #define DRM_IOCTL_DEF(ioctl, func, flags) \ [DRM_IOCTL_NR(ioctl)] = {ioctl, func, flags} +#define DRM_IOCTL_DEF_DRV(ioctl, _func, _flags) \ + [DRM_IOCTL_NR(DRM_##ioctl)] = {.cmd = DRM_##ioctl, .func = _func, .flags = _flags, .cmd_drv = DRM_IOCTL_##ioctl} + typedef struct drm_magic_entry { drm_magic_t magic; struct drm_file *priv; @@ -416,28 +419,30 @@ typedef struct drm_magic_head { } drm_magic_head_t; typedef struct drm_buf { - int idx; /* Index into master buflist */ - int total; /* Buffer size */ - int order; /* log-base-2(total) */ - int used; /* Amount of buffer in use (for DMA) */ - unsigned long offset; /* Byte offset (used internally) */ - void *address; /* Address of buffer */ - unsigned long bus_address; /* Bus address of buffer */ - struct drm_buf *next; /* Kernel-only: used for free list */ - __volatile__ int pending; /* On hardware DMA queue */ - struct drm_file *file_priv; /* Unique identifier of holding process */ - int context; /* Kernel queue for this buffer */ + int idx; /**< Index into master buflist */ + int total; /**< Buffer size */ + int order; /**< log-base-2(total) */ + int used; /**< Amount of buffer in use (for DMA) */ + unsigned long offset; /**< Byte offset (used internally) */ + void *address; /**< Address of buffer */ + unsigned long bus_address; /**< Bus address of buffer */ + struct drm_buf *next; /**< Kernel-only: used for free list */ + __volatile__ int waiting; /**< On kernel DMA queue */ + __volatile__ int pending; /**< On hardware DMA queue */ + struct drm_file *file_priv; /**< Private of holding file descr */ + int context; /**< Kernel queue for this buffer */ + int while_locked; /**< Dispatch this buffer while locked */ enum { - DRM_LIST_NONE = 0, - DRM_LIST_FREE = 1, - DRM_LIST_WAIT = 2, - DRM_LIST_PEND = 3, - DRM_LIST_PRIO = 4, + DRM_LIST_NONE = 0, + DRM_LIST_FREE = 1, + DRM_LIST_WAIT = 2, + DRM_LIST_PEND = 3, + DRM_LIST_PRIO = 4, DRM_LIST_RECLAIM = 5 - } list; /* Which list we're on */ + } list; /**< Which list we're on */ - int dev_priv_size; /* Size of buffer private stoarge */ - void *dev_private; /* Per-buffer private storage */ + int dev_priv_size; /**< Size of buffer private storage */ + void *dev_private; /**< Per-buffer private storage */ } drm_buf_t; typedef struct drm_freelist { @@ -477,6 +482,14 @@ struct drm_pending_event { void (*destroy)(struct drm_pending_event *event); }; +/* initial implementaton using a linked list - todo hashtab */ +struct drm_prime_file_private { + struct list_head head; +#ifdef DUMBBELL_WIP + struct mutex lock; +#endif /* DUMBBELL_WIP */ +}; + typedef TAILQ_HEAD(drm_file_list, drm_file) drm_file_list_t; struct drm_file { TAILQ_ENTRY(drm_file) link; @@ -499,6 +512,8 @@ struct drm_file { struct list_head event_list; int event_space; struct selinfo event_poll; + + struct drm_prime_file_private prime; }; typedef struct drm_lock_data { @@ -517,17 +532,21 @@ typedef struct drm_lock_data { * concurrently accessed, so no locking is needed. */ typedef struct drm_device_dma { - drm_buf_entry_t bufs[DRM_MAX_ORDER+1]; - int buf_count; - drm_buf_t **buflist; /* Vector of pointers info bufs */ - int seg_count; - int page_count; - unsigned long *pagelist; - unsigned long byte_count; + + struct drm_buf_entry bufs[DRM_MAX_ORDER + 1]; /**< buffers, grouped by their size order */ + int buf_count; /**< total number of buffers */ + struct drm_buf **buflist; /**< Vector of pointers into drm_device_dma::bufs */ + int seg_count; + int page_count; /**< number of pages */ + unsigned long *pagelist; /**< page list */ + unsigned long byte_count; enum { _DRM_DMA_USE_AGP = 0x01, - _DRM_DMA_USE_SG = 0x02 + _DRM_DMA_USE_SG = 0x02, + _DRM_DMA_USE_FB = 0x04, + _DRM_DMA_USE_PCI_RO = 0x08 } flags; + } drm_device_dma_t; typedef struct drm_agp_mem { @@ -592,28 +611,13 @@ struct drm_vblank_info { int inmodeset; /* Display driver is setting mode */ }; -/* Size of ringbuffer for vblank timestamps. Just double-buffer - * in initial implementation. - */ -#define DRM_VBLANKTIME_RBSIZE 2 - -/* Flags and return codes for get_vblank_timestamp() driver function. */ -#define DRM_CALLED_FROM_VBLIRQ 1 -#define DRM_VBLANKTIME_SCANOUTPOS_METHOD (1 << 0) -#define DRM_VBLANKTIME_INVBL (1 << 1) - -/* get_scanout_position() return flags */ -#define DRM_SCANOUTPOS_VALID (1 << 0) -#define DRM_SCANOUTPOS_INVBL (1 << 1) -#define DRM_SCANOUTPOS_ACCURATE (1 << 2) - /* location of GART table */ #define DRM_ATI_GART_MAIN 1 #define DRM_ATI_GART_FB 2 -#define DRM_ATI_GART_PCI 1 +#define DRM_ATI_GART_PCI 1 #define DRM_ATI_GART_PCIE 2 -#define DRM_ATI_GART_IGP 3 +#define DRM_ATI_GART_IGP 3 struct drm_ati_pcigart_info { int gart_table_location; @@ -685,10 +689,58 @@ struct drm_gem_object { uint32_t pending_write_domain; void *driver_private; + +#ifdef DUMBBELL_WIP + /* dma buf exported from this GEM object */ + struct dma_buf *export_dma_buf; + + /* dma buf attachment backing this object */ + struct dma_buf_attachment *import_attach; +#endif /* DUMBBELL_WIP */ }; #include "drm_crtc.h" +/* per-master structure */ +struct drm_master { + + u_int refcount; /* refcount for this master */ + + struct list_head head; /**< each minor contains a list of masters */ + struct drm_minor *minor; /**< link back to minor we are a master for */ + + char *unique; /**< Unique identifier: e.g., busid */ + int unique_len; /**< Length of unique field */ + int unique_size; /**< amount allocated */ + + int blocked; /**< Blocked due to VC switch? */ + + /** \name Authentication */ + /*@{ */ + struct drm_open_hash magiclist; + struct list_head magicfree; + /*@} */ + + struct drm_lock_data lock; /**< Information on hardware lock */ + + void *driver_priv; /**< Private structure for driver to use */ +}; + +/* Size of ringbuffer for vblank timestamps. Just double-buffer + * in initial implementation. + */ +#define DRM_VBLANKTIME_RBSIZE 2 + +/* Flags and return codes for get_vblank_timestamp() driver function. */ +#define DRM_CALLED_FROM_VBLIRQ 1 +#define DRM_VBLANKTIME_SCANOUTPOS_METHOD (1 << 0) +#define DRM_VBLANKTIME_INVBL (1 << 1) + +/* get_scanout_position() return flags */ +#define DRM_SCANOUTPOS_VALID (1 << 0) +#define DRM_SCANOUTPOS_INVBL (1 << 1) +#define DRM_SCANOUTPOS_ACCURATE (1 << 2) + #ifndef DMA_BIT_MASK #define DMA_BIT_MASK(n) (((n) == 64) ? ~0ULL : (1ULL<<(n)) - 1) #endif @@ -760,7 +812,7 @@ struct drm_driver_info { * * \param dev DRM device handle * - * \returns + * \returns * One of three values is returned depending on whether or not the * card is absolutely \b not AGP (return of 0), absolutely \b is AGP * (return of 1), or may or may not be AGP (return of 2). @@ -815,6 +867,7 @@ struct drm_cmdline_mode { enum drm_connector_force force; }; + struct drm_pending_vblank_event { struct drm_pending_event base; int pipe; @@ -824,8 +877,9 @@ struct drm_pending_vblank_event { /* Length for the array of resource pointers for drm_get_resource_*. */ #define DRM_MAX_PCI_RESOURCE 6 -/** - * DRM device functions structure +/** + * DRM device structure. This structure represent a complete card that + * may contain multiple heads. */ struct drm_device { struct drm_driver_info *driver; @@ -914,6 +968,7 @@ struct drm_device { struct drm_minor *control; /**< Control node for card */ struct drm_minor *primary; /**< render type primary screen head */ + void *drm_ttm_bdev; struct unrhdr *drw_unrhdr; /* RB tree of drawable infos */ RB_HEAD(drawable_tree, bsd_drm_drawable_info) drw_head; @@ -948,8 +1003,14 @@ struct drm_device { void *sysctl_private; char busid_str[128]; int modesetting; + + int switch_power_state; }; +#define DRM_SWITCH_POWER_ON 0 +#define DRM_SWITCH_POWER_OFF 1 +#define DRM_SWITCH_POWER_CHANGING 2 + static __inline__ int drm_core_check_feature(struct drm_device *dev, int feature) { @@ -1030,6 +1091,41 @@ extern int drm_open_helper(struct cdev *kdev, int DRM_STRUCTPROC *p, struct drm_device *dev); +#ifdef DUMBBELL_WIP +extern int drm_gem_prime_handle_to_fd(struct drm_device *dev, + struct drm_file *file_priv, uint32_t handle, uint32_t flags, + int *prime_fd); +extern int drm_gem_prime_fd_to_handle(struct drm_device *dev, + struct drm_file *file_priv, int prime_fd, uint32_t *handle); + +extern int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); +extern int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); + +#ifdef DUMBBELL_WIP +/* + * See drm_prime.c + * -- dumbbell@ + */ +extern int drm_prime_sg_to_page_addr_arrays(struct sg_table *sgt, vm_page_t *pages, + dma_addr_t *addrs, int max_pages); +#endif /* DUMBBELL_WIP */ +extern struct sg_table *drm_prime_pages_to_sg(vm_page_t *pages, int nr_pages); +extern void drm_prime_gem_destroy(struct drm_gem_object *obj, struct sg_table *sg); + + +void drm_prime_init_file_private(struct drm_prime_file_private *prime_fpriv); +void drm_prime_destroy_file_private(struct drm_prime_file_private *prime_fpriv); +int drm_prime_add_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t handle); +int drm_prime_lookup_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf, uint32_t *handle); +void drm_prime_remove_imported_buf_handle(struct drm_prime_file_private *prime_fpriv, struct dma_buf *dma_buf); + +int drm_prime_add_dma_buf(struct drm_device *dev, struct drm_gem_object *obj); +int drm_prime_lookup_obj(struct drm_device *dev, struct dma_buf *buf, + struct drm_gem_object **obj); +#endif /* DUMBBELL_WIP */ + /* Memory management support (drm_memory.c) */ void drm_mem_init(void); void drm_mem_uninit(void); @@ -1311,10 +1407,16 @@ void drm_gem_release(struct drm_device *dev, struc int drm_gem_create_mmap_offset(struct drm_gem_object *obj); void drm_gem_free_mmap_offset(struct drm_gem_object *obj); -int drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, - struct vm_object **obj_res, int nprot); +int drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); void drm_gem_pager_dtr(void *obj); +struct ttm_bo_device; +int ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); +struct ttm_buffer_object; +void ttm_bo_release_mmap(struct ttm_buffer_object *bo); + void drm_device_lock_mtx(struct drm_device *dev); void drm_device_unlock_mtx(struct drm_device *dev); int drm_device_sleep_mtx(struct drm_device *dev, void *chan, int flags, @@ -1402,31 +1504,6 @@ static __inline__ void drm_core_dropmap(struct drm { } -#define KIB_NOTYET() \ -do { \ - if (drm_debug_flag && drm_notyet_flag) \ - printf("NOTYET: %s at %s:%d\n", __func__, __FILE__, __LINE__); \ -} while (0) - -#define KTR_DRM KTR_DEV -#define KTR_DRM_REG KTR_SPARE3 - -/* Error codes conversion from Linux to FreeBSD. */ -/* XXXKIB what is the right code for EREMOTEIO on FreeBSD? */ -#define EREMOTEIO ENXIO -#define ERESTARTSYS ERESTART - -#define PCI_VENDOR_ID_APPLE 0x106b -#define PCI_VENDOR_ID_ASUSTEK 0x1043 -#define PCI_VENDOR_ID_ATI 0x1002 -#define PCI_VENDOR_ID_DELL 0x1028 -#define PCI_VENDOR_ID_HP 0x103c -#define PCI_VENDOR_ID_IBM 0x1014 -#define PCI_VENDOR_ID_INTEL 0x8086 -#define PCI_VENDOR_ID_SERVERWORKS 0x1166 -#define PCI_VENDOR_ID_SONY 0x104d -#define PCI_VENDOR_ID_VIA 0x1106 - #define DRM_PCIE_SPEED_25 1 #define DRM_PCIE_SPEED_50 2 #define DRM_PCIE_SPEED_80 4 @@ -1433,5 +1510,7 @@ static __inline__ void drm_core_dropmap(struct drm extern int drm_pcie_get_speed_cap_mask(struct drm_device *dev, u32 *speed_mask); +#define drm_can_sleep() (DRM_HZ & 1) + #endif /* __KERNEL__ */ #endif /* _DRM_P_H_ */ Index: sys/dev/drm2/drm_atomic.h =================================================================== --- sys/dev/drm2/drm_atomic.h (revision 261896) +++ sys/dev/drm2/drm_atomic.h (working copy) @@ -32,62 +32,51 @@ #include __FBSDID("$FreeBSD$"); -/* Many of these implementations are rather fake, but good enough. */ +typedef uint32_t atomic_t; +typedef uint64_t atomic64_t; -typedef u_int32_t atomic_t; +#define BITS_TO_LONGS(x) howmany(x, sizeof(long) * NBBY) -#define atomic_set(p, v) (*(p) = (v)) -#define atomic_read(p) (*(p)) -#define atomic_inc(p) atomic_add_int(p, 1) -#define atomic_dec(p) atomic_subtract_int(p, 1) -#define atomic_add(n, p) atomic_add_int(p, n) -#define atomic_sub(n, p) atomic_subtract_int(p, n) +#define atomic_set(p, v) atomic_store_rel_int(p, v) +#define atomic_read(p) atomic_load_acq_int(p) -static __inline atomic_t -test_and_set_bit(int b, volatile void *p) -{ - int s = splhigh(); - unsigned int m = 1<> 5), 1 << (b & 0x1f)); -} +#define atomic_add_return(v, p) (atomic_fetchadd_int(p, v) + (v)) +#define atomic_sub_return(v, p) (atomic_fetchadd_int(p, -(v)) - (v)) +#define atomic_inc_return(p) atomic_add_return(1, p) +#define atomic_dec_return(p) atomic_sub_return(1, p) -static __inline void -set_bit(int b, volatile void *p) -{ - atomic_set_int(((volatile int *)p) + (b >> 5), 1 << (b & 0x1f)); -} +#define atomic_add_and_test(v, p) (atomic_add_return(v, p) == 0) +#define atomic_sub_and_test(v, p) (atomic_sub_return(v, p) == 0) +#define atomic_inc_and_test(p) (atomic_inc_return(p) == 0) +#define atomic_dec_and_test(p) (atomic_dec_return(p) == 0) -static __inline int -test_bit(int b, volatile void *p) -{ - return ((volatile int *)p)[b >> 5] & (1 << (b & 0x1f)); -} +#define atomic_xchg(p, v) atomic_swap_int(p, v) +#define atomic64_xchg(p, v) atomic_swap_64(p, v) +#define clear_bit(b, p) \ + atomic_clear_int((volatile u_int *)(p) + (b) / 32, 1 << (b) % 32) +#define set_bit(b, p) \ + atomic_set_int((volatile u_int *)(p) + (b) / 32, 1 << (b) % 32) +#define test_bit(b, p) \ + (atomic_load_acq_int((volatile u_int *)(p) + (b) / 32) & (1 << (b) % 32)) +#define test_and_set_bit(b, p) \ + atomic_testandset_int((volatile u_int *)(p) + (b) / 32, b) + static __inline int find_first_zero_bit(volatile void *p, int max) { - int b; - volatile int *ptr = (volatile int *)p; + volatile int *np = p; + int i, n; - for (b = 0; b < max; b += 32) { - if (ptr[b >> 5] != ~0) { - for (;;) { - if ((ptr[b >> 5] & (1 << (b & 0x1f))) == 0) - return b; - b++; - } - } + for (i = 0; i < max / (NBBY * sizeof(int)); i++) { + n = ~np[i]; + if (n != 0) + return (i * NBBY * sizeof(int) + ffs(n) - 1); } - return max; + return (max); } - -#define BITS_TO_LONGS(x) (howmany((x), NBBY * sizeof(long))) Index: sys/dev/drm2/drm_crtc.c =================================================================== --- sys/dev/drm2/drm_crtc.c (revision 261896) +++ sys/dev/drm2/drm_crtc.c (working copy) @@ -2317,6 +2317,7 @@ int drm_mode_getfb(struct drm_device *dev, r->depth = fb->depth; r->bpp = fb->bits_per_pixel; r->pitch = fb->pitches[0]; + r->handle = 0; fb->funcs->create_handle(fb, file_priv, &r->handle); out: Index: sys/dev/drm2/drm_drv.c =================================================================== --- sys/dev/drm2/drm_drv.c (revision 261896) +++ sys/dev/drm2/drm_drv.c (working copy) @@ -66,6 +66,8 @@ static int drm_load(struct drm_device *dev); static void drm_unload(struct drm_device *dev); static drm_pci_id_list_t *drm_find_description(int vendor, int device, drm_pci_id_list_t *idlist); +static int drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, + vm_size_t size, struct vm_object **obj_res, int nprot); static int drm_modevent(module_t mod, int type, void *data) @@ -195,7 +197,7 @@ static struct cdevsw drm_cdevsw = { .d_ioctl = drm_ioctl, .d_poll = drm_poll, .d_mmap = drm_mmap, - .d_mmap_single = drm_gem_mmap_single, + .d_mmap_single = drm_mmap_single, .d_name = "drm", .d_flags = D_TRACKCLOSE }; @@ -990,6 +992,23 @@ drm_add_busid_modesetting(struct drm_device *dev, return (0); } +static int +drm_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, + struct vm_object **obj_res, int nprot) +{ + struct drm_device *dev; + + dev = drm_get_device_from_kdev(kdev); + if (dev->drm_ttm_bdev != NULL) { + return (ttm_bo_mmap_single(dev->drm_ttm_bdev, offset, size, + obj_res, nprot)); + } else if ((dev->driver->driver_features & DRIVER_GEM) != 0) { + return (drm_gem_mmap_single(dev, offset, size, obj_res, nprot)); + } else { + return (ENODEV); + } +} + #if DRM_LINUX #include Index: sys/dev/drm2/drm_gem.c =================================================================== --- sys/dev/drm2/drm_gem.c (revision 261896) +++ sys/dev/drm2/drm_gem.c (working copy) @@ -121,7 +121,7 @@ drm_gem_private_object_init(struct drm_device *dev obj->vm_obj = NULL; obj->refcount = 1; - atomic_set(&obj->handle_count, 0); + atomic_store_rel_int(&obj->handle_count, 0); obj->size = size; return (0); @@ -465,16 +465,12 @@ drm_gem_free_mmap_offset(struct drm_gem_object *ob } int -drm_gem_mmap_single(struct cdev *kdev, vm_ooffset_t *offset, vm_size_t size, +drm_gem_mmap_single(struct drm_device *dev, vm_ooffset_t *offset, vm_size_t size, struct vm_object **obj_res, int nprot) { - struct drm_device *dev; struct drm_gem_object *gem_obj; struct vm_object *vm_obj; - dev = drm_get_device_from_kdev(kdev); - if ((dev->driver->driver_features & DRIVER_GEM) == 0) - return (ENODEV); DRM_LOCK(dev); gem_obj = drm_gem_object_from_offset(dev, *offset); if (gem_obj == NULL) { Index: sys/dev/drm2/drm_global.c =================================================================== --- sys/dev/drm2/drm_global.c (revision 261896) +++ sys/dev/drm2/drm_global.c (working copy) @@ -104,6 +104,7 @@ void drm_global_item_unref(struct drm_global_refer MPASS(ref->object == item->object); if (--item->refcount == 0) { ref->release(ref); + free(item->object, M_DRM_GLOBAL); item->object = NULL; } sx_xunlock(&item->mutex); Index: sys/dev/drm2/drm_irq.c =================================================================== --- sys/dev/drm2/drm_irq.c (revision 261896) +++ sys/dev/drm2/drm_irq.c (working copy) @@ -786,7 +786,7 @@ int drm_vblank_get(struct drm_device *dev, int crt mtx_lock(&dev->vbl_lock); /* Going from 0->1 means we have to enable interrupts again */ - if (atomic_fetchadd_int(&dev->vblank_refcount[crtc], 1) == 0) { + if (atomic_add_return(1, &dev->vblank_refcount[crtc]) == 1) { mtx_lock(&dev->vblank_time_lock); if (!dev->vblank_enabled[crtc]) { /* Enable vblank irqs under vblank_time_lock protection. @@ -831,7 +831,7 @@ void drm_vblank_put(struct drm_device *dev, int cr ("Too many drm_vblank_put for crtc %d", crtc)); /* Last user schedules interrupt disable */ - if (atomic_fetchadd_int(&dev->vblank_refcount[crtc], -1) == 1 && + if (atomic_dec_and_test(&dev->vblank_refcount[crtc]) && (drm_vblank_offdelay > 0)) callout_reset(&dev->vblank_disable_callout, (drm_vblank_offdelay * DRM_HZ) / 1000, Index: sys/dev/drm2/i915/i915_gem.c =================================================================== --- sys/dev/drm2/i915/i915_gem.c (revision 261896) +++ sys/dev/drm2/i915/i915_gem.c (working copy) @@ -2504,8 +2504,10 @@ i915_gem_wire_page(vm_object_t object, vm_pindex_t int rv; VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); - m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); + m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | + VM_ALLOC_RETRY); if (m->valid != VM_PAGE_BITS_ALL) { + vm_page_busy(m); if (vm_pager_has_page(object, pindex, NULL, NULL)) { rv = vm_pager_get_pages(object, &m, 1, 0); m = vm_page_lookup(object, pindex); @@ -2522,11 +2524,11 @@ i915_gem_wire_page(vm_object_t object, vm_pindex_t m->valid = VM_PAGE_BITS_ALL; m->dirty = 0; } + vm_page_wakeup(m); } vm_page_lock(m); vm_page_wire(m); vm_page_unlock(m); - vm_page_wakeup(m); atomic_add_long(&i915_gem_wired_pages_cnt, 1); return (m); } Index: sys/dev/drm2/radeon/atom.c =================================================================== --- sys/dev/drm2/radeon/atom.c (revision 254885) +++ sys/dev/drm2/radeon/atom.c (working copy) @@ -89,11 +89,11 @@ static void debug_print_spaces(int n) printf(" "); } -#define DEBUG(...) do if (atom_debug) { printf(__FILE__ __VA_ARGS__); } while (0) -#define SDEBUG(...) do if (atom_debug) { printf(__FILE__); debug_print_spaces(debug_depth); printf(__VA_ARGS__); } while (0) +#define ATOM_DEBUG_PRINT(...) do if (atom_debug) { printf(__FILE__ __VA_ARGS__); } while (0) +#define ATOM_SDEBUG_PRINT(...) do if (atom_debug) { printf(__FILE__); debug_print_spaces(debug_depth); printf(__VA_ARGS__); } while (0) #else -#define DEBUG(...) do { } while (0) -#define SDEBUG(...) do { } while (0) +#define ATOM_DEBUG_PRINT(...) do { } while (0) +#define ATOM_SDEBUG_PRINT(...) do { } while (0) #endif static uint32_t atom_iio_execute(struct atom_context *ctx, int base, @@ -183,7 +183,7 @@ static uint32_t atom_get_src_int(atom_exec_context idx = U16(*ptr); (*ptr) += 2; if (print) - DEBUG("REG[0x%04X]", idx); + ATOM_DEBUG_PRINT("REG[0x%04X]", idx); idx += gctx->reg_block; switch (gctx->io_mode) { case ATOM_IO_MM: @@ -221,13 +221,13 @@ static uint32_t atom_get_src_int(atom_exec_context * tables, noticed on a DEC Alpha. */ val = get_unaligned_le32((u32 *)&ctx->ps[idx]); if (print) - DEBUG("PS[0x%02X,0x%04X]", idx, val); + ATOM_DEBUG_PRINT("PS[0x%02X,0x%04X]", idx, val); break; case ATOM_ARG_WS: idx = U8(*ptr); (*ptr)++; if (print) - DEBUG("WS[0x%02X]", idx); + ATOM_DEBUG_PRINT("WS[0x%02X]", idx); switch (idx) { case ATOM_WS_QUOTIENT: val = gctx->divmul[0]; @@ -265,9 +265,9 @@ static uint32_t atom_get_src_int(atom_exec_context (*ptr) += 2; if (print) { if (gctx->data_block) - DEBUG("ID[0x%04X+%04X]", idx, gctx->data_block); + ATOM_DEBUG_PRINT("ID[0x%04X+%04X]", idx, gctx->data_block); else - DEBUG("ID[0x%04X]", idx); + ATOM_DEBUG_PRINT("ID[0x%04X]", idx); } val = U32(idx + gctx->data_block); break; @@ -281,7 +281,7 @@ static uint32_t atom_get_src_int(atom_exec_context } else val = gctx->scratch[(gctx->fb_base / 4) + idx]; if (print) - DEBUG("FB[0x%02X]", idx); + ATOM_DEBUG_PRINT("FB[0x%02X]", idx); break; case ATOM_ARG_IMM: switch (align) { @@ -289,7 +289,7 @@ static uint32_t atom_get_src_int(atom_exec_context val = U32(*ptr); (*ptr) += 4; if (print) - DEBUG("IMM 0x%08X\n", val); + ATOM_DEBUG_PRINT("IMM 0x%08X\n", val); return val; case ATOM_SRC_WORD0: case ATOM_SRC_WORD8: @@ -297,7 +297,7 @@ static uint32_t atom_get_src_int(atom_exec_context val = U16(*ptr); (*ptr) += 2; if (print) - DEBUG("IMM 0x%04X\n", val); + ATOM_DEBUG_PRINT("IMM 0x%04X\n", val); return val; case ATOM_SRC_BYTE0: case ATOM_SRC_BYTE8: @@ -306,7 +306,7 @@ static uint32_t atom_get_src_int(atom_exec_context val = U8(*ptr); (*ptr)++; if (print) - DEBUG("IMM 0x%02X\n", val); + ATOM_DEBUG_PRINT("IMM 0x%02X\n", val); return val; } return 0; @@ -314,7 +314,7 @@ static uint32_t atom_get_src_int(atom_exec_context idx = U8(*ptr); (*ptr)++; if (print) - DEBUG("PLL[0x%02X]", idx); + ATOM_DEBUG_PRINT("PLL[0x%02X]", idx); val = gctx->card->pll_read(gctx->card, idx); break; case ATOM_ARG_MC: @@ -321,7 +321,7 @@ static uint32_t atom_get_src_int(atom_exec_context idx = U8(*ptr); (*ptr)++; if (print) - DEBUG("MC[0x%02X]", idx); + ATOM_DEBUG_PRINT("MC[0x%02X]", idx); val = gctx->card->mc_read(gctx->card, idx); break; } @@ -332,28 +332,28 @@ static uint32_t atom_get_src_int(atom_exec_context if (print) switch (align) { case ATOM_SRC_DWORD: - DEBUG(".[31:0] -> 0x%08X\n", val); + ATOM_DEBUG_PRINT(".[31:0] -> 0x%08X\n", val); break; case ATOM_SRC_WORD0: - DEBUG(".[15:0] -> 0x%04X\n", val); + ATOM_DEBUG_PRINT(".[15:0] -> 0x%04X\n", val); break; case ATOM_SRC_WORD8: - DEBUG(".[23:8] -> 0x%04X\n", val); + ATOM_DEBUG_PRINT(".[23:8] -> 0x%04X\n", val); break; case ATOM_SRC_WORD16: - DEBUG(".[31:16] -> 0x%04X\n", val); + ATOM_DEBUG_PRINT(".[31:16] -> 0x%04X\n", val); break; case ATOM_SRC_BYTE0: - DEBUG(".[7:0] -> 0x%02X\n", val); + ATOM_DEBUG_PRINT(".[7:0] -> 0x%02X\n", val); break; case ATOM_SRC_BYTE8: - DEBUG(".[15:8] -> 0x%02X\n", val); + ATOM_DEBUG_PRINT(".[15:8] -> 0x%02X\n", val); break; case ATOM_SRC_BYTE16: - DEBUG(".[23:16] -> 0x%02X\n", val); + ATOM_DEBUG_PRINT(".[23:16] -> 0x%02X\n", val); break; case ATOM_SRC_BYTE24: - DEBUG(".[31:24] -> 0x%02X\n", val); + ATOM_DEBUG_PRINT(".[31:24] -> 0x%02X\n", val); break; } return val; @@ -458,7 +458,7 @@ static void atom_put_dst(atom_exec_context *ctx, i case ATOM_ARG_REG: idx = U16(*ptr); (*ptr) += 2; - DEBUG("REG[0x%04X]", idx); + ATOM_DEBUG_PRINT("REG[0x%04X]", idx); idx += gctx->reg_block; switch (gctx->io_mode) { case ATOM_IO_MM: @@ -494,13 +494,13 @@ static void atom_put_dst(atom_exec_context *ctx, i case ATOM_ARG_PS: idx = U8(*ptr); (*ptr)++; - DEBUG("PS[0x%02X]", idx); + ATOM_DEBUG_PRINT("PS[0x%02X]", idx); ctx->ps[idx] = cpu_to_le32(val); break; case ATOM_ARG_WS: idx = U8(*ptr); (*ptr)++; - DEBUG("WS[0x%02X]", idx); + ATOM_DEBUG_PRINT("WS[0x%02X]", idx); switch (idx) { case ATOM_WS_QUOTIENT: gctx->divmul[0] = val; @@ -538,45 +538,45 @@ static void atom_put_dst(atom_exec_context *ctx, i gctx->fb_base + (idx * 4), gctx->scratch_size_bytes); } else gctx->scratch[(gctx->fb_base / 4) + idx] = val; - DEBUG("FB[0x%02X]", idx); + ATOM_DEBUG_PRINT("FB[0x%02X]", idx); break; case ATOM_ARG_PLL: idx = U8(*ptr); (*ptr)++; - DEBUG("PLL[0x%02X]", idx); + ATOM_DEBUG_PRINT("PLL[0x%02X]", idx); gctx->card->pll_write(gctx->card, idx, val); break; case ATOM_ARG_MC: idx = U8(*ptr); (*ptr)++; - DEBUG("MC[0x%02X]", idx); + ATOM_DEBUG_PRINT("MC[0x%02X]", idx); gctx->card->mc_write(gctx->card, idx, val); return; } switch (align) { case ATOM_SRC_DWORD: - DEBUG(".[31:0] <- 0x%08X\n", old_val); + ATOM_DEBUG_PRINT(".[31:0] <- 0x%08X\n", old_val); break; case ATOM_SRC_WORD0: - DEBUG(".[15:0] <- 0x%04X\n", old_val); + ATOM_DEBUG_PRINT(".[15:0] <- 0x%04X\n", old_val); break; case ATOM_SRC_WORD8: - DEBUG(".[23:8] <- 0x%04X\n", old_val); + ATOM_DEBUG_PRINT(".[23:8] <- 0x%04X\n", old_val); break; case ATOM_SRC_WORD16: - DEBUG(".[31:16] <- 0x%04X\n", old_val); + ATOM_DEBUG_PRINT(".[31:16] <- 0x%04X\n", old_val); break; case ATOM_SRC_BYTE0: - DEBUG(".[7:0] <- 0x%02X\n", old_val); + ATOM_DEBUG_PRINT(".[7:0] <- 0x%02X\n", old_val); break; case ATOM_SRC_BYTE8: - DEBUG(".[15:8] <- 0x%02X\n", old_val); + ATOM_DEBUG_PRINT(".[15:8] <- 0x%02X\n", old_val); break; case ATOM_SRC_BYTE16: - DEBUG(".[23:16] <- 0x%02X\n", old_val); + ATOM_DEBUG_PRINT(".[23:16] <- 0x%02X\n", old_val); break; case ATOM_SRC_BYTE24: - DEBUG(".[31:24] <- 0x%02X\n", old_val); + ATOM_DEBUG_PRINT(".[31:24] <- 0x%02X\n", old_val); break; } } @@ -586,12 +586,12 @@ static void atom_op_add(atom_exec_context *ctx, in uint8_t attr = U8((*ptr)++); uint32_t dst, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst += src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -600,12 +600,12 @@ static void atom_op_and(atom_exec_context *ctx, in uint8_t attr = U8((*ptr)++); uint32_t dst, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst &= src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -620,9 +620,9 @@ static void atom_op_calltable(atom_exec_context *c int r = 0; if (idx < ATOM_TABLE_NAMES_CNT) - SDEBUG(" table: %d (%s)\n", idx, atom_table_names[idx]); + ATOM_SDEBUG_PRINT(" table: %d (%s)\n", idx, atom_table_names[idx]); else - SDEBUG(" table: %d\n", idx); + ATOM_SDEBUG_PRINT(" table: %d\n", idx); if (U16(ctx->ctx->cmd_table + 4 + 2 * idx)) r = atom_execute_table_locked(ctx->ctx, idx, ctx->ps + ctx->ps_shift); if (r) { @@ -638,7 +638,7 @@ static void atom_op_clear(atom_exec_context *ctx, attr &= 0x38; attr |= atom_def_dst[attr >> 3] << 6; atom_get_dst(ctx, arg, attr, ptr, &saved, 0); - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, 0, saved); } @@ -646,13 +646,13 @@ static void atom_op_compare(atom_exec_context *ctx { uint8_t attr = U8((*ptr)++); uint32_t dst, src; - SDEBUG(" src1: "); + ATOM_SDEBUG_PRINT(" src1: "); dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); - SDEBUG(" src2: "); + ATOM_SDEBUG_PRINT(" src2: "); src = atom_get_src(ctx, attr, ptr); ctx->ctx->cs_equal = (dst == src); ctx->ctx->cs_above = (dst > src); - SDEBUG(" result: %s %s\n", ctx->ctx->cs_equal ? "EQ" : "NE", + ATOM_SDEBUG_PRINT(" result: %s %s\n", ctx->ctx->cs_equal ? "EQ" : "NE", ctx->ctx->cs_above ? "GT" : "LE"); } @@ -659,7 +659,7 @@ static void atom_op_compare(atom_exec_context *ctx static void atom_op_delay(atom_exec_context *ctx, int *ptr, int arg) { unsigned count = U8((*ptr)++); - SDEBUG(" count: %d\n", count); + ATOM_SDEBUG_PRINT(" count: %d\n", count); if (arg == ATOM_UNIT_MICROSEC) DRM_UDELAY(count); else if (!drm_can_sleep()) @@ -672,9 +672,9 @@ static void atom_op_div(atom_exec_context *ctx, in { uint8_t attr = U8((*ptr)++); uint32_t dst, src; - SDEBUG(" src1: "); + ATOM_SDEBUG_PRINT(" src1: "); dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); - SDEBUG(" src2: "); + ATOM_SDEBUG_PRINT(" src2: "); src = atom_get_src(ctx, attr, ptr); if (src != 0) { ctx->ctx->divmul[0] = dst / src; @@ -720,8 +720,8 @@ static void atom_op_jump(atom_exec_context *ctx, i break; } if (arg != ATOM_COND_ALWAYS) - SDEBUG(" taken: %s\n", execute ? "yes" : "no"); - SDEBUG(" target: 0x%04X\n", target); + ATOM_SDEBUG_PRINT(" taken: %s\n", execute ? "yes" : "no"); + ATOM_SDEBUG_PRINT(" target: 0x%04X\n", target); if (execute) { if (ctx->last_jump == (ctx->start + target)) { cjiffies = jiffies; @@ -748,15 +748,15 @@ static void atom_op_mask(atom_exec_context *ctx, i uint8_t attr = U8((*ptr)++); uint32_t dst, mask, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); mask = atom_get_src_direct(ctx, ((attr >> 3) & 7), ptr); - SDEBUG(" mask: 0x%08x", mask); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" mask: 0x%08x", mask); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst &= mask; dst |= src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -771,9 +771,9 @@ static void atom_op_move(atom_exec_context *ctx, i atom_skip_dst(ctx, arg, attr, ptr); saved = 0xCDCDCDCD; } - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, src, saved); } @@ -781,9 +781,9 @@ static void atom_op_mul(atom_exec_context *ctx, in { uint8_t attr = U8((*ptr)++); uint32_t dst, src; - SDEBUG(" src1: "); + ATOM_SDEBUG_PRINT(" src1: "); dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); - SDEBUG(" src2: "); + ATOM_SDEBUG_PRINT(" src2: "); src = atom_get_src(ctx, attr, ptr); ctx->ctx->divmul[0] = dst * src; } @@ -798,12 +798,12 @@ static void atom_op_or(atom_exec_context *ctx, int uint8_t attr = U8((*ptr)++); uint32_t dst, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst |= src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -810,7 +810,7 @@ static void atom_op_or(atom_exec_context *ctx, int static void atom_op_postcard(atom_exec_context *ctx, int *ptr, int arg) { uint8_t val = U8((*ptr)++); - SDEBUG("POST card output: 0x%02X\n", val); + ATOM_SDEBUG_PRINT("POST card output: 0x%02X\n", val); } static void atom_op_repeat(atom_exec_context *ctx, int *ptr, int arg) @@ -832,7 +832,7 @@ static void atom_op_setdatablock(atom_exec_context { int idx = U8(*ptr); (*ptr)++; - SDEBUG(" block: %d\n", idx); + ATOM_SDEBUG_PRINT(" block: %d\n", idx); if (!idx) ctx->ctx->data_block = 0; else if (idx == 255) @@ -839,13 +839,13 @@ static void atom_op_setdatablock(atom_exec_context ctx->ctx->data_block = ctx->start; else ctx->ctx->data_block = U16(ctx->ctx->data_table + 4 + 2 * idx); - SDEBUG(" base: 0x%04X\n", ctx->ctx->data_block); + ATOM_SDEBUG_PRINT(" base: 0x%04X\n", ctx->ctx->data_block); } static void atom_op_setfbbase(atom_exec_context *ctx, int *ptr, int arg) { uint8_t attr = U8((*ptr)++); - SDEBUG(" fb_base: "); + ATOM_SDEBUG_PRINT(" fb_base: "); ctx->ctx->fb_base = atom_get_src(ctx, attr, ptr); } @@ -856,9 +856,9 @@ static void atom_op_setport(atom_exec_context *ctx case ATOM_PORT_ATI: port = U16(*ptr); if (port < ATOM_IO_NAMES_CNT) - SDEBUG(" port: %d (%s)\n", port, atom_io_names[port]); + ATOM_SDEBUG_PRINT(" port: %d (%s)\n", port, atom_io_names[port]); else - SDEBUG(" port: %d\n", port); + ATOM_SDEBUG_PRINT(" port: %d\n", port); if (!port) ctx->ctx->io_mode = ATOM_IO_MM; else @@ -880,7 +880,7 @@ static void atom_op_setregblock(atom_exec_context { ctx->ctx->reg_block = U16(*ptr); (*ptr) += 2; - SDEBUG(" base: 0x%04X\n", ctx->ctx->reg_block); + ATOM_SDEBUG_PRINT(" base: 0x%04X\n", ctx->ctx->reg_block); } static void atom_op_shift_left(atom_exec_context *ctx, int *ptr, int arg) @@ -890,12 +890,12 @@ static void atom_op_shift_left(atom_exec_context * int dptr = *ptr; attr &= 0x38; attr |= atom_def_dst[attr >> 3] << 6; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); shift = atom_get_src_direct(ctx, ATOM_SRC_BYTE0, ptr); - SDEBUG(" shift: %d\n", shift); + ATOM_SDEBUG_PRINT(" shift: %d\n", shift); dst <<= shift; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -906,12 +906,12 @@ static void atom_op_shift_right(atom_exec_context int dptr = *ptr; attr &= 0x38; attr |= atom_def_dst[attr >> 3] << 6; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); shift = atom_get_src_direct(ctx, ATOM_SRC_BYTE0, ptr); - SDEBUG(" shift: %d\n", shift); + ATOM_SDEBUG_PRINT(" shift: %d\n", shift); dst >>= shift; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -921,16 +921,16 @@ static void atom_op_shl(atom_exec_context *ctx, in uint32_t saved, dst; int dptr = *ptr; uint32_t dst_align = atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & 3]; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); /* op needs to full dst value */ dst = saved; shift = atom_get_src(ctx, attr, ptr); - SDEBUG(" shift: %d\n", shift); + ATOM_SDEBUG_PRINT(" shift: %d\n", shift); dst <<= shift; dst &= atom_arg_mask[dst_align]; dst >>= atom_arg_shift[dst_align]; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -940,16 +940,16 @@ static void atom_op_shr(atom_exec_context *ctx, in uint32_t saved, dst; int dptr = *ptr; uint32_t dst_align = atom_dst_to_src[(attr >> 3) & 7][(attr >> 6) & 3]; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); /* op needs to full dst value */ dst = saved; shift = atom_get_src(ctx, attr, ptr); - SDEBUG(" shift: %d\n", shift); + ATOM_SDEBUG_PRINT(" shift: %d\n", shift); dst >>= shift; dst &= atom_arg_mask[dst_align]; dst >>= atom_arg_shift[dst_align]; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -958,12 +958,12 @@ static void atom_op_sub(atom_exec_context *ctx, in uint8_t attr = U8((*ptr)++); uint32_t dst, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst -= src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -971,18 +971,18 @@ static void atom_op_switch(atom_exec_context *ctx, { uint8_t attr = U8((*ptr)++); uint32_t src, val, target; - SDEBUG(" switch: "); + ATOM_SDEBUG_PRINT(" switch: "); src = atom_get_src(ctx, attr, ptr); while (U16(*ptr) != ATOM_CASE_END) if (U8(*ptr) == ATOM_CASE_MAGIC) { (*ptr)++; - SDEBUG(" case: "); + ATOM_SDEBUG_PRINT(" case: "); val = atom_get_src(ctx, (attr & 0x38) | ATOM_ARG_IMM, ptr); target = U16(*ptr); if (val == src) { - SDEBUG(" target: %04X\n", target); + ATOM_SDEBUG_PRINT(" target: %04X\n", target); *ptr = ctx->start + target; return; } @@ -998,12 +998,12 @@ static void atom_op_test(atom_exec_context *ctx, i { uint8_t attr = U8((*ptr)++); uint32_t dst, src; - SDEBUG(" src1: "); + ATOM_SDEBUG_PRINT(" src1: "); dst = atom_get_dst(ctx, arg, attr, ptr, NULL, 1); - SDEBUG(" src2: "); + ATOM_SDEBUG_PRINT(" src2: "); src = atom_get_src(ctx, attr, ptr); ctx->ctx->cs_equal = ((dst & src) == 0); - SDEBUG(" result: %s\n", ctx->ctx->cs_equal ? "EQ" : "NE"); + ATOM_SDEBUG_PRINT(" result: %s\n", ctx->ctx->cs_equal ? "EQ" : "NE"); } static void atom_op_xor(atom_exec_context *ctx, int *ptr, int arg) @@ -1011,12 +1011,12 @@ static void atom_op_xor(atom_exec_context *ctx, in uint8_t attr = U8((*ptr)++); uint32_t dst, src, saved; int dptr = *ptr; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); dst = atom_get_dst(ctx, arg, attr, ptr, &saved, 1); - SDEBUG(" src: "); + ATOM_SDEBUG_PRINT(" src: "); src = atom_get_src(ctx, attr, ptr); dst ^= src; - SDEBUG(" dst: "); + ATOM_SDEBUG_PRINT(" dst: "); atom_put_dst(ctx, arg, attr, &dptr, dst, saved); } @@ -1169,7 +1169,7 @@ static int atom_execute_table_locked(struct atom_c ps = CU8(base + ATOM_CT_PS_PTR) & ATOM_CT_PS_MASK; ptr = base + ATOM_CT_CODE_PTR; - SDEBUG(">> execute %04X (len %d, WS %d, PS %d)\n", base, len, ws, ps); + ATOM_SDEBUG_PRINT(">> execute %04X (len %d, WS %d, PS %d)\n", base, len, ws, ps); ectx.ctx = ctx; ectx.ps_shift = ps / 4; @@ -1186,9 +1186,9 @@ static int atom_execute_table_locked(struct atom_c while (1) { op = CU8(ptr++); if (op < ATOM_OP_NAMES_CNT) - SDEBUG("%s @ 0x%04X\n", atom_op_names[op], ptr - 1); + ATOM_SDEBUG_PRINT("%s @ 0x%04X\n", atom_op_names[op], ptr - 1); else - SDEBUG("[%d] @ 0x%04X\n", op, ptr - 1); + ATOM_SDEBUG_PRINT("[%d] @ 0x%04X\n", op, ptr - 1); if (ectx.abort) { DRM_ERROR("atombios stuck executing %04X (len %d, WS %d, PS %d) @ 0x%04X\n", base, len, ws, ps, ptr - 1); @@ -1206,7 +1206,7 @@ static int atom_execute_table_locked(struct atom_c break; } debug_depth--; - SDEBUG("<<\n"); + ATOM_SDEBUG_PRINT("<<\n"); free: if (ws) Index: sys/dev/drm2/radeon/atombios_dp.c =================================================================== --- sys/dev/drm2/radeon/atombios_dp.c (revision 254885) +++ sys/dev/drm2/radeon/atombios_dp.c (working copy) @@ -272,7 +272,7 @@ int radeon_dp_i2c_aux_ch(device_t dev, int mode, u case AUX_I2C_REPLY_ACK: if (mode == MODE_I2C_READ) *read_byte = reply[0]; - return ret; + return (0); /* Return ret on Linux. */ case AUX_I2C_REPLY_NACK: DRM_DEBUG_KMS("aux_i2c nack\n"); return -EREMOTEIO; Index: sys/dev/drm2/radeon/r100.c =================================================================== --- sys/dev/drm2/radeon/r100.c (revision 254885) +++ sys/dev/drm2/radeon/r100.c (working copy) @@ -2647,7 +2647,7 @@ int r100_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); /* save PCI state */ - pci_save_state(rdev->dev); + pci_save_state(device_get_parent(rdev->dev)); /* disable bus mastering */ r100_bm_disable(rdev); WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_SE(1) | @@ -2669,7 +2669,7 @@ int r100_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ - pci_restore_state(rdev->dev); + pci_restore_state(device_get_parent(rdev->dev)); r100_enable_bm(rdev); /* Check if GPU is idle */ if (G_000E40_SE_BUSY(status) || G_000E40_RE_BUSY(status) || Index: sys/dev/drm2/radeon/r300.c =================================================================== --- sys/dev/drm2/radeon/r300.c (revision 254885) +++ sys/dev/drm2/radeon/r300.c (working copy) @@ -401,7 +401,7 @@ int r300_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); /* save PCI state */ - pci_save_state(rdev->dev); + pci_save_state(device_get_parent(rdev->dev)); /* disable bus mastering */ r100_bm_disable(rdev); WREG32(R_0000F0_RBBM_SOFT_RESET, S_0000F0_SOFT_RESET_VAP(1) | @@ -425,7 +425,7 @@ int r300_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ - pci_restore_state(rdev->dev); + pci_restore_state(device_get_parent(rdev->dev)); r100_enable_bm(rdev); /* Check if GPU is idle */ if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) { Index: sys/dev/drm2/radeon/r600.c =================================================================== --- sys/dev/drm2/radeon/r600.c (revision 254885) +++ sys/dev/drm2/radeon/r600.c (working copy) @@ -3012,6 +3012,12 @@ int r600_init(struct radeon_device *rdev) rdev->accel_working = false; } + /* Don't start up if the ucode is missing. */ + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + DRM_ERROR("radeon: ucode required for R600+.\n"); + return -EINVAL; + } + return 0; } Index: sys/dev/drm2/radeon/radeon_agp.c =================================================================== --- sys/dev/drm2/radeon/radeon_agp.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_agp.c (working copy) @@ -153,11 +153,11 @@ int radeon_agp_init(struct radeon_device *rdev) return ret; } - if (rdev->ddev->agp->info.ai_aperture_size < 32) { + if ((rdev->ddev->agp->info.ai_aperture_size >> 20) < 32) { drm_agp_release(rdev->ddev); dev_warn(rdev->dev, "AGP aperture too small (%zuM) " "need at least 32M, disabling AGP\n", - rdev->ddev->agp->info.ai_aperture_size); + rdev->ddev->agp->info.ai_aperture_size >> 20); return -EINVAL; } @@ -246,7 +246,7 @@ int radeon_agp_init(struct radeon_device *rdev) } rdev->mc.agp_base = rdev->ddev->agp->info.ai_aperture_base; - rdev->mc.gtt_size = rdev->ddev->agp->info.ai_aperture_size << 20; + rdev->mc.gtt_size = rdev->ddev->agp->info.ai_aperture_size; rdev->mc.gtt_start = rdev->mc.agp_base; rdev->mc.gtt_end = rdev->mc.gtt_start + rdev->mc.gtt_size - 1; dev_info(rdev->dev, "GTT: %juM 0x%08jX - 0x%08jX\n", Index: sys/dev/drm2/radeon/radeon_bios.c =================================================================== --- sys/dev/drm2/radeon/radeon_bios.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_bios.c (working copy) @@ -100,14 +100,16 @@ static bool igp_read_bios_from_vram(struct radeon_ static bool radeon_read_bios(struct radeon_device *rdev) { + device_t vga_dev; uint8_t __iomem *bios; size_t size; DRM_INFO("%s: ===> Try PCI Expansion ROM...\n", __func__); + vga_dev = device_get_parent(rdev->dev); rdev->bios = NULL; /* XXX: some cards may return 0 for rom size? ddx has a workaround */ - bios = vga_pci_map_bios(rdev->dev, &size); + bios = vga_pci_map_bios(vga_dev, &size); if (!bios) { return false; } @@ -120,11 +122,12 @@ static bool radeon_read_bios(struct radeon_device DRM_INFO("%s: Incorrect BIOS signature: 0x%02X%02X\n", __func__, bios[0], bios[1]); } - vga_pci_unmap_bios(rdev->dev, bios); + vga_pci_unmap_bios(vga_dev, bios); + return false; } rdev->bios = malloc(size, DRM_MEM_DRIVER, M_WAITOK); memcpy(rdev->bios, bios, size); - vga_pci_unmap_bios(rdev->dev, bios); + vga_pci_unmap_bios(vga_dev, bios); return true; } Index: sys/dev/drm2/radeon/radeon_device.c =================================================================== --- sys/dev/drm2/radeon/radeon_device.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_device.c (working copy) @@ -1316,7 +1316,7 @@ int radeon_suspend_kms(struct drm_device *dev) radeon_agp_suspend(rdev); - pci_save_state(dev->device); + pci_save_state(device_get_parent(rdev->dev)); #ifdef DUMBBELL_WIP if (state.event == PM_EVENT_SUSPEND) { /* Shut down the device */ @@ -1356,7 +1356,7 @@ int radeon_resume_kms(struct drm_device *dev) console_lock(); #endif /* DUMBBELL_WIP */ pci_set_powerstate(dev->device, PCI_POWERSTATE_D0); - pci_restore_state(dev->device); + pci_restore_state(device_get_parent(rdev->dev)); #ifdef DUMBBELL_WIP if (pci_enable_device(dev->pdev)) { console_unlock(); Index: sys/dev/drm2/radeon/radeon_drv.c =================================================================== --- sys/dev/drm2/radeon/radeon_drv.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_drv.c (working copy) @@ -338,6 +338,12 @@ static const struct file_operations radeon_driver_ }; #endif /* DUMBBELL_WIP */ +static int radeon_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx, + struct sysctl_oid *top) +{ + return drm_add_busid_modesetting(dev, ctx, top); +} + static struct drm_driver_info kms_driver = { .driver_features = DRIVER_USE_AGP | DRIVER_USE_MTRR | DRIVER_PCI_DMA | DRIVER_SG | @@ -367,6 +373,7 @@ static struct drm_driver_info kms_driver = { .irq_postinstall = radeon_driver_irq_postinstall_kms, .irq_uninstall = radeon_driver_irq_uninstall_kms, .irq_handler = radeon_driver_irq_handler_kms, + .sysctl_init = radeon_sysctl_init, .ioctls = radeon_ioctls_kms, .gem_init_object = radeon_gem_object_init, .gem_free_object = radeon_gem_object_free, Index: sys/dev/drm2/radeon/radeon_fence.c =================================================================== --- sys/dev/drm2/radeon/radeon_fence.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_fence.c (working copy) @@ -321,6 +321,8 @@ static int radeon_fence_wait_seq(struct radeon_dev &rdev->fence_queue_mtx, timeout); } + if (r == EINTR) + r = ERESTARTSYS; if (r != 0) { if (r == EWOULDBLOCK) { signaled = @@ -334,7 +336,7 @@ static int radeon_fence_wait_seq(struct radeon_dev mtx_unlock(&rdev->fence_queue_mtx); } radeon_irq_kms_sw_irq_put(rdev, ring); - if (unlikely(r == EINTR || r == ERESTART)) { + if (unlikely(r == ERESTARTSYS)) { return -r; } CTR2(KTR_DRM, "radeon fence: wait end (ring=%d, seq=%d)", @@ -514,6 +516,8 @@ static int radeon_fence_wait_any_seq(struct radeon &rdev->fence_queue_mtx, timeout); } + if (r == EINTR) + r = ERESTARTSYS; if (r != 0) { if (r == EWOULDBLOCK) { signaled = @@ -531,7 +535,7 @@ static int radeon_fence_wait_any_seq(struct radeon radeon_irq_kms_sw_irq_put(rdev, i); } } - if (unlikely(r == EINTR || r == ERESTART)) { + if (unlikely(r == ERESTARTSYS)) { return -r; } CTR2(KTR_DRM, "radeon fence: wait end (ring=%d, target_seq=%d)", Index: sys/dev/drm2/radeon/radeon_gem.c =================================================================== --- sys/dev/drm2/radeon/radeon_gem.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_gem.c (working copy) @@ -567,6 +567,7 @@ int radeon_mode_dumb_create(struct drm_file *file_ if (r) return -ENOMEM; + handle = 0; r = drm_gem_handle_create(file_priv, gobj, &handle); /* drop reference from allocate - handle holds it now */ drm_gem_object_unreference_unlocked(gobj); Index: sys/dev/drm2/radeon/radeon_sa.c =================================================================== --- sys/dev/drm2/radeon/radeon_sa.c (revision 254885) +++ sys/dev/drm2/radeon/radeon_sa.c (working copy) @@ -363,6 +363,8 @@ int radeon_sa_bo_new(struct radeon_device *rdev, while (!radeon_sa_event(sa_manager, size, align)) { r = -cv_wait_sig(&sa_manager->wq, &sa_manager->wq_lock); + if (r == -EINTR) + r = -ERESTARTSYS; if (r != 0) break; } Index: sys/dev/drm2/radeon/rs600.c =================================================================== --- sys/dev/drm2/radeon/rs600.c (revision 254885) +++ sys/dev/drm2/radeon/rs600.c (working copy) @@ -373,7 +373,7 @@ int rs600_asic_reset(struct radeon_device *rdev) WREG32(RADEON_CP_RB_RPTR_WR, 0); WREG32(RADEON_CP_RB_WPTR, 0); WREG32(RADEON_CP_RB_CNTL, tmp); - pci_save_state(rdev->dev); + pci_save_state(device_get_parent(rdev->dev)); /* disable bus mastering */ pci_disable_busmaster(rdev->dev); DRM_MDELAY(1); @@ -403,7 +403,7 @@ int rs600_asic_reset(struct radeon_device *rdev) status = RREG32(R_000E40_RBBM_STATUS); dev_info(rdev->dev, "(%s:%d) RBBM_STATUS=0x%08X\n", __func__, __LINE__, status); /* restore PCI & busmastering */ - pci_restore_state(rdev->dev); + pci_restore_state(device_get_parent(rdev->dev)); /* Check if GPU is idle */ if (G_000E40_GA_BUSY(status) || G_000E40_VAP_BUSY(status)) { dev_err(rdev->dev, "failed to reset GPU\n"); Index: sys/dev/drm2/radeon/rs690.c =================================================================== --- sys/dev/drm2/radeon/rs690.c (revision 254885) +++ sys/dev/drm2/radeon/rs690.c (working copy) @@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$"); #include +#include #include "radeon.h" #include "radeon_asic.h" #include "atom.h" Index: sys/dev/drm2/radeon/rv770.c =================================================================== --- sys/dev/drm2/radeon/rv770.c (revision 254885) +++ sys/dev/drm2/radeon/rv770.c (working copy) @@ -1186,6 +1186,12 @@ int rv770_init(struct radeon_device *rdev) rdev->accel_working = false; } + /* Don't start up if the ucode is missing. */ + if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) { + DRM_ERROR("radeon: ucode required for R600+.\n"); + return -EINVAL; + } + return 0; } Index: sys/dev/drm2/ttm/ttm_bo.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_bo.c (working copy) @@ -131,7 +131,7 @@ static void ttm_bo_release_list(struct ttm_buffer_ ttm_mem_global_free(bdev->glob->mem_glob, acc_size); } -int +static int ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible) { const char *wmsg; @@ -145,8 +145,10 @@ ttm_bo_wait_unreserved_locked(struct ttm_buffer_ob flags = 0; wmsg = "ttbowu"; } - while (!ttm_bo_is_reserved(bo)) { + while (ttm_bo_is_reserved(bo)) { ret = -msleep(bo, &bo->glob->lru_lock, flags, wmsg, 0); + if (ret == -EINTR) + ret = -ERESTARTSYS; if (ret != 0) break; } @@ -196,13 +198,13 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object * return put_count; } -int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, +int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, bool interruptible, bool no_wait, bool use_sequence, uint32_t sequence) { int ret; - while (unlikely(atomic_read(&bo->reserved) != 0)) { + while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) { /** * Deadlock avoidance for multi-bo reserving. */ @@ -224,12 +226,13 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object * return -EBUSY; ret = ttm_bo_wait_unreserved_locked(bo, interruptible); + if (unlikely(ret)) return ret; } - atomic_set(&bo->reserved, 1); if (use_sequence) { + bool wake_up = false; /** * Wake up waiters that may need to recheck for deadlock, * if we decreased the sequence number. @@ -236,10 +239,22 @@ int ttm_bo_del_from_lru(struct ttm_buffer_object * */ if (unlikely((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid)) - wakeup(bo); + wake_up = true; + /* + * In the worst case with memory ordering these values can be + * seen in the wrong order. However since we call wake_up_all + * in that case, this will hopefully not pose a problem, + * and the worst case would only cause someone to accidentally + * hit -EAGAIN in ttm_bo_reserve when they see old value of + * val_seq. However this would only happen if seq_valid was + * written before val_seq was, and just means some slightly + * increased cpu usage + */ bo->val_seq = sequence; bo->seq_valid = true; + if (wake_up) + wakeup(bo); } else { bo->seq_valid = false; } @@ -268,15 +283,67 @@ int ttm_bo_reserve(struct ttm_buffer_object *bo, int put_count = 0; int ret; - mtx_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, interruptible, no_wait, use_sequence, - sequence); - if (likely(ret == 0)) + mtx_lock(&bo->glob->lru_lock); + ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence, + sequence); + if (likely(ret == 0)) { put_count = ttm_bo_del_from_lru(bo); - mtx_unlock(&glob->lru_lock); + mtx_unlock(&glob->lru_lock); + ttm_bo_list_ref_sub(bo, put_count, true); + } else + mtx_unlock(&bo->glob->lru_lock); - ttm_bo_list_ref_sub(bo, put_count, true); + return ret; +} +int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, + bool interruptible, uint32_t sequence) +{ + bool wake_up = false; + int ret; + + while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) { + if (bo->seq_valid && sequence == bo->val_seq) { + DRM_ERROR( + "%s: bo->seq_valid && sequence == bo->val_seq", + __func__); + } + + ret = ttm_bo_wait_unreserved_locked(bo, interruptible); + + if (unlikely(ret)) + return ret; + } + + if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid) + wake_up = true; + + /** + * Wake up waiters that may need to recheck for deadlock, + * if we decreased the sequence number. + */ + bo->val_seq = sequence; + bo->seq_valid = true; + if (wake_up) + wakeup(bo); + + return 0; +} + +int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, + bool interruptible, uint32_t sequence) +{ + struct ttm_bo_global *glob = bo->glob; + int put_count, ret; + + mtx_lock(&glob->lru_lock); + ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence); + if (likely(!ret)) { + put_count = ttm_bo_del_from_lru(bo); + mtx_unlock(&glob->lru_lock); + ttm_bo_list_ref_sub(bo, put_count, true); + } else + mtx_unlock(&glob->lru_lock); return ret; } @@ -412,6 +479,7 @@ static int ttm_bo_handle_move_mem(struct ttm_buffe bo->mem = tmp_mem; bdev->driver->move_notify(bo, mem); bo->mem = *mem; + *mem = tmp_mem; } goto out_err; @@ -488,7 +556,7 @@ static void ttm_bo_cleanup_refs_or_queue(struct tt int ret; mtx_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); mtx_lock(&bdev->fence_lock); (void) ttm_bo_wait(bo, false, false, true); @@ -580,7 +648,7 @@ static int ttm_bo_cleanup_refs_and_unlock(struct t return ret; mtx_lock(&glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); /* * We raced, and lost, someone else holds the reservation now, @@ -644,7 +712,12 @@ static int ttm_bo_delayed_delete(struct ttm_bo_dev refcount_acquire(&nentry->list_kref); } - ret = ttm_bo_reserve_locked(entry, false, !remove_all, false, 0); + ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); + if (remove_all && ret) { + ret = ttm_bo_reserve_nolru(entry, false, false, + false, 0); + } + if (!ret) ret = ttm_bo_cleanup_refs_and_unlock(entry, false, !remove_all); @@ -796,7 +869,7 @@ static int ttm_mem_evict_first(struct ttm_bo_devic mtx_lock(&glob->lru_lock); list_for_each_entry(bo, &man->lru, lru) { - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); if (!ret) break; } @@ -1400,7 +1473,6 @@ static void ttm_bo_global_kobj_release(struct ttm_ ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); vm_page_free(glob->dummy_read_page); - free(glob, M_DRM_GLOBAL); } void ttm_bo_global_release(struct drm_global_reference *ref) @@ -1566,13 +1638,8 @@ bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) { - struct ttm_bo_device *bdev = bo->bdev; - /* off_t offset = (off_t)bo->addr_space_offset;XXXKIB */ - /* off_t holelen = ((off_t)bo->mem.num_pages) << PAGE_SHIFT;XXXKIB */ - if (!bdev->dev_mapping) - return; - /* unmap_mapping_range(bdev->dev_mapping, offset, holelen, 1); XXXKIB */ + ttm_bo_release_mmap(bo); ttm_mem_io_free_vm(bo); } @@ -1658,7 +1725,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (driver->sync_obj_signaled(bo->sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); + atomic_clear_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); driver->sync_obj_unref(&tmp_obj); mtx_lock(&bdev->fence_lock); @@ -1681,8 +1749,8 @@ int ttm_bo_wait(struct ttm_buffer_object *bo, if (likely(bo->sync_obj == sync_obj)) { void *tmp_obj = bo->sync_obj; bo->sync_obj = NULL; - clear_bit(TTM_BO_PRIV_FLAG_MOVING, - &bo->priv_flags); + atomic_clear_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); driver->sync_obj_unref(&sync_obj); driver->sync_obj_unref(&tmp_obj); @@ -1738,7 +1806,7 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *s mtx_lock(&glob->lru_lock); list_for_each_entry(bo, &glob->swap_lru, swap) { - ret = ttm_bo_reserve_locked(bo, false, true, false, 0); + ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); if (!ret) break; } Index: sys/dev/drm2/ttm/ttm_bo_driver.h =================================================================== --- sys/dev/drm2/ttm/ttm_bo_driver.h (revision 247835) +++ sys/dev/drm2/ttm/ttm_bo_driver.h (working copy) @@ -791,16 +791,7 @@ extern void ttm_mem_io_unlock(struct ttm_mem_type_ * to make room for a buffer already reserved. (Buffers are reserved before * they are evicted). The following algorithm prevents such deadlocks from * occurring: - * 1) Buffers are reserved with the lru spinlock held. Upon successful - * reservation they are removed from the lru list. This stops a reserved buffer - * from being evicted. However the lru spinlock is released between the time - * a buffer is selected for eviction and the time it is reserved. - * Therefore a check is made when a buffer is reserved for eviction, that it - * is still the first buffer in the lru list, before it is removed from the - * list. @check_lru == 1 forces this check. If it fails, the function returns - * -EINVAL, and the caller should then choose a new buffer to evict and repeat - * the procedure. - * 2) Processes attempting to reserve multiple buffers other than for eviction, + * Processes attempting to reserve multiple buffers other than for eviction, * (typically execbuf), should first obtain a unique 32-bit * validation sequence number, * and call this function with @use_sequence == 1 and @sequence == the unique @@ -831,10 +822,40 @@ extern int ttm_bo_reserve(struct ttm_buffer_object bool interruptible, bool no_wait, bool use_sequence, uint32_t sequence); +/** + * ttm_bo_reserve_slowpath_nolru: + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @sequence: Set (@bo)->sequence to this value after lock + * + * This is called after ttm_bo_reserve returns -EAGAIN and we backed off + * from all our other reservations. Because there are no other reservations + * held by us, this function cannot deadlock any more. + * + * Will not remove reserved buffers from the lru lists. + * Otherwise identical to ttm_bo_reserve_slowpath. + */ +extern int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, + bool interruptible, + uint32_t sequence); + /** - * ttm_bo_reserve_locked: + * ttm_bo_reserve_slowpath: + * @bo: A pointer to a struct ttm_buffer_object. + * @interruptible: Sleep interruptible if waiting. + * @sequence: Set (@bo)->sequence to this value after lock * + * This is called after ttm_bo_reserve returns -EAGAIN and we backed off + * from all our other reservations. Because there are no other reservations + * held by us, this function cannot deadlock any more. + */ +extern int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, + bool interruptible, uint32_t sequence); + +/** + * ttm_bo_reserve_nolru: + * * @bo: A pointer to a struct ttm_buffer_object. * @interruptible: Sleep interruptible if waiting. * @no_wait: Don't sleep while trying to reserve, rather return -EBUSY. @@ -841,9 +862,7 @@ extern int ttm_bo_reserve(struct ttm_buffer_object * @use_sequence: If @bo is already reserved, Only sleep waiting for * it to become unreserved if @sequence < (@bo)->sequence. * - * Must be called with struct ttm_bo_global::lru_lock held, - * and will not remove reserved buffers from the lru lists. - * The function may release the LRU spinlock if it needs to sleep. + * Will not remove reserved buffers from the lru lists. * Otherwise identical to ttm_bo_reserve. * * Returns: @@ -856,7 +875,7 @@ extern int ttm_bo_reserve(struct ttm_buffer_object * -EDEADLK: Bo already reserved using @sequence. This error code will only * be returned if @use_sequence is set to true. */ -extern int ttm_bo_reserve_locked(struct ttm_buffer_object *bo, +extern int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, bool interruptible, bool no_wait, bool use_sequence, uint32_t sequence); @@ -880,18 +899,6 @@ extern void ttm_bo_unreserve(struct ttm_buffer_obj */ extern void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo); -/** - * ttm_bo_wait_unreserved - * - * @bo: A pointer to a struct ttm_buffer_object. - * - * Wait for a struct ttm_buffer_object to become unreserved. - * This is typically used in the execbuf code to relax cpu-usage when - * a potential deadlock condition backoff. - */ -extern int ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, - bool interruptible); - /* * ttm_bo_util.c */ Index: sys/dev/drm2/ttm/ttm_bo_util.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo_util.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_bo_util.c (working copy) @@ -321,8 +321,12 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *b if (ttm->state == tt_unpopulated) { ret = ttm->bdev->driver->ttm_tt_populate(ttm); - if (ret) + if (ret) { + /* if we fail here don't nuke the mm node + * as the bo still owns it */ + old_copy.mm_node = NULL; goto out1; + } } add = 0; @@ -346,8 +350,11 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *b prot); } else ret = ttm_copy_io_page(new_iomap, old_iomap, page); - if (ret) + if (ret) { + /* failing here, means keep old copy as-is */ + old_copy.mm_node = NULL; goto out1; + } } mb(); out2: @@ -391,14 +398,15 @@ static void ttm_transfered_destroy(struct ttm_buff * !0: Failure. */ -static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, - struct ttm_buffer_object **new_obj) +static int +ttm_buffer_object_transfer(struct ttm_buffer_object *bo, + struct ttm_buffer_object **new_obj) { struct ttm_buffer_object *fbo; struct ttm_bo_device *bdev = bo->bdev; struct ttm_bo_driver *driver = bdev->driver; - fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_ZERO | M_WAITOK); + fbo = malloc(sizeof(*fbo), M_TTM_TRANSF_OBJ, M_WAITOK); *fbo = *bo; /** @@ -413,7 +421,12 @@ static void ttm_transfered_destroy(struct ttm_buff fbo->vm_node = NULL; atomic_set(&fbo->cpu_writers, 0); - fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + mtx_lock(&bdev->fence_lock); + if (bo->sync_obj) + fbo->sync_obj = driver->sync_obj_ref(bo->sync_obj); + else + fbo->sync_obj = NULL; + mtx_unlock(&bdev->fence_lock); refcount_init(&fbo->list_kref, 1); refcount_init(&fbo->kref, 1); fbo->destroy = &ttm_transfered_destroy; @@ -625,14 +638,13 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_ob * operation has completed. */ - set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); - - /* ttm_buffer_object_transfer accesses bo->sync_obj */ - ret = ttm_buffer_object_transfer(bo, &ghost_obj); + atomic_set_long(&bo->priv_flags, + 1UL << TTM_BO_PRIV_FLAG_MOVING); mtx_unlock(&bdev->fence_lock); if (tmp_obj) driver->sync_obj_unref(&tmp_obj); + ret = ttm_buffer_object_transfer(bo, &ghost_obj); if (ret) return ret; Index: sys/dev/drm2/ttm/ttm_bo_vm.c =================================================================== --- sys/dev/drm2/ttm/ttm_bo_vm.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_bo_vm.c (working copy) @@ -75,13 +75,16 @@ static struct ttm_buffer_object *ttm_bo_vm_lookup_ struct ttm_buffer_object *bo; struct ttm_buffer_object *best_bo = NULL; - RB_FOREACH(bo, ttm_bo_device_buffer_objects, &bdev->addr_space_rb) { + bo = RB_ROOT(&bdev->addr_space_rb); + while (bo != NULL) { cur_offset = bo->vm_node->start; if (page_start >= cur_offset) { best_bo = bo; if (page_start == cur_offset) break; - } + bo = RB_RIGHT(bo, vm_rb); + } else + bo = RB_LEFT(bo, vm_rb); } if (unlikely(best_bo == NULL)) @@ -102,7 +105,7 @@ ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t o struct ttm_buffer_object *bo = vm_obj->handle; struct ttm_bo_device *bdev = bo->bdev; struct ttm_tt *ttm = NULL; - vm_page_t m, oldm; + vm_page_t m, m1, oldm; int ret; int retval = VM_PAGER_OK; struct ttm_mem_type_manager *man = @@ -122,9 +125,7 @@ retry: m = NULL; reserve: - mtx_lock(&bo->glob->lru_lock); - ret = ttm_bo_reserve_locked(bo, false, false, false, 0); - mtx_unlock(&bo->glob->lru_lock); + ret = ttm_bo_reserve(bo, false, false, false, 0); if (unlikely(ret != 0)) { if (ret == -EBUSY) { kern_yield(0); @@ -154,8 +155,25 @@ reserve: */ mtx_lock(&bdev->fence_lock); - if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) { - ret = ttm_bo_wait(bo, false, true, false); + if ((atomic_load_acq_long(&bo->priv_flags) & + (1UL << TTM_BO_PRIV_FLAG_MOVING)) != 0) { + /* + * Here, the behavior differs between Linux and FreeBSD. + * + * On Linux, the wait is interruptible (3rd argument to + * ttm_bo_wait). There must be some mechanism to resume + * page fault handling, once the signal is processed. + * + * On FreeBSD, the wait is uninteruptible. This is not a + * problem as we can't end up with an unkillable process + * here, because the wait will eventually time out. + * + * An example of this situation is the Xorg process + * which uses SIGALRM internally. The signal could + * interrupt the wait, causing the page fault to fail + * and the process to receive SIGSEGV. + */ + ret = ttm_bo_wait(bo, false, false, false); mtx_unlock(&bdev->fence_lock); if (unlikely(ret != 0)) { retval = VM_PAGER_ERROR; @@ -222,9 +240,14 @@ reserve: } m->valid = VM_PAGE_BITS_ALL; *mres = m; - vm_page_lock(m); - vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); - vm_page_unlock(m); + m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); + if (m1 == NULL) { + vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); + } else { + KASSERT(m == m1, + ("inconsistent insert bo %p m %p m1 %p offset %jx", + bo, m, m1, (uintmax_t)offset)); + } vm_page_busy(m); if (oldm != NULL) { @@ -253,10 +276,21 @@ static int ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color) { - struct ttm_buffer_object *bo = handle; + /* + * On Linux, a reference to the buffer object is acquired here. + * The reason is that this function is not called when the + * mmap() is initialized, but only when a process forks for + * instance. Therefore on Linux, the reference on the bo is + * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's + * then released in ttm_bo_vm_close(). + * + * Here, this function is called during mmap() intialization. + * Thus, the reference acquired in ttm_bo_mmap_single() is + * sufficient. + */ + *color = 0; - (void)ttm_bo_reference(bo); return (0); } @@ -320,6 +354,32 @@ out_unref: return ret; } +void +ttm_bo_release_mmap(struct ttm_buffer_object *bo) +{ + vm_object_t vm_obj; + vm_page_t m; + int i; + + vm_obj = cdev_pager_lookup(bo); + if (vm_obj == NULL) + return; + + VM_OBJECT_LOCK(vm_obj); +retry: + for (i = 0; i < bo->num_pages; i++) { + m = vm_page_lookup(vm_obj, i); + if (m == NULL) + continue; + if (vm_page_sleep_if_busy(m, true, "ttm_unm")) + goto retry; + cdev_pager_free_page(vm_obj, m); + } + VM_OBJECT_UNLOCK(vm_obj); + + vm_object_deallocate(vm_obj); +} + #if 0 int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo) { Index: sys/dev/drm2/ttm/ttm_execbuf_util.c =================================================================== --- sys/dev/drm2/ttm/ttm_execbuf_util.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_execbuf_util.c (working copy) @@ -83,19 +83,6 @@ static void ttm_eu_list_ref_sub(struct list_head * } } -static int ttm_eu_wait_unreserved_locked(struct list_head *list, - struct ttm_buffer_object *bo) -{ - int ret; - - ttm_eu_del_from_lru_locked(list); - ret = ttm_bo_wait_unreserved_locked(bo, true); - if (unlikely(ret != 0)) - ttm_eu_backoff_reservation_locked(list); - return ret; -} - - void ttm_eu_backoff_reservation(struct list_head *list) { struct ttm_validate_buffer *entry; @@ -143,47 +130,62 @@ int ttm_eu_reserve_buffers(struct list_head *list) glob = entry->bo->glob; mtx_lock(&glob->lru_lock); -retry_locked: val_seq = entry->bo->bdev->val_seq++; +retry_locked: list_for_each_entry(entry, list, head) { struct ttm_buffer_object *bo = entry->bo; -retry_this_bo: - ret = ttm_bo_reserve_locked(bo, true, true, true, val_seq); + /* already slowpath reserved? */ + if (entry->reserved) + continue; + + ret = ttm_bo_reserve_nolru(bo, true, true, true, val_seq); switch (ret) { case 0: break; case -EBUSY: - ret = ttm_eu_wait_unreserved_locked(list, bo); - if (unlikely(ret != 0)) { - mtx_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; - } - goto retry_this_bo; + ttm_eu_del_from_lru_locked(list); + ret = ttm_bo_reserve_nolru(bo, true, false, + true, val_seq); + if (!ret) + break; + + if (unlikely(ret != -EAGAIN)) + goto err; + + /* fallthrough */ case -EAGAIN: ttm_eu_backoff_reservation_locked(list); + + /* + * temporarily increase sequence number every retry, + * to prevent us from seeing our old reservation + * sequence when someone else reserved the buffer, + * but hasn't updated the seq_valid/seqno members yet. + */ + val_seq = entry->bo->bdev->val_seq++; + ttm_eu_list_ref_sub(list); - ret = ttm_bo_wait_unreserved_locked(bo, true); + ret = ttm_bo_reserve_slowpath_nolru(bo, true, val_seq); if (unlikely(ret != 0)) { mtx_unlock(&glob->lru_lock); return ret; } + entry->reserved = true; + if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { + ret = -EBUSY; + goto err; + } goto retry_locked; default: - ttm_eu_backoff_reservation_locked(list); - mtx_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return ret; + goto err; } entry->reserved = true; if (unlikely(atomic_read(&bo->cpu_writers) > 0)) { - ttm_eu_backoff_reservation_locked(list); - mtx_unlock(&glob->lru_lock); - ttm_eu_list_ref_sub(list); - return -EBUSY; + ret = -EBUSY; + goto err; } } @@ -192,6 +194,12 @@ int ttm_eu_reserve_buffers(struct list_head *list) ttm_eu_list_ref_sub(list); return 0; + +err: + ttm_eu_backoff_reservation_locked(list); + mtx_unlock(&glob->lru_lock); + ttm_eu_list_ref_sub(list); + return ret; } void ttm_eu_fence_buffer_objects(struct list_head *list, void *sync_obj) Index: sys/dev/drm2/ttm/ttm_lock.c =================================================================== --- sys/dev/drm2/ttm/ttm_lock.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_lock.c (working copy) @@ -107,6 +107,8 @@ ttm_read_lock(struct ttm_lock *lock, bool interrup mtx_lock(&lock->lock); while (!__ttm_read_lock(lock)) { ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret == EINTR) + ret = ERESTARTSYS; if (ret != 0) break; } @@ -151,6 +153,8 @@ int ttm_read_trylock(struct ttm_lock *lock, bool i mtx_lock(&lock->lock); while (!__ttm_read_trylock(lock, &locked)) { ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret == EINTR) + ret = ERESTARTSYS; if (ret != 0) break; } @@ -204,6 +208,8 @@ ttm_write_lock(struct ttm_lock *lock, bool interru /* XXXKIB: linux uses __ttm_read_lock for uninterruptible sleeps */ while (!__ttm_write_lock(lock)) { ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret == EINTR) + ret = ERESTARTSYS; if (interruptible && ret != 0) { lock->flags &= ~TTM_WRITE_LOCK_PENDING; wakeup(lock); @@ -280,6 +286,8 @@ int ttm_vt_lock(struct ttm_lock *lock, mtx_lock(&lock->lock); while (!__ttm_vt_lock(lock)) { ret = msleep(lock, &lock->lock, flags, wmsg, 0); + if (ret == EINTR) + ret = ERESTARTSYS; if (interruptible && ret != 0) { lock->flags &= ~TTM_VT_LOCK_PENDING; wakeup(lock); Index: sys/dev/drm2/ttm/ttm_lock.h =================================================================== --- sys/dev/drm2/ttm/ttm_lock.h (revision 247835) +++ sys/dev/drm2/ttm/ttm_lock.h (working copy) @@ -125,27 +125,6 @@ extern int ttm_read_lock(struct ttm_lock *lock, bo extern int ttm_read_trylock(struct ttm_lock *lock, bool interruptible); /** - * ttm_write_unlock - * - * @lock: Pointer to a struct ttm_lock - * - * Releases a write lock. - */ -extern void ttm_write_unlock(struct ttm_lock *lock); - -/** - * ttm_write_lock - * - * @lock: Pointer to a struct ttm_lock - * @interruptible: Interruptible sleeping while waiting for a lock. - * - * Takes the lock in write mode. - * Returns: - * -ERESTARTSYS If interrupted by a signal and interruptible is true. - */ -extern int ttm_write_lock(struct ttm_lock *lock, bool interruptible); - -/** * ttm_lock_downgrade * * @lock: Pointer to a struct ttm_lock Index: sys/dev/drm2/ttm/ttm_memory.c =================================================================== --- sys/dev/drm2/ttm/ttm_memory.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_memory.c (working copy) @@ -51,7 +51,7 @@ MALLOC_DEFINE(M_TTM_ZONE, "ttm_zone", "TTM Zone"); static void ttm_mem_zone_kobj_release(struct ttm_mem_zone *zone) { - printf("pTTM] Zone %7s: Used memory at exit: %llu kiB\n", + printf("[TTM] Zone %7s: Used memory at exit: %llu kiB\n", zone->name, (unsigned long long)zone->used_mem >> 10); free(zone, M_TTM_ZONE); } @@ -125,8 +125,6 @@ static ssize_t ttm_mem_zone_store(struct ttm_mem_z static void ttm_mem_global_kobj_release(struct ttm_mem_global *glob) { - - free(glob, M_TTM_ZONE); } static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob, Index: sys/dev/drm2/ttm/ttm_page_alloc.c =================================================================== --- sys/dev/drm2/ttm/ttm_page_alloc.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_page_alloc.c (working copy) @@ -49,8 +49,6 @@ __FBSDID("$FreeBSD$"); #include #endif -#define VM_ALLOC_DMA32 VM_ALLOC_RESERVED1 - #define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(vm_page_t)) #define SMALL_ALLOCATION 16 #define FREE_ALL_PAGES (~0U) @@ -113,16 +111,22 @@ struct ttm_pool_manager { struct ttm_pool_opts options; union { - struct ttm_page_pool pools[NUM_POOLS]; - struct { - struct ttm_page_pool wc_pool; - struct ttm_page_pool uc_pool; - struct ttm_page_pool wc_pool_dma32; - struct ttm_page_pool uc_pool_dma32; - } ; - }; + struct ttm_page_pool u_pools[NUM_POOLS]; + struct _utag { + struct ttm_page_pool u_wc_pool; + struct ttm_page_pool u_uc_pool; + struct ttm_page_pool u_wc_pool_dma32; + struct ttm_page_pool u_uc_pool_dma32; + } _ut; + } _u; }; +#define pools _u.u_pools +#define wc_pool _u._ut.u_wc_pool +#define uc_pool _u._ut.u_uc_pool +#define wc_pool_dma32 _u._ut.u_wc_pool_dma32 +#define uc_pool_dma32 _u._ut.u_uc_pool_dma32 + MALLOC_DEFINE(M_TTM_POOLMGR, "ttm_poolmgr", "TTM Pool Manager"); static void @@ -314,6 +318,7 @@ static int ttm_page_pool_free(struct ttm_page_pool vm_page_t *pages_to_free; unsigned freed_pages = 0, npages_to_free = nr_free; + unsigned i; if (NUM_PAGES_TO_ALLOC < nr_free) npages_to_free = NUM_PAGES_TO_ALLOC; @@ -332,7 +337,8 @@ restart: /* We can only remove NUM_PAGES_TO_ALLOC at a time. */ if (freed_pages >= NUM_PAGES_TO_ALLOC) { /* remove range of pages from the pool */ - TAILQ_REMOVE(&pool->list, p, pageq); + for (i = 0; i < freed_pages; i++) + TAILQ_REMOVE(&pool->list, pages_to_free[i], pageq); ttm_pool_update_free_locked(pool, freed_pages); /** @@ -367,7 +373,8 @@ restart: /* remove range of pages from the pool */ if (freed_pages) { - TAILQ_REMOVE(&pool->list, p, pageq); + for (i = 0; i < freed_pages; i++) + TAILQ_REMOVE(&pool->list, pages_to_free[i], pageq); ttm_pool_update_free_locked(pool, freed_pages); nr_free -= freed_pages; Index: sys/dev/drm2/ttm/ttm_tt.c =================================================================== --- sys/dev/drm2/ttm/ttm_tt.c (revision 247835) +++ sys/dev/drm2/ttm/ttm_tt.c (working copy) @@ -288,8 +288,10 @@ int ttm_tt_swapin(struct ttm_tt *ttm) VM_OBJECT_LOCK(obj); vm_object_pip_add(obj, 1); for (i = 0; i < ttm->num_pages; ++i) { - from_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); + from_page = vm_page_grab(obj, i, VM_ALLOC_NOBUSY | + VM_ALLOC_RETRY); if (from_page->valid != VM_PAGE_BITS_ALL) { + vm_page_busy(from_page); if (vm_pager_has_page(obj, i, NULL, NULL)) { rv = vm_pager_get_pages(obj, &from_page, 1, 0); if (rv != VM_PAGER_OK) { @@ -301,15 +303,14 @@ int ttm_tt_swapin(struct ttm_tt *ttm) } } else vm_page_zero_invalid(from_page, TRUE); + vm_page_wakeup(from_page); } to_page = ttm->pages[i]; if (unlikely(to_page == NULL)) { - vm_page_wakeup(from_page); ret = -ENOMEM; goto err_ret; } pmap_copy_page(from_page, to_page); - vm_page_wakeup(from_page); } vm_object_pip_wakeup(obj); VM_OBJECT_UNLOCK(obj); @@ -354,8 +355,8 @@ int ttm_tt_swapout(struct ttm_tt *ttm, vm_object_t continue; to_page = vm_page_grab(obj, i, VM_ALLOC_RETRY); pmap_copy_page(from_page, to_page); + to_page->valid = VM_PAGE_BITS_ALL; vm_page_dirty(to_page); - to_page->valid = VM_PAGE_BITS_ALL; vm_page_wakeup(to_page); } vm_object_pip_wakeup(obj); Index: sys/dev/iicbus/iicbb.c =================================================================== --- sys/dev/iicbus/iicbb.c (revision 261896) +++ sys/dev/iicbus/iicbb.c (working copy) @@ -75,6 +75,7 @@ static int iicbb_stop(device_t); static int iicbb_write(device_t, const char *, int, int *, int); static int iicbb_read(device_t, char *, int, int *, int, int); static int iicbb_reset(device_t, u_char, u_char, u_char *); +static int iicbb_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs); static device_method_t iicbb_methods[] = { /* device interface */ @@ -94,7 +95,7 @@ static device_method_t iicbb_methods[] = { DEVMETHOD(iicbus_write, iicbb_write), DEVMETHOD(iicbus_read, iicbb_read), DEVMETHOD(iicbus_reset, iicbb_reset), - DEVMETHOD(iicbus_transfer, iicbus_transfer_gen), + DEVMETHOD(iicbus_transfer, iicbb_transfer), { 0, 0 } }; @@ -413,6 +414,21 @@ iicbb_read(device_t dev, char * buf, int len, int return (0); } +static int +iicbb_transfer(device_t dev, struct iic_msg *msgs, uint32_t nmsgs) +{ + int error; + + error = IICBB_PRE_XFER(device_get_parent(dev)); + if (error) + return (error); + + error = iicbus_transfer_gen(dev, msgs, nmsgs); + + IICBB_POST_XFER(device_get_parent(dev)); + return (error); +} + DRIVER_MODULE(iicbus, iicbb, iicbus_driver, iicbus_devclass, 0, 0); MODULE_DEPEND(iicbb, iicbus, IICBUS_MINVER, IICBUS_PREFVER, IICBUS_MAXVER); Index: sys/dev/iicbus/iicbb_if.m =================================================================== --- sys/dev/iicbus/iicbb_if.m (revision 261896) +++ sys/dev/iicbus/iicbb_if.m (working copy) @@ -31,6 +31,29 @@ INTERFACE iicbb; # +# Default implementation of optional methods +# +CODE { + static int + null_pre_xfer(device_t dev) + { + return 0; + } + + static void + null_post_xfer(device_t dev) + + { + } + + static int + null_callback(device_t dev, int index, caddr_t data) + { + return 0; + } +}; + +# # iicbus callback # METHOD int callback { @@ -37,9 +60,23 @@ METHOD int callback { device_t dev; int index; caddr_t data; -}; +} DEFAULT null_callback; # +# Prepare device for I2C transfer +# +METHOD int pre_xfer { + device_t dev; +} DEFAULT null_pre_xfer; + +# +# Cleanup device after I2C transfer +# +METHOD void post_xfer { + device_t dev; +} DEFAULT null_post_xfer; + +# # Set I2C bus data line # METHOD void setsda { Index: sys/dev =================================================================== --- sys/dev (revision 261896) +++ sys/dev (working copy) Property changes on: sys/dev ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/dev:r232365,247835,247848-247849,248060,248657,248661,248663,248666,251452,252864,253710,254822,254858,254860-254868,254870-254871,254873-254880,254885,254894,255572-255573,255587,257869,259003,259101,259104,259612,259684,261497 Index: sys/i386/include/atomic.h =================================================================== --- sys/i386/include/atomic.h (revision 261896) +++ sys/i386/include/atomic.h (working copy) @@ -54,6 +54,7 @@ * atomic_clear_int(P, V) (*(u_int *)(P) &= ~(V)) * atomic_add_int(P, V) (*(u_int *)(P) += (V)) * atomic_subtract_int(P, V) (*(u_int *)(P) -= (V)) + * atomic_swap_int(P, V) (return (*(u_int *)(P)); *(u_int *)(P) = (V);) * atomic_readandclear_int(P) (return (*(u_int *)(P)); *(u_int *)(P) = 0;) * * atomic_set_long(P, V) (*(u_long *)(P) |= (V)) @@ -60,6 +61,7 @@ * atomic_clear_long(P, V) (*(u_long *)(P) &= ~(V)) * atomic_add_long(P, V) (*(u_long *)(P) += (V)) * atomic_subtract_long(P, V) (*(u_long *)(P) -= (V)) + * atomic_swap_long(P, V) (return (*(u_long *)(P)); *(u_long *)(P) = (V);) * atomic_readandclear_long(P) (return (*(u_long *)(P)); *(u_long *)(P) = 0;) */ @@ -78,6 +80,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); +int atomic_testandset_int(volatile u_int *p, u_int v); #define ATOMIC_LOAD(TYPE, LOP) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -84,6 +87,9 @@ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) +int atomic_cmpset_64(volatile uint64_t *, uint64_t, uint64_t); +uint64_t atomic_swap_64(volatile uint64_t *, uint64_t); + #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* @@ -281,6 +287,23 @@ atomic_fetchadd_int(volatile u_int *p, u_int v) return (v); } +static __inline int +atomic_testandset_int(volatile u_int *p, u_int v) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " btsl %2,%1 ; " + " setc %0 ; " + "# atomic_testandset_int" + : "=q" (res), /* 0 */ + "+m" (*p) /* 1 */ + : "Ir" (v & 0x1f) /* 2 */ + : "cc"); + return (res); +} + /* * We assume that a = b will do atomic loads and stores. Due to the * IA32 memory model, a simple store guarantees release semantics. @@ -335,6 +358,125 @@ struct __hack #endif /* _KERNEL && !SMP */ +#ifdef _KERNEL + +#ifdef WANT_FUNCTIONS +int atomic_cmpset_64_i386(volatile uint64_t *, uint64_t, uint64_t); +int atomic_cmpset_64_i586(volatile uint64_t *, uint64_t, uint64_t); +uint64_t atomic_swap_64_i386(volatile uint64_t *, uint64_t); +uint64_t atomic_swap_64_i586(volatile uint64_t *, uint64_t); +#endif + +/* I486 does not support SMP or CMPXCHG8B. */ +static __inline int +atomic_cmpset_64_i386(volatile uint64_t *dst, uint64_t expect, uint64_t src) +{ + volatile uint32_t *p; + u_char res; + + p = (volatile uint32_t *)dst; + __asm __volatile( + " pushfl ; " + " cli ; " + " xorl %1,%%eax ; " + " xorl %2,%%edx ; " + " orl %%edx,%%eax ; " + " jne 1f ; " + " movl %4,%1 ; " + " movl %5,%2 ; " + "1: " + " sete %3 ; " + " popfl" + : "+A" (expect), /* 0 */ + "+m" (*p), /* 1 */ + "+m" (*(p + 1)), /* 2 */ + "=q" (res) /* 3 */ + : "r" ((uint32_t)src), /* 4 */ + "r" ((uint32_t)(src >> 32)) /* 5 */ + : "memory", "cc"); + return (res); +} + +static __inline uint64_t +atomic_swap_64_i386(volatile uint64_t *p, uint64_t v) +{ + volatile uint32_t *q; + uint64_t res; + + q = (volatile uint32_t *)p; + __asm __volatile( + " pushfl ; " + " cli ; " + " movl %1,%%eax ; " + " movl %2,%%edx ; " + " movl %4,%2 ; " + " movl %3,%1 ; " + " popfl" + : "=&A" (res), /* 0 */ + "+m" (*q), /* 1 */ + "+m" (*(q + 1)) /* 2 */ + : "r" ((uint32_t)v), /* 3 */ + "r" ((uint32_t)(v >> 32))); /* 4 */ + return (res); +} + +static __inline int +atomic_cmpset_64_i586(volatile uint64_t *dst, uint64_t expect, uint64_t src) +{ + u_char res; + + __asm __volatile( + " " MPLOCKED " " + " cmpxchg8b %1 ; " + " sete %0" + : "=q" (res), /* 0 */ + "+m" (*dst), /* 1 */ + "+A" (expect) /* 2 */ + : "b" ((uint32_t)src), /* 3 */ + "c" ((uint32_t)(src >> 32)) /* 4 */ + : "memory", "cc"); + return (res); +} + +static __inline uint64_t +atomic_swap_64_i586(volatile uint64_t *p, uint64_t v) +{ + + __asm __volatile( + " movl %%eax,%%ebx ; " + " movl %%edx,%%ecx ; " + "1: " + " " MPLOCKED " " + " cmpxchg8b %0 ; " + " jne 1b" + : "+m" (*p), /* 0 */ + "+A" (v) /* 1 */ + : : "ebx", "ecx", "memory", "cc"); + return (v); +} + +static __inline int +atomic_cmpset_64(volatile uint64_t *dst, uint64_t expect, uint64_t src) +{ + + if ((cpu_feature & CPUID_CX8) == 0) + return (atomic_cmpset_64_i386(dst, expect, src)); + else + return (atomic_cmpset_64_i586(dst, expect, src)); +} + +static __inline uint64_t +atomic_swap_64(volatile uint64_t *p, uint64_t v) +{ + + if ((cpu_feature & CPUID_CX8) == 0) + return (atomic_swap_64_i386(p, v)); + else + return (atomic_swap_64_i586(p, v)); +} + +#endif /* _KERNEL */ + #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); @@ -393,6 +535,13 @@ atomic_fetchadd_long(volatile u_long *p, u_long v) return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v)); } +static __inline int +atomic_testandset_long(volatile u_long *p, u_int v) +{ + + return (atomic_testandset_int((volatile u_int *)p, v)); +} + /* Read the current value and store a zero in the destination. */ #ifdef __GNUCLIKE_ASM @@ -428,10 +577,31 @@ atomic_readandclear_long(volatile u_long *addr) return (res); } +static __inline u_int +atomic_swap_int(volatile u_int *p, u_int v) +{ + + __asm __volatile( + " xchgl %1,%0 ; " + "# atomic_swap_int" + : "+r" (v), /* 0 */ + "+m" (*p)); /* 1 */ + return (v); +} + +static __inline u_long +atomic_swap_long(volatile u_long *p, u_long v) +{ + + return (atomic_swap_int((volatile u_int *)p, (u_int)v)); +} + #else /* !__GNUCLIKE_ASM */ u_int atomic_readandclear_int(volatile u_int *addr); u_long atomic_readandclear_long(volatile u_long *addr); +u_int atomic_swap_int(volatile u_int *p, u_int v); +u_long atomic_swap_long(volatile u_long *p, u_long v); #endif /* __GNUCLIKE_ASM */ @@ -525,8 +695,10 @@ u_long atomic_readandclear_long(volatile u_long *a #define atomic_cmpset_32 atomic_cmpset_int #define atomic_cmpset_acq_32 atomic_cmpset_acq_int #define atomic_cmpset_rel_32 atomic_cmpset_rel_int +#define atomic_swap_32 atomic_swap_int #define atomic_readandclear_32 atomic_readandclear_int #define atomic_fetchadd_32 atomic_fetchadd_int +#define atomic_testandset_32 atomic_testandset_int /* Operations on pointers. */ #define atomic_set_ptr(p, v) \ @@ -565,6 +737,8 @@ u_long atomic_readandclear_long(volatile u_long *a #define atomic_cmpset_rel_ptr(dst, old, new) \ atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \ (u_int)(new)) +#define atomic_swap_ptr(p, v) \ + atomic_swap_int((volatile u_int *)(p), (u_int)(v)) #define atomic_readandclear_ptr(p) \ atomic_readandclear_int((volatile u_int *)(p)) Index: sys/kern/kern_malloc.c =================================================================== --- sys/kern/kern_malloc.c (revision 261896) +++ sys/kern/kern_malloc.c (working copy) @@ -406,6 +406,43 @@ malloc_type_freed(struct malloc_type *mtp, unsigne } /* + * contigmalloc: + * + * Allocate a block of physically contiguous memory. + * + * If M_NOWAIT is set, this routine will not block and return NULL if + * the allocation fails. + */ +void * +contigmalloc(unsigned long size, struct malloc_type *type, int flags, + vm_paddr_t low, vm_paddr_t high, unsigned long alignment, + unsigned long boundary) +{ + void *ret; + + ret = (void *)kmem_alloc_contig(kernel_map, size, flags, low, high, + alignment, boundary, VM_MEMATTR_DEFAULT); + if (ret != NULL) + malloc_type_allocated(type, round_page(size)); + return (ret); +} + +/* + * contigfree: + * + * Free a block of memory allocated by contigmalloc. + * + * This routine may not block. + */ +void +contigfree(void *addr, unsigned long size, struct malloc_type *type) +{ + + kmem_free(kernel_map, (vm_offset_t)addr, size); + malloc_type_freed(type, round_page(size)); +} + +/* * malloc: * * Allocate a block of memory. Index: sys/mips/mips/pmap.c =================================================================== --- sys/mips/mips/pmap.c (revision 261896) +++ sys/mips/mips/pmap.c (working copy) @@ -1073,7 +1073,8 @@ pmap_alloc_direct_page(unsigned int index, int req { vm_page_t m; - m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req); + m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, req | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); if (m == NULL) return (NULL); @@ -1081,8 +1082,6 @@ pmap_alloc_direct_page(unsigned int index, int req pmap_zero_page(m); m->pindex = index; - atomic_add_int(&cnt.v_wire_count, 1); - m->wire_count = 1; return (m); } Index: sys/modules/drm2/Makefile =================================================================== --- sys/modules/drm2/Makefile (revision 261896) +++ sys/modules/drm2/Makefile (working copy) @@ -2,8 +2,14 @@ .include +.if ${MK_SOURCELESS_UCODE} != "no" +_radeonkmsfw= radeonkmsfw +.endif + SUBDIR = \ drm2 \ - i915kms + i915kms \ + radeonkms \ + ${_radeonkmsfw} .include Index: sys/modules/drm2/drm2/Makefile =================================================================== --- sys/modules/drm2/drm2/Makefile (revision 261896) +++ sys/modules/drm2/drm2/Makefile (working copy) @@ -1,6 +1,6 @@ # $FreeBSD$ -.PATH: ${.CURDIR}/../../../dev/drm2 +.PATH: ${.CURDIR}/../../../dev/drm2 ${.CURDIR}/../../../dev/drm2/ttm KMOD = drm2 SRCS = \ drm_agpsupport.c \ @@ -34,7 +34,20 @@ SRCS = \ drm_sman.c \ drm_stub.c \ drm_sysctl.c \ - drm_vm.c + drm_vm.c \ + ttm_lock.c \ + ttm_object.c \ + ttm_tt.c \ + ttm_bo_util.c \ + ttm_bo.c \ + ttm_bo_manager.c \ + ttm_execbuf_util.c \ + ttm_memory.c \ + ttm_page_alloc.c \ + ttm_bo_vm.c \ + ati_pcigart.c +#ttm_agp_backend.c +#ttm_page_alloc_dma.c .if ${MACHINE_CPUARCH} == "amd64" SRCS += drm_ioc32.c Index: sys/modules =================================================================== --- sys/modules (revision 261896) +++ sys/modules (working copy) Property changes on: sys/modules ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys/modules:r247835,247848-247849,254885,254894,255572-255573,255587,257869,259003,259101,259104,259684,261497 Index: sys/powerpc/aim/slb.c =================================================================== --- sys/powerpc/aim/slb.c (revision 261896) +++ sys/powerpc/aim/slb.c (working copy) @@ -40,7 +40,6 @@ #include #include #include -#include #include #include @@ -478,15 +477,22 @@ slb_uma_real_alloc(uma_zone_t zone, int bytes, u_i static vm_offset_t realmax = 0; void *va; vm_page_t m; + int pflags; if (realmax == 0) realmax = platform_real_maxaddr(); *flags = UMA_SLAB_PRIV; + if ((wait & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) + pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; + else + pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED; + if (wait & M_ZERO) + pflags |= VM_ALLOC_ZERO; for (;;) { - m = vm_phys_alloc_contig(1, 0, realmax, PAGE_SIZE, - PAGE_SIZE); + m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax, + PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT); if (m == NULL) { if (wait & M_NOWAIT) return (NULL); @@ -503,10 +509,6 @@ slb_uma_real_alloc(uma_zone_t zone, int bytes, u_i if ((wait & M_ZERO) && (m->flags & PG_ZERO) == 0) bzero(va, PAGE_SIZE); - /* vm_phys_alloc_contig does not track wiring */ - atomic_add_int(&cnt.v_wire_count, 1); - m->wire_count = 1; - return (va); } Index: sys/vm/vm_contig.c =================================================================== --- sys/vm/vm_contig.c (revision 261896) +++ sys/vm/vm_contig.c (working copy) @@ -66,7 +66,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include #include @@ -84,7 +83,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include static int @@ -188,22 +186,6 @@ vm_contig_launder(int queue, int tries, vm_paddr_t } /* - * Frees the given physically contiguous pages. - * - * N.B.: Any pages with PG_ZERO set must, in fact, be zero filled. - */ -static void -vm_page_release_contig(vm_page_t m, vm_pindex_t count) -{ - - while (count--) { - /* Leave PG_ZERO unchanged. */ - vm_page_free_toq(m); - m++; - } -} - -/* * Increase the number of cached pages. The specified value, "tries", * determines which categories of pages are cached: * @@ -264,9 +246,10 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, int vm_paddr_t high, vm_memattr_t memattr) { vm_object_t object = kernel_object; - vm_offset_t addr, i, offset; + vm_offset_t addr; + vm_ooffset_t end_offset, offset; vm_page_t m; - int tries; + int pflags, tries; size = round_page(size); vm_map_lock(map); @@ -278,14 +261,22 @@ kmem_alloc_attr(vm_map_t map, vm_size_t size, int vm_object_reference(object); vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0); + if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) + pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY; + else + pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY; + if (flags & M_ZERO) + pflags |= VM_ALLOC_ZERO; VM_OBJECT_LOCK(object); - for (i = 0; i < size; i += PAGE_SIZE) { + end_offset = offset + size; + for (; offset < end_offset; offset += PAGE_SIZE) { tries = 0; retry: - m = vm_phys_alloc_contig(1, low, high, PAGE_SIZE, 0); + m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, 1, + low, high, PAGE_SIZE, 0, memattr); if (m == NULL) { + VM_OBJECT_UNLOCK(object); if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { - VM_OBJECT_UNLOCK(object); vm_map_unlock(map); vm_contig_grow_cache(tries, low, high); vm_map_lock(map); @@ -293,20 +284,16 @@ retry: tries++; goto retry; } - while (i != 0) { - i -= PAGE_SIZE; - m = vm_page_lookup(object, OFF_TO_IDX(offset + - i)); - vm_page_free(m); - } - VM_OBJECT_UNLOCK(object); + /* + * Since the pages that were allocated by any previous + * iterations of this loop are not busy, they can be + * freed by vm_object_page_remove(), which is called + * by vm_map_delete(). + */ vm_map_delete(map, addr, addr + size); vm_map_unlock(map); return (0); } - if (memattr != VM_MEMATTR_DEFAULT) - pmap_page_set_memattr(m, memattr); - vm_page_insert(m, object, OFF_TO_IDX(offset + i)); if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL; @@ -326,92 +313,61 @@ retry: * specified through the given flags, then the pages are zeroed * before they are mapped. */ -static vm_offset_t -contigmapping(vm_map_t map, vm_size_t size, vm_page_t m, vm_memattr_t memattr, - int flags) +vm_offset_t +kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, + vm_paddr_t high, u_long alignment, vm_paddr_t boundary, + vm_memattr_t memattr) { vm_object_t object = kernel_object; - vm_offset_t addr, tmp_addr; + vm_offset_t addr; + vm_ooffset_t offset; + vm_page_t end_m, m; + int pflags, tries; + size = round_page(size); vm_map_lock(map); if (vm_map_findspace(map, vm_map_min(map), size, &addr)) { vm_map_unlock(map); return (0); } + offset = addr - VM_MIN_KERNEL_ADDRESS; vm_object_reference(object); - vm_map_insert(map, object, addr - VM_MIN_KERNEL_ADDRESS, - addr, addr + size, VM_PROT_ALL, VM_PROT_ALL, 0); - vm_map_unlock(map); + vm_map_insert(map, object, offset, addr, addr + size, VM_PROT_ALL, + VM_PROT_ALL, 0); + if ((flags & (M_NOWAIT | M_USE_RESERVE)) == M_NOWAIT) + pflags = VM_ALLOC_INTERRUPT | VM_ALLOC_NOBUSY; + else + pflags = VM_ALLOC_SYSTEM | VM_ALLOC_NOBUSY; + if (flags & M_ZERO) + pflags |= VM_ALLOC_ZERO; VM_OBJECT_LOCK(object); - for (tmp_addr = addr; tmp_addr < addr + size; tmp_addr += PAGE_SIZE) { - if (memattr != VM_MEMATTR_DEFAULT) - pmap_page_set_memattr(m, memattr); - vm_page_insert(m, object, - OFF_TO_IDX(tmp_addr - VM_MIN_KERNEL_ADDRESS)); - if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) - pmap_zero_page(m); - m->valid = VM_PAGE_BITS_ALL; - m++; - } - VM_OBJECT_UNLOCK(object); - vm_map_wire(map, addr, addr + size, - VM_MAP_WIRE_SYSTEM | VM_MAP_WIRE_NOHOLES); - return (addr); -} - -void * -contigmalloc( - unsigned long size, /* should be size_t here and for malloc() */ - struct malloc_type *type, - int flags, - vm_paddr_t low, - vm_paddr_t high, - unsigned long alignment, - unsigned long boundary) -{ - void *ret; - - ret = (void *)kmem_alloc_contig(kernel_map, size, flags, low, high, - alignment, boundary, VM_MEMATTR_DEFAULT); - if (ret != NULL) - malloc_type_allocated(type, round_page(size)); - return (ret); -} - -vm_offset_t -kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, - vm_paddr_t high, unsigned long alignment, unsigned long boundary, - vm_memattr_t memattr) -{ - vm_offset_t ret; - vm_page_t pages; - unsigned long npgs; - int tries; - - size = round_page(size); - npgs = size >> PAGE_SHIFT; tries = 0; retry: - pages = vm_phys_alloc_contig(npgs, low, high, alignment, boundary); - if (pages == NULL) { + m = vm_page_alloc_contig(object, OFF_TO_IDX(offset), pflags, + atop(size), low, high, alignment, boundary, memattr); + if (m == NULL) { + VM_OBJECT_UNLOCK(object); if (tries < ((flags & M_NOWAIT) != 0 ? 1 : 3)) { + vm_map_unlock(map); vm_contig_grow_cache(tries, low, high); + vm_map_lock(map); + VM_OBJECT_LOCK(object); tries++; goto retry; } - ret = 0; - } else { - ret = contigmapping(map, size, pages, memattr, flags); - if (ret == 0) - vm_page_release_contig(pages, npgs); + vm_map_delete(map, addr, addr + size); + vm_map_unlock(map); + return (0); } - return (ret); + end_m = m + atop(size); + for (; m < end_m; m++) { + if ((flags & M_ZERO) && (m->flags & PG_ZERO) == 0) + pmap_zero_page(m); + m->valid = VM_PAGE_BITS_ALL; + } + VM_OBJECT_UNLOCK(object); + vm_map_unlock(map); + vm_map_wire(map, addr, addr + size, VM_MAP_WIRE_SYSTEM | + VM_MAP_WIRE_NOHOLES); + return (addr); } - -void -contigfree(void *addr, unsigned long size, struct malloc_type *type) -{ - - kmem_free(kernel_map, (vm_offset_t)addr, size); - malloc_type_freed(type, round_page(size)); -} Index: sys/vm/vm_extern.h =================================================================== --- sys/vm/vm_extern.h (revision 261896) +++ sys/vm/vm_extern.h (working copy) @@ -44,8 +44,8 @@ vm_offset_t kmem_alloc(vm_map_t, vm_size_t); vm_offset_t kmem_alloc_attr(vm_map_t map, vm_size_t size, int flags, vm_paddr_t low, vm_paddr_t high, vm_memattr_t memattr); vm_offset_t kmem_alloc_contig(vm_map_t map, vm_size_t size, int flags, - vm_paddr_t low, vm_paddr_t high, unsigned long alignment, - unsigned long boundary, vm_memattr_t memattr); + vm_paddr_t low, vm_paddr_t high, u_long alignment, vm_paddr_t boundary, + vm_memattr_t memattr); vm_offset_t kmem_alloc_nofault(vm_map_t, vm_size_t); vm_offset_t kmem_alloc_nofault_space(vm_map_t, vm_size_t, int); vm_offset_t kmem_alloc_wait(vm_map_t, vm_size_t); Index: sys/vm/vm_page.c =================================================================== --- sys/vm/vm_page.c (revision 261896) +++ sys/vm/vm_page.c (working copy) @@ -137,6 +137,7 @@ SYSCTL_INT(_vm, OID_AUTO, tryrelock_restart, CTLFL static uma_zone_t fakepg_zone; +static struct vnode *vm_page_alloc_init(vm_page_t m); static void vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits); static void vm_page_queue_remove(int queue, vm_page_t m); static void vm_page_enqueue(int queue, vm_page_t m); @@ -1379,8 +1380,9 @@ vm_page_is_cached(vm_object_t object, vm_pindex_t /* * vm_page_alloc: * - * Allocate and return a memory cell associated - * with this VM object/offset pair. + * Allocate and return a page that is associated with the specified + * object and offset pair. By default, this page has the flag VPO_BUSY + * set. * * The caller must always specify an allocation class. * @@ -1390,13 +1392,16 @@ vm_page_is_cached(vm_object_t object, vm_pindex_t * VM_ALLOC_INTERRUPT interrupt time request * * optional allocation flags: - * VM_ALLOC_ZERO prefer a zeroed page - * VM_ALLOC_WIRED wire the allocated page - * VM_ALLOC_NOOBJ page is not associated with a vm object - * VM_ALLOC_NOBUSY do not set the page busy + * VM_ALLOC_COUNT(number) the number of additional pages that the caller + * intends to allocate * VM_ALLOC_IFCACHED return page only if it is cached * VM_ALLOC_IFNOTCACHED return NULL, do not reactivate if the page * is cached + * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page + * VM_ALLOC_NOOBJ page is not associated with an object and + * should not have the flag VPO_BUSY set + * VM_ALLOC_WIRED wire the allocated page + * VM_ALLOC_ZERO prefer a zeroed page * * This routine may not sleep. */ @@ -1406,27 +1411,26 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind struct vnode *vp = NULL; vm_object_t m_object; vm_page_t m; - int flags, page_req; + int flags, req_class; - if ((req & VM_ALLOC_NOOBJ) == 0) { - KASSERT(object != NULL, - ("vm_page_alloc: NULL object.")); + KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0), + ("vm_page_alloc: inconsistent object/req")); + if (object != NULL) VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); - } - page_req = req & VM_ALLOC_CLASS_MASK; + req_class = req & VM_ALLOC_CLASS_MASK; /* - * The pager is allowed to eat deeper into the free page list. + * The page daemon is allowed to dig deeper into the free page list. */ - if ((curproc == pageproc) && (page_req != VM_ALLOC_INTERRUPT)) - page_req = VM_ALLOC_SYSTEM; + if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) + req_class = VM_ALLOC_SYSTEM; mtx_lock(&vm_page_queue_free_mtx); if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved || - (page_req == VM_ALLOC_SYSTEM && + (req_class == VM_ALLOC_SYSTEM && cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) || - (page_req == VM_ALLOC_INTERRUPT && + (req_class == VM_ALLOC_INTERRUPT && cnt.v_free_count + cnt.v_cache_count > 0)) { /* * Allocate from the free queue if the number of free pages @@ -1474,7 +1478,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind */ mtx_unlock(&vm_page_queue_free_mtx); atomic_add_int(&vm_pageout_deficit, - MAX((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); + max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); pagedaemon_wakeup(); return (NULL); } @@ -1482,7 +1486,6 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind /* * At this point we had better have found a good page. */ - KASSERT(m != NULL, ("vm_page_alloc: missing page")); KASSERT(m->queue == PQ_NONE, ("vm_page_alloc: page %p has unexpected queue %d", m, m->queue)); @@ -1494,6 +1497,8 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind ("vm_page_alloc: page %p has unexpected memattr %d", m, pmap_page_get_memattr(m))); if ((m->flags & PG_CACHED) != 0) { + KASSERT((m->flags & PG_ZERO) == 0, + ("vm_page_alloc: cached page %p is PG_ZERO", m)); KASSERT(m->valid != 0, ("vm_page_alloc: cached page %p is invalid", m)); if (m->object == object && m->pindex == pindex) @@ -1572,12 +1577,163 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pind } /* + * vm_page_alloc_contig: + * + * Allocate a contiguous set of physical pages of the given size "npages" + * from the free lists. All of the physical pages must be at or above + * the given physical address "low" and below the given physical address + * "high". The given value "alignment" determines the alignment of the + * first physical page in the set. If the given value "boundary" is + * non-zero, then the set of physical pages cannot cross any physical + * address boundary that is a multiple of that value. Both "alignment" + * and "boundary" must be a power of two. + * + * If the specified memory attribute, "memattr", is VM_MEMATTR_DEFAULT, + * then the memory attribute setting for the physical pages is configured + * to the object's memory attribute setting. Otherwise, the memory + * attribute setting for the physical pages is configured to "memattr", + * overriding the object's memory attribute setting. However, if the + * object's memory attribute setting is not VM_MEMATTR_DEFAULT, then the + * memory attribute setting for the physical pages cannot be configured + * to VM_MEMATTR_DEFAULT. + * + * The caller must always specify an allocation class. + * + * allocation classes: + * VM_ALLOC_NORMAL normal process request + * VM_ALLOC_SYSTEM system *really* needs a page + * VM_ALLOC_INTERRUPT interrupt time request + * + * optional allocation flags: + * VM_ALLOC_NOBUSY do not set the flag VPO_BUSY on the page + * VM_ALLOC_NOOBJ page is not associated with an object and + * should not have the flag VPO_BUSY set + * VM_ALLOC_WIRED wire the allocated page + * VM_ALLOC_ZERO prefer a zeroed page + * + * This routine may not sleep. + */ +vm_page_t +vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, + u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, + vm_paddr_t boundary, vm_memattr_t memattr) +{ + struct vnode *drop; + vm_page_t deferred_vdrop_list, m, m_ret; + u_int flags, oflags; + int req_class; + + KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0), + ("vm_page_alloc_contig: inconsistent object/req")); + if (object != NULL) { + VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + KASSERT(object->type == OBJT_PHYS, + ("vm_page_alloc_contig: object %p isn't OBJT_PHYS", + object)); + } + KASSERT(npages > 0, ("vm_page_alloc_contig: npages is zero")); + req_class = req & VM_ALLOC_CLASS_MASK; + + /* + * The page daemon is allowed to dig deeper into the free page list. + */ + if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) + req_class = VM_ALLOC_SYSTEM; + + deferred_vdrop_list = NULL; + mtx_lock(&vm_page_queue_free_mtx); + if (cnt.v_free_count + cnt.v_cache_count >= npages + + cnt.v_free_reserved || (req_class == VM_ALLOC_SYSTEM && + cnt.v_free_count + cnt.v_cache_count >= npages + + cnt.v_interrupt_free_min) || (req_class == VM_ALLOC_INTERRUPT && + cnt.v_free_count + cnt.v_cache_count >= npages)) { +#if VM_NRESERVLEVEL > 0 +retry: +#endif + m_ret = vm_phys_alloc_contig(npages, low, high, alignment, + boundary); + } else { + mtx_unlock(&vm_page_queue_free_mtx); + atomic_add_int(&vm_pageout_deficit, npages); + pagedaemon_wakeup(); + return (NULL); + } + if (m_ret != NULL) + for (m = m_ret; m < &m_ret[npages]; m++) { + drop = vm_page_alloc_init(m); + if (drop != NULL) { + /* + * Enqueue the vnode for deferred vdrop(). + * + * Once the pages are removed from the free + * page list, "pageq" can be safely abused to + * construct a short-lived list of vnodes. + */ + m->pageq.tqe_prev = (void *)drop; + m->pageq.tqe_next = deferred_vdrop_list; + deferred_vdrop_list = m; + } + } + else { +#if VM_NRESERVLEVEL > 0 + if (vm_reserv_reclaim_contig(npages << PAGE_SHIFT, low, high, + alignment, boundary)) + goto retry; +#endif + } + mtx_unlock(&vm_page_queue_free_mtx); + if (m_ret == NULL) + return (NULL); + + /* + * Initialize the pages. Only the PG_ZERO flag is inherited. + */ + flags = 0; + if ((req & VM_ALLOC_ZERO) != 0) + flags = PG_ZERO; + if ((req & VM_ALLOC_WIRED) != 0) + atomic_add_int(&cnt.v_wire_count, npages); + oflags = VPO_UNMANAGED; + if (object != NULL) { + if ((req & VM_ALLOC_NOBUSY) == 0) + oflags |= VPO_BUSY; + if (object->memattr != VM_MEMATTR_DEFAULT && + memattr == VM_MEMATTR_DEFAULT) + memattr = object->memattr; + } + for (m = m_ret; m < &m_ret[npages]; m++) { + m->aflags = 0; + m->flags &= flags; + if ((req & VM_ALLOC_WIRED) != 0) + m->wire_count = 1; + /* Unmanaged pages don't use "act_count". */ + m->oflags = oflags; + if (memattr != VM_MEMATTR_DEFAULT) + pmap_page_set_memattr(m, memattr); + if (object != NULL) + vm_page_insert(m, object, pindex); + else + m->pindex = pindex; + pindex++; + } + while (deferred_vdrop_list != NULL) { + vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev); + deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next; + } + if (vm_paging_needed()) + pagedaemon_wakeup(); + return (m_ret); +} + +/* * Initialize a page that has been freshly dequeued from a freelist. * The caller has to drop the vnode returned, if it is not NULL. * + * This function may only be used to initialize unmanaged pages. + * * To be called with vm_page_queue_free_mtx held. */ -struct vnode * +static struct vnode * vm_page_alloc_init(vm_page_t m) { struct vnode *drop; @@ -1600,11 +1756,12 @@ vm_page_alloc_init(vm_page_t m) mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); drop = NULL; if ((m->flags & PG_CACHED) != 0) { + KASSERT((m->flags & PG_ZERO) == 0, + ("vm_page_alloc_init: cached page %p is PG_ZERO", m)); m->valid = 0; m_object = m->object; vm_page_cache_remove(m); - if (m_object->type == OBJT_VNODE && - m_object->cache == NULL) + if (m_object->type == OBJT_VNODE && m_object->cache == NULL) drop = m_object->handle; } else { KASSERT(VM_PAGE_IS_FREE(m), @@ -1612,23 +1769,33 @@ vm_page_alloc_init(vm_page_t m) KASSERT(m->valid == 0, ("vm_page_alloc_init: free page %p is valid", m)); cnt.v_free_count--; + if ((m->flags & PG_ZERO) != 0) + vm_page_zero_count--; } - if (m->flags & PG_ZERO) - vm_page_zero_count--; /* Don't clear the PG_ZERO flag; we'll need it later. */ m->flags &= PG_ZERO; - m->aflags = 0; - m->oflags = VPO_UNMANAGED; - /* Unmanaged pages don't use "act_count". */ return (drop); } /* * vm_page_alloc_freelist: - * - * Allocate a page from the specified freelist. - * Only the ALLOC_CLASS values in req are honored, other request flags - * are ignored. + * + * Allocate a physical page from the specified free page list. + * + * The caller must always specify an allocation class. + * + * allocation classes: + * VM_ALLOC_NORMAL normal process request + * VM_ALLOC_SYSTEM system *really* needs a page + * VM_ALLOC_INTERRUPT interrupt time request + * + * optional allocation flags: + * VM_ALLOC_COUNT(number) the number of additional pages that the caller + * intends to allocate + * VM_ALLOC_WIRED wire the allocated page + * VM_ALLOC_ZERO prefer a zeroed page + * + * This routine may not sleep. */ vm_page_t vm_page_alloc_freelist(int flind, int req) @@ -1635,20 +1802,33 @@ vm_page_alloc_freelist(int flind, int req) { struct vnode *drop; vm_page_t m; - int page_req; + u_int flags; + int req_class; - m = NULL; - page_req = req & VM_ALLOC_CLASS_MASK; - mtx_lock(&vm_page_queue_free_mtx); + req_class = req & VM_ALLOC_CLASS_MASK; + /* + * The page daemon is allowed to dig deeper into the free page list. + */ + if (curproc == pageproc && req_class != VM_ALLOC_INTERRUPT) + req_class = VM_ALLOC_SYSTEM; + + /* * Do not allocate reserved pages unless the req has asked for it. */ + mtx_lock(&vm_page_queue_free_mtx); if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved || - (page_req == VM_ALLOC_SYSTEM && + (req_class == VM_ALLOC_SYSTEM && cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) || - (page_req == VM_ALLOC_INTERRUPT && - cnt.v_free_count + cnt.v_cache_count > 0)) { + (req_class == VM_ALLOC_INTERRUPT && + cnt.v_free_count + cnt.v_cache_count > 0)) m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0); + else { + mtx_unlock(&vm_page_queue_free_mtx); + atomic_add_int(&vm_pageout_deficit, + max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1)); + pagedaemon_wakeup(); + return (NULL); } if (m == NULL) { mtx_unlock(&vm_page_queue_free_mtx); @@ -1656,8 +1836,29 @@ vm_page_alloc_freelist(int flind, int req) } drop = vm_page_alloc_init(m); mtx_unlock(&vm_page_queue_free_mtx); - if (drop) + + /* + * Initialize the page. Only the PG_ZERO flag is inherited. + */ + m->aflags = 0; + flags = 0; + if ((req & VM_ALLOC_ZERO) != 0) + flags = PG_ZERO; + m->flags &= flags; + if ((req & VM_ALLOC_WIRED) != 0) { + /* + * The page lock is not required for wiring a page that does + * not belong to an object. + */ + atomic_add_int(&cnt.v_wire_count, 1); + m->wire_count = 1; + } + /* Unmanaged pages don't use "act_count". */ + m->oflags = VPO_UNMANAGED; + if (drop != NULL) vdrop(drop); + if (vm_paging_needed()) + pagedaemon_wakeup(); return (m); } Index: sys/vm/vm_page.h =================================================================== --- sys/vm/vm_page.h (revision 261896) +++ sys/vm/vm_page.h (working copy) @@ -365,8 +365,10 @@ void vm_pageq_remove(vm_page_t m); void vm_page_activate (vm_page_t); vm_page_t vm_page_alloc (vm_object_t, vm_pindex_t, int); +vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, + u_long npages, vm_paddr_t low, vm_paddr_t high, u_long alignment, + vm_paddr_t boundary, vm_memattr_t memattr); vm_page_t vm_page_alloc_freelist(int, int); -struct vnode *vm_page_alloc_init(vm_page_t); vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int); void vm_page_cache(vm_page_t); void vm_page_cache_free(vm_object_t, vm_pindex_t, vm_pindex_t); Index: sys/vm/vm_phys.c =================================================================== --- sys/vm/vm_phys.c (revision 261896) +++ sys/vm/vm_phys.c (working copy) @@ -29,11 +29,17 @@ * POSSIBILITY OF SUCH DAMAGE. */ +/* + * Physical memory system implementation + * + * Any external functions defined by this module are only to be used by the + * virtual memory system. + */ + #include __FBSDID("$FreeBSD$"); #include "opt_ddb.h" -#include "opt_vm.h" #include #include @@ -45,7 +51,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include @@ -55,7 +60,6 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include /* * VM_FREELIST_DEFAULT is split into VM_NDOMAIN lists, one for each @@ -543,26 +547,6 @@ vm_phys_alloc_domain_pages(int domain, int flind, } /* - * Allocate physical memory from phys_avail[]. - */ -vm_paddr_t -vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment) -{ - vm_paddr_t pa; - int i; - - size = round_page(size); - for (i = 0; phys_avail[i + 1] != 0; i += 2) { - if (phys_avail[i + 1] - phys_avail[i] < size) - continue; - pa = phys_avail[i]; - phys_avail[i] += size; - return (pa); - } - panic("vm_phys_bootstrap_alloc"); -} - -/* * Find the vm_page corresponding to the given physical address. */ vm_page_t @@ -713,7 +697,7 @@ vm_phys_free_pages(vm_page_t m, int order) { struct vm_freelist *fl; struct vm_phys_seg *seg; - vm_paddr_t pa, pa_buddy; + vm_paddr_t pa; vm_page_t m_buddy; KASSERT(m->order == VM_NFREEORDER, @@ -725,25 +709,26 @@ vm_phys_free_pages(vm_page_t m, int order) KASSERT(order < VM_NFREEORDER, ("vm_phys_free_pages: order %d is out of range", order)); mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - pa = VM_PAGE_TO_PHYS(m); seg = &vm_phys_segs[m->segind]; - while (order < VM_NFREEORDER - 1) { - pa_buddy = pa ^ (1 << (PAGE_SHIFT + order)); - if (pa_buddy < seg->start || - pa_buddy >= seg->end) - break; - m_buddy = &seg->first_page[atop(pa_buddy - seg->start)]; - if (m_buddy->order != order) - break; - fl = (*seg->free_queues)[m_buddy->pool]; - TAILQ_REMOVE(&fl[m_buddy->order].pl, m_buddy, pageq); - fl[m_buddy->order].lcnt--; - m_buddy->order = VM_NFREEORDER; - if (m_buddy->pool != m->pool) - vm_phys_set_pool(m->pool, m_buddy, order); - order++; - pa &= ~((1 << (PAGE_SHIFT + order)) - 1); - m = &seg->first_page[atop(pa - seg->start)]; + if (order < VM_NFREEORDER - 1) { + pa = VM_PAGE_TO_PHYS(m); + do { + pa ^= ((vm_paddr_t)1 << (PAGE_SHIFT + order)); + if (pa < seg->start || pa >= seg->end) + break; + m_buddy = &seg->first_page[atop(pa - seg->start)]; + if (m_buddy->order != order) + break; + fl = (*seg->free_queues)[m_buddy->pool]; + TAILQ_REMOVE(&fl[order].pl, m_buddy, pageq); + fl[order].lcnt--; + m_buddy->order = VM_NFREEORDER; + if (m_buddy->pool != m->pool) + vm_phys_set_pool(m->pool, m_buddy, order); + order++; + pa &= ~(((vm_paddr_t)1 << (PAGE_SHIFT + order)) - 1); + m = &seg->first_page[atop(pa - seg->start)]; + } while (order < VM_NFREEORDER - 1); } m->order = order; fl = (*seg->free_queues)[m->pool]; @@ -752,6 +737,47 @@ vm_phys_free_pages(vm_page_t m, int order) } /* + * Free a contiguous, arbitrarily sized set of physical pages. + * + * The free page queues must be locked. + */ +void +vm_phys_free_contig(vm_page_t m, u_long npages) +{ + u_int n; + int order; + + /* + * Avoid unnecessary coalescing by freeing the pages in the largest + * possible power-of-two-sized subsets. + */ + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); + for (;; npages -= n) { + /* + * Unsigned "min" is used here so that "order" is assigned + * "VM_NFREEORDER - 1" when "m"'s physical address is zero + * or the low-order bits of its physical address are zero + * because the size of a physical address exceeds the size of + * a long. + */ + order = min(ffsl(VM_PAGE_TO_PHYS(m) >> PAGE_SHIFT) - 1, + VM_NFREEORDER - 1); + n = 1 << order; + if (npages < n) + break; + vm_phys_free_pages(m, order); + m += n; + } + /* The residual "npages" is less than "1 << (VM_NFREEORDER - 1)". */ + for (; npages > 0; npages -= n) { + order = flsl(npages) - 1; + n = 1 << order; + vm_phys_free_pages(m, order); + m += n; + } +} + +/* * Set the pool for a contiguous, power of two-sized set of physical pages. */ void @@ -887,16 +913,17 @@ vm_phys_zero_pages_idle(void) * "alignment" and "boundary" must be a power of two. */ vm_page_t -vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high, - unsigned long alignment, unsigned long boundary) +vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, + u_long alignment, vm_paddr_t boundary) { struct vm_freelist *fl; struct vm_phys_seg *seg; - struct vnode *vp; vm_paddr_t pa, pa_last, size; - vm_page_t deferred_vdrop_list, m, m_ret; - int domain, flind, i, oind, order, pind; + vm_page_t m, m_ret; + u_long npages_end; + int domain, flind, oind, order, pind; + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); #if VM_NDOMAIN > 1 domain = PCPU_GET(domain); #else @@ -909,13 +936,8 @@ vm_page_t ("vm_phys_alloc_contig: alignment must be a power of 2")); KASSERT((boundary & (boundary - 1)) == 0, ("vm_phys_alloc_contig: boundary must be a power of 2")); - deferred_vdrop_list = NULL; /* Compute the queue that is the best fit for npages. */ for (order = 0; (1 << order) < npages; order++); - mtx_lock(&vm_page_queue_free_mtx); -#if VM_NRESERVLEVEL > 0 -retry: -#endif for (flind = 0; flind < vm_nfreelists; flind++) { for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { for (pind = 0; pind < VM_NFREEPOOL; pind++) { @@ -974,11 +996,6 @@ vm_page_t } } } -#if VM_NRESERVLEVEL > 0 - if (vm_reserv_reclaim_contig(size, low, high, alignment, boundary)) - goto retry; -#endif - mtx_unlock(&vm_page_queue_free_mtx); return (NULL); done: for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { @@ -991,34 +1008,10 @@ done: vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); fl = (*seg->free_queues)[m_ret->pool]; vm_phys_split_pages(m_ret, oind, fl, order); - for (i = 0; i < npages; i++) { - m = &m_ret[i]; - vp = vm_page_alloc_init(m); - if (vp != NULL) { - /* - * Enqueue the vnode for deferred vdrop(). - * - * Unmanaged pages don't use "pageq", so it - * can be safely abused to construct a short- - * lived queue of vnodes. - */ - m->pageq.tqe_prev = (void *)vp; - m->pageq.tqe_next = deferred_vdrop_list; - deferred_vdrop_list = m; - } - } - for (; i < roundup2(npages, 1 << imin(oind, order)); i++) { - m = &m_ret[i]; - KASSERT(m->order == VM_NFREEORDER, - ("vm_phys_alloc_contig: page %p has unexpected order %d", - m, m->order)); - vm_phys_free_pages(m, 0); - } - mtx_unlock(&vm_page_queue_free_mtx); - while (deferred_vdrop_list != NULL) { - vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev); - deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next; - } + /* Return excess pages to the free lists. */ + npages_end = roundup2(npages, 1 << imin(oind, order)); + if (npages < npages_end) + vm_phys_free_contig(&m_ret[npages], npages_end - npages); return (m_ret); } Index: sys/vm/vm_phys.h =================================================================== --- sys/vm/vm_phys.h (revision 261896) +++ sys/vm/vm_phys.h (working copy) @@ -49,13 +49,15 @@ struct mem_affinity { extern struct mem_affinity *mem_affinity; +/* + * The following functions are only to be used by the virtual memory system. + */ void vm_phys_add_page(vm_paddr_t pa); -vm_page_t vm_phys_alloc_contig(unsigned long npages, - vm_paddr_t low, vm_paddr_t high, - unsigned long alignment, unsigned long boundary); +vm_page_t vm_phys_alloc_contig(u_long npages, vm_paddr_t low, vm_paddr_t high, + u_long alignment, vm_paddr_t boundary); vm_page_t vm_phys_alloc_freelist_pages(int flind, int pool, int order); vm_page_t vm_phys_alloc_pages(int pool, int order); -vm_paddr_t vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment); +void vm_phys_free_contig(vm_page_t m, u_long npages); int vm_phys_fictitious_reg_range(vm_paddr_t start, vm_paddr_t end, vm_memattr_t memattr); void vm_phys_fictitious_unreg_range(vm_paddr_t start, vm_paddr_t end); Index: sys/vm/vm_reserv.c =================================================================== --- sys/vm/vm_reserv.c (revision 261896) +++ sys/vm/vm_reserv.c (working copy) @@ -630,7 +630,7 @@ vm_reserv_reclaim_inactive(void) */ boolean_t vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low, vm_paddr_t high, - unsigned long alignment, unsigned long boundary) + u_long alignment, vm_paddr_t boundary) { vm_paddr_t pa, pa_length; vm_reserv_t rv; Index: sys/vm/vm_reserv.h =================================================================== --- sys/vm/vm_reserv.h (revision 261896) +++ sys/vm/vm_reserv.h (working copy) @@ -49,8 +49,7 @@ void vm_reserv_init(void); int vm_reserv_level_iffullpop(vm_page_t m); boolean_t vm_reserv_reactivate_page(vm_page_t m); boolean_t vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low, - vm_paddr_t high, unsigned long alignment, - unsigned long boundary); + vm_paddr_t high, u_long alignment, vm_paddr_t boundary); boolean_t vm_reserv_reclaim_inactive(void); void vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, vm_pindex_t old_object_offset); Index: sys =================================================================== --- sys (revision 261896) +++ sys (working copy) Property changes on: sys ___________________________________________________________________ Modified: svn:mergeinfo Merged /head/sys:r226824,226848,226891,226928,227012,227072,227127,227568,232365,247835,247848-247849,248060,248657,248661,248663,248666,251452,252864,253710,254617,254620,254822,254858,254860-254868,254870-254871,254873-254880,254885,254894,255572-255573,255587,257869,259003,259101,259104,259612,259684,261497