--- //depot/vendor/freebsd/src/sys/amd64/amd64/busdma_machdep.c 2009/04/23 20:30:26 +++ //depot/user/jhb/bio/amd64/amd64/busdma_machdep.c 2009/09/02 18:56:14 @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +40,8 @@ #include #include #include +#include +#include #include #include @@ -51,6 +54,9 @@ #include #define MAX_BPAGES 8192 +#define BUS_DMA_PASS_THROUGH BUS_DMA_BUS2 +#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 +#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 struct bounce_zone; @@ -70,14 +76,15 @@ int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; + struct sglist_seg *phys_segments; bus_dma_segment_t *segments; struct bounce_zone *bounce_zone; }; struct bounce_page { - vm_offset_t vaddr; /* kva of bounce buffer */ + void *vaddr; /* kva of bounce buffer */ bus_addr_t busaddr; /* Physical address */ - vm_offset_t datavaddr; /* kva of client data */ + vm_paddr_t datapaddr; /* pa of client data */ bus_size_t datacount; /* client data count */ STAILQ_ENTRY(bounce_page) links; }; @@ -116,6 +123,8 @@ int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; + struct bio *bio; + struct sglist *sglist; void *buf; /* unmapped buffer pointer */ bus_size_t buflen; /* unmapped buffer length */ bus_dmamap_callback_t *callback; @@ -133,9 +142,11 @@ static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit); static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_offset_t vaddr, bus_size_t size); + vm_paddr_t paddr, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); -static __inline int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); +static int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); +static int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, int flags); /* * Return true if a match is made. @@ -145,7 +156,7 @@ * If paddr is within the bounds of the dma tag then call the filter callback * to check for a match, if there is no filter callback then assume a match. */ -static __inline int +int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) { int retval; @@ -159,7 +170,7 @@ || (*dmat->filter)(dmat->filterarg, paddr) != 0)) retval = 1; - dmat = dmat->parent; + dmat = dmat->parent; } while (retval == 0 && dmat != NULL); return (retval); } @@ -200,8 +211,6 @@ panic("driver error: busdma dflt_lock called"); } -#define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 -#define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 /* * Allocate a device specific dma_tag. */ @@ -231,7 +240,7 @@ M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", - __func__, newtag, 0, error); + __func__, newtag, 0, ENOMEM); return (ENOMEM); } @@ -247,7 +256,7 @@ newtag->nsegments = nsegments; newtag->maxsegsz = maxsegsz; newtag->flags = flags; - newtag->ref_count = 1; /* Count ourself */ + refcount_init(&newtag->ref_count, 1); /* Count ourself */ newtag->map_count = 0; if (lockfunc != NULL) { newtag->lockfunc = lockfunc; @@ -256,6 +265,14 @@ newtag->lockfunc = dflt_lock; newtag->lockfuncarg = NULL; } + newtag->phys_segments = malloc(sizeof(struct sglist_seg) * nsegments, + M_DEVBUF, M_NOWAIT); + if (newtag->phys_segments == NULL) { + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, 0, ENOMEM); + free(newtag, M_DEVBUF); + return (ENOMEM); + } newtag->segments = NULL; /* Take into account any restrictions imposed by our parent tag */ @@ -267,6 +284,9 @@ else if (parent->boundary != 0) newtag->boundary = MIN(parent->boundary, newtag->boundary); + if ((newtag->filter != NULL) || + ((parent->flags & BUS_DMA_COULD_BOUNCE) != 0)) + newtag->flags |= BUS_DMA_COULD_BOUNCE; if (newtag->filter == NULL) { /* * Short circuit looking at our parent directly @@ -277,12 +297,15 @@ newtag->parent = parent->parent; } if (newtag->parent != NULL) - atomic_add_int(&parent->ref_count, 1); + refcount_acquire(&parent->ref_count); } if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) || newtag->alignment > 1) newtag->flags |= BUS_DMA_COULD_BOUNCE; + else if (newtag->boundary == 0 && + newtag->maxsegsz == BUS_SPACE_MAXSIZE_32BIT) + newtag->flags |= BUS_DMA_PASS_THROUGH; if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && (flags & BUS_DMA_ALLOCNOW) != 0) { @@ -291,6 +314,7 @@ /* Must bounce */ if ((error = alloc_bounce_zone(newtag)) != 0) { + free(newtag->phys_segments, M_DEVBUF); free(newtag, M_DEVBUF); return (error); } @@ -308,8 +332,9 @@ /* Performed initial allocation */ newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; } - + if (error != 0) { + free(newtag->phys_segments, M_DEVBUF); free(newtag, M_DEVBUF); } else { *dmat = newtag; @@ -339,10 +364,10 @@ bus_dma_tag_t parent; parent = dmat->parent; - atomic_subtract_int(&dmat->ref_count, 1); - if (dmat->ref_count == 0) { + if (refcount_release(&dmat->ref_count)) { if (dmat->segments != NULL) free(dmat->segments, M_DEVBUF); + free(dmat->phys_segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so @@ -500,7 +525,7 @@ if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; - /* + /* * XXX: * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact * alignment guarantees of malloc need to be nailed down, and the @@ -562,56 +587,34 @@ CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, dmat->flags); } -/* - * Utility function to load a linear buffer. lastaddrp holds state - * between invocations (for multiple-buffer loads). segp contains - * the starting segment on entrace, and the ending segment on exit. - * first indicates if this is the first invocation of this function. - */ -static __inline int -_bus_dmamap_load_buffer(bus_dma_tag_t dmat, - bus_dmamap_t map, - void *buf, bus_size_t buflen, - pmap_t pmap, - int flags, - bus_addr_t *lastaddrp, - bus_dma_segment_t *segs, - int *segp, - int first) +int +_bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, int flags) { - bus_size_t sgsize; - bus_addr_t curaddr, lastaddr, baddr, bmask; - vm_offset_t vaddr; - bus_addr_t paddr; + vm_paddr_t paddr, pendaddr; int seg; - if (map == NULL) - map = &nobounce_dmamap; - - if ((map != &nobounce_dmamap && map->pagesneeded == 0) - && ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0)) { - vm_offset_t vendaddr; - + if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " "alignment= %d", dmat->lowaddr, ptoa((vm_paddr_t)Maxmem), dmat->boundary, dmat->alignment); CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", map, &nobounce_dmamap, map->pagesneeded); + /* * Count the number of bounce pages * needed in order to complete this transfer */ - vaddr = (vm_offset_t)buf; - vendaddr = (vm_offset_t)buf + buflen; - - while (vaddr < vendaddr) { - if (pmap) - paddr = pmap_extract(pmap, vaddr); - else - paddr = pmap_kextract(vaddr); - if (run_filter(dmat, paddr) != 0) - map->pagesneeded++; - vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); + for (seg = 0; seg < sg->sg_nseg; seg++) { + paddr = sg->sg_segs[seg].ss_paddr; + pendaddr = paddr + sg->sg_segs[seg].ss_len; + while (paddr < pendaddr) { + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + run_filter(dmat, paddr) != 0) { + map->pagesneeded++; + } + paddr += (PAGE_SIZE - (paddr & PAGE_MASK)); + } } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } @@ -628,8 +631,6 @@ if (reserve_bounce_pages(dmat, map, 1) != 0) { /* Queue us for resources */ map->dmat = dmat; - map->buf = buf; - map->buflen = buflen; STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); mtx_unlock(&bounce_lock); @@ -639,74 +640,130 @@ mtx_unlock(&bounce_lock); } - vaddr = (vm_offset_t)buf; - lastaddr = *lastaddrp; - bmask = ~(dmat->boundary - 1); + return (0); +} + +/* + * Utility function to load a physical scatter/gather list. The + * caller provides a default destination if we generate a new list in + * 'segs'. However, if this tag doesn't have any restrictions, then + * the scatter/gather list from the incoming sglist can be used + * directly. Thus, this routine sets '*segp' to point to which list + * (either the list in 'sg' or the caller's list in 'segs') holds the + * DMA scatter/gather list. The caller should then pass '*segsp' to + * the driver's callback routine. + */ +static __inline int +_bus_dmamap_load_sglist(bus_dma_tag_t dmat, + bus_dmamap_t map, + struct sglist *sg, + int flags, + bus_dma_segment_t *segs, + bus_dma_segment_t **segsp, + int *nsegp) +{ + struct sglist_seg *ss; + bus_size_t curlen, sgsize; + bus_addr_t curaddr, lastaddr, baddr, bmask; + int seg, error, phys_seg, first; - for (seg = *segp; buflen > 0 ; ) { - /* - * Get the physical address for this segment. - */ - if (pmap) - curaddr = pmap_extract(pmap, vaddr); - else - curaddr = pmap_kextract(vaddr); + /* + * If this tag has no restrictions, then just use the raw + * scatter/gather list from the sglist. + */ + if ((dmat->flags & BUS_DMA_PASS_THROUGH) != 0) { + CTR0(KTR_DEV, "dma: using raw sglist directly"); + *segsp = (bus_dma_segment_t *)sg->sg_segs; + if (sg->sg_nseg > dmat->nsegments) + return (EFBIG); + *nsegp = sg->sg_nseg; + return (0); + } + *segsp = segs; - /* - * Compute the segment size, and adjust counts. - */ - sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); - if (sgsize > dmat->maxsegsz) - sgsize = dmat->maxsegsz; - if (buflen < sgsize) - sgsize = buflen; + if (map == NULL) + map = &nobounce_dmamap; - /* - * Make sure we don't cross any boundaries. - */ - if (dmat->boundary > 0) { - baddr = (curaddr + dmat->boundary) & bmask; - if (sgsize > (baddr - curaddr)) - sgsize = (baddr - curaddr); + if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { + error = _bus_dmamap_count_pages(dmat, map, sg, flags); + if (error) { + CTR1(KTR_DEV, "dma: count_pages failed - %d", error); + return (error); } + } + + first = 1; + seg = 0; + lastaddr = 0; + bmask = ~(dmat->boundary - 1); + for (phys_seg = 0, ss = sg->sg_segs; phys_seg < sg->sg_nseg; + phys_seg++, ss++) { + + curaddr = ss->ss_paddr; + curlen = ss->ss_len; + CTR3(KTR_DEV, "dma: phys[%d] = %08lx:%08lx", phys_seg, curaddr, + curlen); + while (curlen > 0) { + /* + * Compute the segment size, and adjust + * counts. + */ + sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); + if (sgsize > dmat->maxsegsz) + sgsize = dmat->maxsegsz; + if (curlen < sgsize) + sgsize = curlen; + + /* + * Make sure we don't cross any boundaries. + */ + if (dmat->boundary > 0) { + baddr = (curaddr + dmat->boundary) & bmask; + if (sgsize > (baddr - curaddr)) + sgsize = (baddr - curaddr); + } - if (map->pagesneeded != 0 && run_filter(dmat, curaddr)) - curaddr = add_bounce_page(dmat, map, vaddr, sgsize); + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) + curaddr = add_bounce_page(dmat, map, curaddr, + sgsize); - /* - * Insert chunk into a segment, coalescing with - * previous segment if possible. - */ - if (first) { - segs[seg].ds_addr = curaddr; - segs[seg].ds_len = sgsize; - first = 0; - } else { - if (curaddr == lastaddr && + /* + * Insert chunk into a segment, coalescing + * with previous segment if possible. + */ + if (first) { + CTR2(KTR_DEV, "dma: segs[0] = %08lx:%08lx", + curaddr, sgsize); + segs[seg].ds_addr = curaddr; + segs[seg].ds_len = sgsize; + first = 0; + } else if (curaddr == lastaddr && (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && (dmat->boundary == 0 || - (segs[seg].ds_addr & bmask) == (curaddr & bmask))) + (segs[seg].ds_addr & bmask) == (curaddr & bmask))) { + CTR2(KTR_DEV, "dma: segs[%d] += %08lx", seg, + sgsize); segs[seg].ds_len += sgsize; - else { + } else { if (++seg >= dmat->nsegments) - break; + /* XXX better return value here? */ + return (EFBIG); + CTR3(KTR_DEV, "dma: segs[%d] = %08lx:%08lx", + seg, curaddr, sgsize); segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } + + lastaddr = curaddr + sgsize; + curaddr += sgsize; + curlen -= sgsize; } - - lastaddr = curaddr + sgsize; - vaddr += sgsize; - buflen -= sgsize; } - *segp = seg; - *lastaddrp = lastaddr; + *nsegp = seg + 1; - /* - * Did we fit? - */ - return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ + return (0); } /* @@ -717,33 +774,145 @@ bus_size_t buflen, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { - bus_addr_t lastaddr = 0; - int error, nsegs = 0; + bus_dma_segment_t *segs; + struct sglist sg; + int error, nsegs; + + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append(&sg, buf, buflen); + if (error == 0) { + if (map != NULL) { + flags |= BUS_DMA_WAITOK; + map->bio = NULL; + map->sglist = NULL; + map->buf = buf; + map->buflen = buflen; + map->callback = callback; + map->callback_arg = callback_arg; + } + + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); + + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs); + + if (error == EINPROGRESS) { + return (error); + } + } + + if (error) + (*callback)(callback_arg, NULL, 0, error); + else + (*callback)(callback_arg, segs, nsegs, 0); + + /* + * Return ENOMEM to the caller so that it can pass it up the stack. + * This error only happens when NOWAIT is set, so deferral is disabled. + */ + if (error == ENOMEM) + return (error); + + return (0); +} + +/* + * Like bus_dmamap_load(), but maps the buffer associated with an IO + * request. + */ +int +bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bip, + bus_dmamap_callback_t *callback, void *callback_arg, int flags) +{ + bus_dma_segment_t *segs; + struct sglist sg, *sgp; + int error, nsegs; + + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + if (bip->bio_sglist != NULL) { + sgp = &sg; + error = sglist_slice(bip->bio_sglist, &sgp, bip->bio_start, + bip->bio_length, 0); + } else + error = sglist_append(&sg, bip->bio_data, bip->bio_length); + if (error == 0) { + if (map != NULL) { + flags |= BUS_DMA_WAITOK; + map->bio = bip; + map->sglist = NULL; + map->buf = NULL; + map->buflen = 0; + map->callback = callback; + map->callback_arg = callback_arg; + } + + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); + + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs); + + if (error == EINPROGRESS) { + return (error); + } + } + + if (error) + (*callback)(callback_arg, NULL, 0, error); + else + (*callback)(callback_arg, segs, nsegs, 0); + + /* + * Return ENOMEM to the caller so that it can pass it up the stack. + * This error only happens when NOWAIT is set, so deferral is disabled. + */ + if (error == ENOMEM) + return (error); + + return (0); +} + +/* + * Like bus_dmamap_load(), but maps a set of physical address ranges + * described by a scatter/gather list. + */ +int +bus_dmamap_load_sglist(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, bus_dmamap_callback_t *callback, void *callback_arg, + int flags) +{ + bus_dma_segment_t *segs; + int error, nsegs; if (map != NULL) { flags |= BUS_DMA_WAITOK; + map->bio = NULL; + map->sglist = sg; + map->buf = NULL; + map->buflen = 0; map->callback = callback; map->callback_arg = callback_arg; } - error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, - &lastaddr, dmat->segments, &nsegs, 1); + error = _bus_dmamap_load_sglist(dmat, map, sg, flags, dmat->segments, + &segs, &nsegs); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); + __func__, dmat, dmat->flags, error, nsegs); if (error == EINPROGRESS) { return (error); } if (error) - (*callback)(callback_arg, dmat->segments, 0, error); + (*callback)(callback_arg, NULL, 0, error); else - (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); + (*callback)(callback_arg, segs, nsegs, 0); /* * Return ENOMEM to the caller so that it can pass it up the stack. - * This error only happens when NOWAIT is set, so deferal is disabled. + * This error only happens when NOWAIT is set, so deferral is disabled. */ if (error == ENOMEM) return (error); @@ -751,50 +920,53 @@ return (0); } - /* - * Like _bus_dmamap_load(), but for mbufs. + * Like bus_dmamap_load(), but for mbufs. */ +static __inline int +_bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, + struct mbuf *m0, bus_dma_segment_t *segs, + bus_dma_segment_t **segsp, int *nsegs, int flags) +{ + struct sglist sg; + int error; + + M_ASSERTPKTHDR(m0); + + *nsegs = 0; + if (m0->m_pkthdr.len <= dmat->maxsize) { + flags |= BUS_DMA_NOWAIT; + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append_mbuf(&sg, m0); + if (error == 0) + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + segs, segsp, nsegs); + } else { + error = EINVAL; + } + return (error); +} + int bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, struct mbuf *m0, bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { + bus_dma_segment_t *segs; int nsegs, error; - M_ASSERTPKTHDR(m0); + error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &segs, + &nsegs, flags); - flags |= BUS_DMA_NOWAIT; - nsegs = 0; - error = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - int first = 1; - bus_addr_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - m->m_data, m->m_len, - NULL, flags, &lastaddr, - dmat->segments, &nsegs, first); - first = 0; - } - } - } else { - error = EINVAL; - } - if (error) { /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { - (*callback)(callback_arg, dmat->segments, - nsegs+1, m0->m_pkthdr.len, error); + (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); } CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); + __func__, dmat, dmat->flags, error, nsegs); return (error); } @@ -803,40 +975,20 @@ struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) { + bus_dma_segment_t *segs2; int error; - M_ASSERTPKTHDR(m0); - - flags |= BUS_DMA_NOWAIT; - *nsegs = 0; - error = 0; - if (m0->m_pkthdr.len <= dmat->maxsize) { - int first = 1; - bus_addr_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - m->m_data, m->m_len, - NULL, flags, &lastaddr, - segs, nsegs, first); - first = 0; - } - } - } else { - error = EINVAL; - } - - /* XXX FIXME: Having to increment nsegs is really annoying */ - ++*nsegs; + error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, &segs2, nsegs, + flags); + if (error == 0 && segs2 != segs) + bcopy(segs2, segs, *nsegs * sizeof(bus_dma_segment_t)); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, dmat->flags, error, *nsegs); return (error); } /* - * Like _bus_dmamap_load(), but for uios. + * Like bus_dmamap_load(), but for uios. */ int bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -844,54 +996,26 @@ bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { - bus_addr_t lastaddr = 0; - int nsegs, error, first, i; - bus_size_t resid; - struct iovec *iov; - pmap_t pmap; + bus_dma_segment_t *segs; + struct sglist sg; + int nsegs, error; flags |= BUS_DMA_NOWAIT; - resid = uio->uio_resid; - iov = uio->uio_iov; - - if (uio->uio_segflg == UIO_USERSPACE) { - KASSERT(uio->uio_td != NULL, - ("bus_dmamap_load_uio: USERSPACE but no proc")); - pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); - } else - pmap = NULL; - nsegs = 0; - error = 0; - first = 1; - for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { - /* - * Now at the first iovec to load. Load each iovec - * until we have exhausted the residual count. - */ - bus_size_t minlen = - resid < iov[i].iov_len ? resid : iov[i].iov_len; - caddr_t addr = (caddr_t) iov[i].iov_base; - - if (minlen > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - addr, minlen, pmap, flags, &lastaddr, - dmat->segments, &nsegs, first); - first = 0; + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append_uio(&sg, uio); + if (error == 0) + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); - resid -= minlen; - } - } - if (error) { /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { - (*callback)(callback_arg, dmat->segments, - nsegs+1, uio->uio_resid, error); + (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); } CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); + __func__, dmat, dmat->flags, error, nsegs); return (error); } @@ -913,6 +1037,7 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; + void *datavaddr; if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { /* @@ -923,10 +1048,11 @@ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing bounce", __func__, op, dmat, dmat->flags); + datavaddr = (void *)PHYS_TO_DMAP(bpage->datapaddr); if (op & BUS_DMASYNC_PREWRITE) { while (bpage != NULL) { - bcopy((void *)bpage->datavaddr, - (void *)bpage->vaddr, + bcopy(datavaddr, + bpage->vaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } @@ -935,8 +1061,8 @@ if (op & BUS_DMASYNC_POSTREAD) { while (bpage != NULL) { - bcopy((void *)bpage->vaddr, - (void *)bpage->datavaddr, + bcopy(bpage->vaddr, + datavaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } @@ -1059,16 +1185,16 @@ if (bpage == NULL) break; - bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, + bpage->vaddr = contigmalloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); - if (bpage->vaddr == 0) { + if (bpage->vaddr == NULL) { free(bpage, M_DEVBUF); break; } - bpage->busaddr = pmap_kextract(bpage->vaddr); + bpage->busaddr = pmap_kextract((vm_offset_t)bpage->vaddr); mtx_lock(&bounce_lock); STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); total_bpages++; @@ -1101,7 +1227,7 @@ } static bus_addr_t -add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, +add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t paddr, bus_size_t size) { struct bounce_zone *bz; @@ -1135,7 +1261,7 @@ bpage->vaddr |= vaddr & PAGE_MASK; bpage->busaddr |= vaddr & PAGE_MASK; } - bpage->datavaddr = vaddr; + bpage->datapaddr = paddr; bpage->datacount = size; STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); return (bpage->busaddr); @@ -1148,7 +1274,7 @@ struct bounce_zone *bz; bz = dmat->bounce_zone; - bpage->datavaddr = 0; + bpage->datapaddr = 0; bpage->datacount = 0; if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { /* @@ -1189,8 +1315,15 @@ mtx_unlock(&bounce_lock); dmat = map->dmat; (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); - bus_dmamap_load(map->dmat, map, map->buf, map->buflen, - map->callback, map->callback_arg, /*flags*/0); + if (map->bio != NULL) + bus_dmamap_load_bio(map->dmat, map, map->bio, + map->callback, map->callback_arg, /*flags*/0); + else if (map->sglist != NULL) + bus_dmamap_load_sglist(map->dmat, map, map->sglist, + map->callback, map->callback_arg, /*flags*/0); + else + bus_dmamap_load(map->dmat, map, map->buf, map->buflen, + map->callback, map->callback_arg, /*flags*/0); (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); mtx_lock(&bounce_lock); } --- //depot/vendor/freebsd/src/sys/dev/ata/ata-dma.c 2009/06/24 15:40:14 +++ //depot/user/jhb/bio/dev/ata/ata-dma.c 2009/09/02 18:56:14 @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -233,6 +234,8 @@ } } +#include + static void ata_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, int error) { @@ -246,6 +249,8 @@ for (i = 0; i < nsegs; i++) { prd[i].addr = htole32(segs[i].ds_addr); prd[i].count = htole32(segs[i].ds_len); + CTR3(KTR_DEV, "ata prd[%d] = %08x:%08x", i, + (uint32_t)segs[i].ds_addr, (uint32_t)segs[i].ds_len); } prd[i - 1].count |= htole32(ATA_DMA_EOT); KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); @@ -293,10 +298,15 @@ else dspa.dmatab = request->dma->sg; - if ((error = bus_dmamap_load(request->dma->data_tag, request->dma->data_map, - request->data, request->bytecount, - ch->dma.setprd, &dspa, BUS_DMA_NOWAIT)) || - (error = dspa.error)) { + if (request->bio != NULL) + error = bus_dmamap_load_bio(request->dma->data_tag, + request->dma->data_map, request->bio, + ch->dma.setprd, &dspa, BUS_DMA_NOWAIT); + else + error = bus_dmamap_load(request->dma->data_tag, + request->dma->data_map, request->data, request->bytecount, + ch->dma.setprd, &dspa, BUS_DMA_NOWAIT); + if (error != 0 || (error = dspa.error)) { device_printf(request->dev, "FAILURE - load data\n"); goto error; } --- //depot/vendor/freebsd/src/sys/geom/geom.h 2009/06/30 14:35:13 +++ //depot/user/jhb/bio/geom/geom.h 2009/09/02 18:56:14 @@ -293,6 +293,9 @@ int g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length); int g_delete_data(struct g_consumer *cp, off_t offset, off_t length); void g_print_bio(struct bio *bp); +#if defined INVARIANTS || defined(INVARIANT_SUPPORT) +void g_check_bio(struct bio *bp); +#endif /* geom_kern.c / geom_kernsim.c */ --- //depot/vendor/freebsd/src/sys/geom/geom_disk.c 2009/04/10 04:15:21 +++ //depot/user/jhb/bio/geom/geom_disk.c 2009/09/02 18:56:14 @@ -267,7 +267,10 @@ do { bp2->bio_offset += off; bp2->bio_length -= off; - bp2->bio_data += off; + if (bp->bio_data != 0) + bp2->bio_data += off; + if (bp2->bio_sglist != NULL) + bp2->bio_start += off; if (bp2->bio_length > dp->d_maxsize) { /* * XXX: If we have a stripesize we should really @@ -289,6 +292,9 @@ bp2->bio_disk = dp; devstat_start_transaction_bio(dp->d_devstat, bp2); g_disk_lock_giant(dp); +#ifdef INVARIANTS + g_check_bio(bp2); +#endif dp->d_strategy(bp2); g_disk_unlock_giant(dp); bp2 = bp3; @@ -324,6 +330,9 @@ bp2->bio_done = g_disk_done; bp2->bio_disk = dp; g_disk_lock_giant(dp); +#ifdef INVARIANTS + g_check_bio(bp2); +#endif dp->d_strategy(bp2); g_disk_unlock_giant(dp); break; --- //depot/vendor/freebsd/src/sys/geom/geom_io.c 2009/06/30 14:35:13 +++ //depot/user/jhb/bio/geom/geom_io.c 2009/09/02 18:56:14 @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -165,6 +166,8 @@ CTRSTACK(KTR_GEOM, &st, 3, 0); } #endif + if (bp->bio_sglist) + sglist_free(bp->bio_sglist); uma_zfree(biozone, bp); } @@ -180,6 +183,8 @@ bp2->bio_length = bp->bio_length; bp2->bio_offset = bp->bio_offset; bp2->bio_data = bp->bio_data; + if (bp->bio_sglist) + bp2->bio_sglist = sglist_hold(bp->bio_sglist); bp2->bio_attribute = bp->bio_attribute; /* Inherit classification info from the parent */ bp2->bio_classifier1 = bp->bio_classifier1; @@ -209,6 +214,8 @@ bp2->bio_length = bp->bio_length; bp2->bio_offset = bp->bio_offset; bp2->bio_data = bp->bio_data; + if (bp->bio_sglist) + bp2->bio_sglist = sglist_hold(bp->bio_sglist); bp2->bio_attribute = bp->bio_attribute; bp->bio_children++; #ifdef KTR @@ -413,11 +420,11 @@ #endif if (bp->bio_cmd & (BIO_READ|BIO_WRITE|BIO_GETATTR)) { - KASSERT(bp->bio_data != NULL, + KASSERT(bp->bio_data != NULL || bp->bio_sglist != NULL, ("NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd)); } if (bp->bio_cmd & (BIO_DELETE|BIO_FLUSH)) { - KASSERT(bp->bio_data == NULL, + KASSERT(bp->bio_data == NULL && bp->bio_sglist == NULL, ("non-NULL bp->data in g_io_request(cmd=%hhu)", bp->bio_cmd)); } @@ -429,6 +436,9 @@ ("wrong length %jd for sectorsize %u", bp->bio_length, cp->provider->sectorsize)); } +#ifdef INVARIANTS + g_check_bio(bp); +#endif g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d", bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd); @@ -786,3 +796,68 @@ } /* NOTREACHED */ } + +#ifdef INVARIANT_SUPPORT +void +g_check_bio(struct bio *bp) +{ + const char *cmd; + size_t space; + + space = 0; + if (bp->bio_sglist != NULL) + space = sglist_length(bp->bio_sglist); + + cmd = NULL; + switch (bp->bio_cmd) { + case BIO_GETATTR: + if (bp->bio_attribute == NULL) + panic("BIO_GETATTR with no attribute"); + if (bp->bio_length != 0 && bp->bio_data == NULL) + panic("BIO_GETATTR with no data"); + if (bp->bio_sglist) + panic("BIO_GETATTR with sglist"); + break; + case BIO_FLUSH: + if (bp->bio_attribute != NULL) + panic("BIO_FLUSH with attribute"); + if (bp->bio_data != NULL || bp->bio_length != 0 || + bp->bio_sglist != NULL) + panic("BIO_FLUSH with data"); + break; + case BIO_READ: + cmd = "BIO_READ"; + /* FALLTHROUGH */ + case BIO_WRITE: + if (cmd == NULL) + cmd = "BIO_WRITE"; + + if (bp->bio_attribute != NULL) + panic("%s with attribute", cmd); + if (bp->bio_length == 0) { + /* panic("%s with zero length", cmd); */ + } else { + if (bp->bio_data == NULL && bp->bio_sglist == NULL) + panic("%s with no data", cmd); + if (bp->bio_sglist != NULL && + (space < bp->bio_start + bp->bio_length)) + panic( + "%s sglist too short (bio_length %jd, bio_start %zd, sg %zd)", + cmd, (uintmax_t)bp->bio_length, + bp->bio_start, space); + } + break; + case BIO_DELETE: + if (bp->bio_attribute != NULL) + panic("BIO_DELETE with attribute"); + if (bp->bio_data != NULL || bp->bio_sglist != NULL) + panic("BIO_DELETE with data"); + if (bp->bio_length == 0) + panic("BIO_DELETE with zero length"); + break; + default: + panic("Unknown bio command %d", bp->bio_cmd); + break; + } +} +#endif --- //depot/vendor/freebsd/src/sys/geom/geom_vfs.c 2009/07/01 20:20:13 +++ //depot/user/jhb/bio/geom/geom_vfs.c 2009/09/02 18:56:14 @@ -32,6 +32,7 @@ #include #include #include +#include #include #include /* XXX Temporary for VFS_LOCK_GIANT */ @@ -127,6 +128,9 @@ bip->bio_done = g_vfs_done; bip->bio_caller2 = bp; bip->bio_length = bp->b_bcount; +#if 0 + bip->bio_sglist = sglist_build(bp->b_data, bp->b_bcount, M_WAITOK); +#endif g_io_request(bip, cp); } --- //depot/vendor/freebsd/src/sys/gnu/fs/xfs/FreeBSD/xfs_mountops.c 2009/05/11 15:35:16 +++ //depot/user/jhb/bio/gnu/fs/xfs/FreeBSD/xfs_mountops.c 2009/09/02 18:56:14 @@ -34,6 +34,7 @@ #include #include #include +#include #include #include @@ -468,6 +469,9 @@ bip->bio_done = xfs_geom_biodone; bip->bio_caller2 = bp; bip->bio_length = bp->b_bcount; +#if 0 + bip->bio_sglist = sglist_build(bp->b_data, bp->b_bcount, M_WAITOK); +#endif g_io_request(bip, cp); } --- //depot/vendor/freebsd/src/sys/i386/i386/busdma_machdep.c 2009/04/23 20:26:39 +++ //depot/user/jhb/bio/i386/i386/busdma_machdep.c 2009/09/02 18:56:14 @@ -28,11 +28,9 @@ __FBSDID("$FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.99 2009/04/23 20:24:19 jhb Exp $"); #include -#include -#include -#include #include #include +#include #include #include #include @@ -42,6 +40,8 @@ #include #include #include +#include +#include #include #include @@ -54,6 +54,7 @@ #include #define MAX_BPAGES 512 +#define BUS_DMA_PASS_THROUGH BUS_DMA_BUS2 #define BUS_DMA_COULD_BOUNCE BUS_DMA_BUS3 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 @@ -75,14 +76,15 @@ int map_count; bus_dma_lock_t *lockfunc; void *lockfuncarg; + struct sglist_seg *phys_segments; bus_dma_segment_t *segments; struct bounce_zone *bounce_zone; }; struct bounce_page { - vm_offset_t vaddr; /* kva of bounce buffer */ + void *vaddr; /* kva of bounce buffer */ bus_addr_t busaddr; /* Physical address */ - vm_offset_t datavaddr; /* kva of client data */ + vm_paddr_t datapaddr; /* pa of client data */ bus_size_t datacount; /* client data count */ STAILQ_ENTRY(bounce_page) links; }; @@ -121,6 +123,8 @@ int pagesneeded; int pagesreserved; bus_dma_tag_t dmat; + struct bio *bio; + struct sglist *sglist; void *buf; /* unmapped buffer pointer */ bus_size_t buflen; /* unmapped buffer length */ bus_dmamap_callback_t *callback; @@ -138,11 +142,11 @@ static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit); static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, - vm_offset_t vaddr, bus_size_t size); + vm_paddr_t paddr, bus_size_t size); static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); -int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); -int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, - void *buf, bus_size_t buflen, int flags); +static int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); +static int _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, int flags); #ifdef XEN #undef pmap_kextract @@ -171,7 +175,7 @@ || (*dmat->filter)(dmat->filterarg, paddr) != 0)) retval = 1; - dmat = dmat->parent; + dmat = dmat->parent; } while (retval == 0 && dmat != NULL); return (retval); } @@ -241,7 +245,7 @@ M_ZERO | M_NOWAIT); if (newtag == NULL) { CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", - __func__, newtag, 0, error); + __func__, newtag, 0, ENOMEM); return (ENOMEM); } @@ -257,7 +261,7 @@ newtag->nsegments = nsegments; newtag->maxsegsz = maxsegsz; newtag->flags = flags; - newtag->ref_count = 1; /* Count ourself */ + refcount_init(&newtag->ref_count, 1); /* Count ourself */ newtag->map_count = 0; if (lockfunc != NULL) { newtag->lockfunc = lockfunc; @@ -266,6 +270,14 @@ newtag->lockfunc = dflt_lock; newtag->lockfuncarg = NULL; } + newtag->phys_segments = malloc(sizeof(struct sglist_seg) * nsegments, + M_DEVBUF, M_NOWAIT); + if (newtag->phys_segments == NULL) { + CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", + __func__, newtag, 0, ENOMEM); + free(newtag, M_DEVBUF); + return (ENOMEM); + } newtag->segments = NULL; /* Take into account any restrictions imposed by our parent tag */ @@ -290,12 +302,15 @@ newtag->parent = parent->parent; } if (newtag->parent != NULL) - atomic_add_int(&parent->ref_count, 1); + refcount_acquire(&parent->ref_count); } if (newtag->lowaddr < ptoa((vm_paddr_t)Maxmem) || newtag->alignment > 1) newtag->flags |= BUS_DMA_COULD_BOUNCE; + else if (newtag->boundary == 0 && + newtag->maxsegsz == BUS_SPACE_MAXSIZE_32BIT) + newtag->flags |= BUS_DMA_PASS_THROUGH; if (((newtag->flags & BUS_DMA_COULD_BOUNCE) != 0) && (flags & BUS_DMA_ALLOCNOW) != 0) { @@ -304,6 +319,7 @@ /* Must bounce */ if ((error = alloc_bounce_zone(newtag)) != 0) { + free(newtag->phys_segments, M_DEVBUF); free(newtag, M_DEVBUF); return (error); } @@ -321,8 +337,9 @@ /* Performed initial allocation */ newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; } - + if (error != 0) { + free(newtag->phys_segments, M_DEVBUF); free(newtag, M_DEVBUF); } else { *dmat = newtag; @@ -352,10 +369,10 @@ bus_dma_tag_t parent; parent = dmat->parent; - atomic_subtract_int(&dmat->ref_count, 1); - if (dmat->ref_count == 0) { + if (refcount_release(&dmat->ref_count)) { if (dmat->segments != NULL) free(dmat->segments, M_DEVBUF); + free(dmat->phys_segments, M_DEVBUF); free(dmat, M_DEVBUF); /* * Last reference count, so @@ -513,7 +530,7 @@ if (flags & BUS_DMA_ZERO) mflags |= M_ZERO; - /* + /* * XXX: * (dmat->alignment < dmat->maxsize) is just a quick hack; the exact * alignment guarantees of malloc need to be nailed down, and the @@ -579,9 +596,8 @@ _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, void *buf, bus_size_t buflen, int flags) { - vm_offset_t vaddr; - vm_offset_t vendaddr; - bus_addr_t paddr; + vm_paddr_t paddr, pendaddr; + int seg; if ((map != &nobounce_dmamap && map->pagesneeded == 0)) { CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " @@ -589,23 +605,21 @@ dmat->boundary, dmat->alignment); CTR3(KTR_BUSDMA, "map= %p, nobouncemap= %p, pagesneeded= %d", map, &nobounce_dmamap, map->pagesneeded); + /* * Count the number of bounce pages * needed in order to complete this transfer */ - vaddr = (vm_offset_t)buf; - vendaddr = (vm_offset_t)buf + buflen; - - while (vaddr < vendaddr) { - if (pmap) - paddr = pmap_extract(pmap, vaddr); - else - paddr = pmap_kextract(vaddr); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - run_filter(dmat, paddr) != 0) { - map->pagesneeded++; + for (seg = 0; seg < sg->sg_nseg; seg++) { + paddr = sg->sg_segs[seg].ss_paddr; + pendaddr = paddr + sg->sg_segs[seg].ss_len; + while (paddr < pendaddr) { + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + run_filter(dmat, paddr) != 0) { + map->pagesneeded++; + } + paddr += (PAGE_SIZE - (paddr & PAGE_MASK)); } - vaddr += (PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK)); } CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); } @@ -622,8 +636,6 @@ if (reserve_bounce_pages(dmat, map, 1) != 0) { /* Queue us for resources */ map->dmat = dmat; - map->buf = buf; - map->buflen = buflen; STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); mtx_unlock(&bounce_lock); @@ -637,105 +649,115 @@ } /* - * Utility function to load a linear buffer. lastaddrp holds state - * between invocations (for multiple-buffer loads). segp contains - * the starting segment on entrace, and the ending segment on exit. - * first indicates if this is the first invocation of this function. + * Utility function to load a physical scatter/gather list. The + * caller provides a default destination if we generate a new list in + * 'segs'. However, if this tag doesn't have any restrictions, then + * the scatter/gather list from the incoming sglist can be used + * directly. Thus, this routine sets '*segp' to point to which list + * (either the list in 'sg' or the caller's list in 'segs') holds the + * DMA scatter/gather list. The caller should then pass '*segsp' to + * the driver's callback routine. */ static __inline int -_bus_dmamap_load_buffer(bus_dma_tag_t dmat, - bus_dmamap_t map, - void *buf, bus_size_t buflen, - pmap_t pmap, +_bus_dmamap_load_sglist(bus_dma_tag_t dmat, + bus_dmamap_t map, + struct sglist *sg, int flags, - bus_addr_t *lastaddrp, bus_dma_segment_t *segs, - int *segp, - int first) + bus_dma_segment_t **segsp, + int *nsegp) { - bus_size_t sgsize; + struct sglist_seg *ss; + bus_size_t curlen, sgsize; bus_addr_t curaddr, lastaddr, baddr, bmask; - vm_offset_t vaddr; - int seg, error; + int seg, error, phys_seg, first; + + /* + * If this tag has no restrictions, then just use the raw + * scatter/gather list from the sglist. + */ + if ((dmat->flags & BUS_DMA_PASS_THROUGH) != 0) { + *segsp = (bus_dma_segment_t *)sg->sg_segs; + if (sg->sg_nseg > dmat->nsegments) + return (EFBIG); + *nsegp = sg->sg_nseg; + return (0); + } + *segsp = segs; if (map == NULL) map = &nobounce_dmamap; if ((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) { - error = _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); + error = _bus_dmamap_count_pages(dmat, map, sg, flags); if (error) return (error); } - vaddr = (vm_offset_t)buf; - lastaddr = *lastaddrp; + first = 1; + seg = 0; + lastaddr = 0; bmask = ~(dmat->boundary - 1); + for (phys_seg = 0, ss = sg->sg_segs; phys_seg < sg->sg_nseg; + phys_seg++, ss++) { - for (seg = *segp; buflen > 0 ; ) { - /* - * Get the physical address for this segment. - */ - if (pmap) - curaddr = pmap_extract(pmap, vaddr); - else - curaddr = pmap_kextract(vaddr); + curaddr = ss->ss_paddr; + curlen = ss->ss_len; + while (curlen > 0) { + /* + * Compute the segment size, and adjust + * counts. + */ + sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); + if (sgsize > dmat->maxsegsz) + sgsize = dmat->maxsegsz; + if (curlen < sgsize) + sgsize = curlen; - /* - * Compute the segment size, and adjust counts. - */ - sgsize = PAGE_SIZE - ((u_long)curaddr & PAGE_MASK); - if (sgsize > dmat->maxsegsz) - sgsize = dmat->maxsegsz; - if (buflen < sgsize) - sgsize = buflen; + /* + * Make sure we don't cross any boundaries. + */ + if (dmat->boundary > 0) { + baddr = (curaddr + dmat->boundary) & bmask; + if (sgsize > (baddr - curaddr)) + sgsize = (baddr - curaddr); + } - /* - * Make sure we don't cross any boundaries. - */ - if (dmat->boundary > 0) { - baddr = (curaddr + dmat->boundary) & bmask; - if (sgsize > (baddr - curaddr)) - sgsize = (baddr - curaddr); - } + if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && + map->pagesneeded != 0 && run_filter(dmat, curaddr)) + curaddr = add_bounce_page(dmat, map, curaddr, + sgsize); - if (((dmat->flags & BUS_DMA_COULD_BOUNCE) != 0) && - map->pagesneeded != 0 && run_filter(dmat, curaddr)) - curaddr = add_bounce_page(dmat, map, vaddr, sgsize); - - /* - * Insert chunk into a segment, coalescing with - * previous segment if possible. - */ - if (first) { - segs[seg].ds_addr = curaddr; - segs[seg].ds_len = sgsize; - first = 0; - } else { - if (curaddr == lastaddr && + /* + * Insert chunk into a segment, coalescing + * with previous segment if possible. + */ + if (first) { + segs[seg].ds_addr = curaddr; + segs[seg].ds_len = sgsize; + first = 0; + } else if (curaddr == lastaddr && (segs[seg].ds_len + sgsize) <= dmat->maxsegsz && (dmat->boundary == 0 || (segs[seg].ds_addr & bmask) == (curaddr & bmask))) segs[seg].ds_len += sgsize; else { if (++seg >= dmat->nsegments) - break; + /* XXX better return value here? */ + return (EFBIG); segs[seg].ds_addr = curaddr; segs[seg].ds_len = sgsize; } + + lastaddr = curaddr + sgsize; + curaddr += sgsize; + curlen -= sgsize; } - - lastaddr = curaddr + sgsize; - vaddr += sgsize; - buflen -= sgsize; } - *segp = seg; - *lastaddrp = lastaddr; + *nsegp = seg + 1; - /* - * Did we fit? - */ - return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ + return (0); } /* @@ -746,33 +768,145 @@ bus_size_t buflen, bus_dmamap_callback_t *callback, void *callback_arg, int flags) { - bus_addr_t lastaddr = 0; - int error, nsegs = 0; + bus_dma_segment_t *segs; + struct sglist sg; + int error, nsegs; + + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append(&sg, buf, buflen); + if (error == 0) { + if (map != NULL) { + flags |= BUS_DMA_WAITOK; + map->bio = NULL; + map->sglist = NULL; + map->buf = buf; + map->buflen = buflen; + map->callback = callback; + map->callback_arg = callback_arg; + } + + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); + + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs); + + if (error == EINPROGRESS) { + return (error); + } + } + + if (error) + (*callback)(callback_arg, NULL, 0, error); + else + (*callback)(callback_arg, segs, nsegs, 0); + + /* + * Return ENOMEM to the caller so that it can pass it up the stack. + * This error only happens when NOWAIT is set, so deferral is disabled. + */ + if (error == ENOMEM) + return (error); + + return (0); +} + +/* + * Like bus_dmamap_load(), but maps the buffer associated with an IO + * request. + */ +int +bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bip, + bus_dmamap_callback_t *callback, void *callback_arg, int flags) +{ + bus_dma_segment_t *segs; + struct sglist sg, *sgp; + int error, nsegs; + + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + if (bip->bio_sglist != NULL) { + sgp = &sg; + error = sglist_slice(bip->bio_sglist, &sgp, bip->bio_start, + bip->bio_length, 0); + } else + error = sglist_append(&sg, bip->bio_data, bip->bio_length); + if (error == 0) { + if (map != NULL) { + flags |= BUS_DMA_WAITOK; + map->bio = bip; + map->sglist = NULL; + map->buf = NULL; + map->buflen = 0; + map->callback = callback; + map->callback_arg = callback_arg; + } + + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); + + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, nsegs); + + if (error == EINPROGRESS) { + return (error); + } + } + + if (error) + (*callback)(callback_arg, NULL, 0, error); + else + (*callback)(callback_arg, segs, nsegs, 0); + + /* + * Return ENOMEM to the caller so that it can pass it up the stack. + * This error only happens when NOWAIT is set, so deferral is disabled. + */ + if (error == ENOMEM) + return (error); + + return (0); +} + +/* + * Like bus_dmamap_load(), but maps a set of physical address ranges + * described by a scatter/gather list. + */ +int +bus_dmamap_load_sglist(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, bus_dmamap_callback_t *callback, void *callback_arg, + int flags) +{ + bus_dma_segment_t *segs; + int error, nsegs; if (map != NULL) { flags |= BUS_DMA_WAITOK; + map->bio = NULL; + map->sglist = sg; + map->buf = NULL; + map->buflen = 0; map->callback = callback; map->callback_arg = callback_arg; } - error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, NULL, flags, - &lastaddr, dmat->segments, &nsegs, 1); + error = _bus_dmamap_load_sglist(dmat, map, sg, flags, dmat->segments, + &segs, &nsegs); CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); + __func__, dmat, dmat->flags, error, nsegs); if (error == EINPROGRESS) { return (error); } if (error) - (*callback)(callback_arg, dmat->segments, 0, error); + (*callback)(callback_arg, NULL, 0, error); else - (*callback)(callback_arg, dmat->segments, nsegs + 1, 0); + (*callback)(callback_arg, segs, nsegs, 0); /* * Return ENOMEM to the caller so that it can pass it up the stack. - * This error only happens when NOWAIT is set, so deferal is disabled. + * This error only happens when NOWAIT is set, so deferral is disabled. */ if (error == ENOMEM) return (error); @@ -780,44 +914,30 @@ return (0); } - /* - * Like _bus_dmamap_load(), but for mbufs. + * Like bus_dmamap_load(), but for mbufs. */ static __inline int _bus_dmamap_load_mbuf_sg(bus_dma_tag_t dmat, bus_dmamap_t map, - struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, - int flags) + struct mbuf *m0, bus_dma_segment_t *segs, + bus_dma_segment_t **segsp, int *nsegs, int flags) { + struct sglist sg; int error; M_ASSERTPKTHDR(m0); - flags |= BUS_DMA_NOWAIT; *nsegs = 0; - error = 0; if (m0->m_pkthdr.len <= dmat->maxsize) { - int first = 1; - bus_addr_t lastaddr = 0; - struct mbuf *m; - - for (m = m0; m != NULL && error == 0; m = m->m_next) { - if (m->m_len > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - m->m_data, m->m_len, - NULL, flags, &lastaddr, - segs, nsegs, first); - first = 0; - } - } + flags |= BUS_DMA_NOWAIT; + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append_mbuf(&sg, m0); + if (error == 0) + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + segs, segsp, nsegs); } else { error = EINVAL; } - - /* XXX FIXME: Having to increment nsegs is really annoying */ - ++*nsegs; - CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, *nsegs); return (error); } @@ -827,17 +947,17 @@ bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { + bus_dma_segment_t *segs; int nsegs, error; - error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &nsegs, - flags); + error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, dmat->segments, &segs, + &nsegs, flags); if (error) { /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { - (*callback)(callback_arg, dmat->segments, - nsegs, m0->m_pkthdr.len, error); + (*callback)(callback_arg, segs, nsegs, m0->m_pkthdr.len, error); } CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", __func__, dmat, dmat->flags, error, nsegs); @@ -849,11 +969,20 @@ struct mbuf *m0, bus_dma_segment_t *segs, int *nsegs, int flags) { - return (_bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, nsegs, flags)); + bus_dma_segment_t *segs2; + int error; + + error = _bus_dmamap_load_mbuf_sg(dmat, map, m0, segs, &segs2, nsegs, + flags); + if (error == 0 && segs2 != segs) + bcopy(segs2, segs, *nsegs * sizeof(bus_dma_segment_t)); + CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", + __func__, dmat, dmat->flags, error, *nsegs); + return (error); } /* - * Like _bus_dmamap_load(), but for uios. + * Like bus_dmamap_load(), but for uios. */ int bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, @@ -861,55 +990,26 @@ bus_dmamap_callback2_t *callback, void *callback_arg, int flags) { - bus_addr_t lastaddr; - int nsegs, error, first, i; - bus_size_t resid; - struct iovec *iov; - pmap_t pmap; + bus_dma_segment_t *segs; + struct sglist sg; + int nsegs, error; flags |= BUS_DMA_NOWAIT; - resid = uio->uio_resid; - iov = uio->uio_iov; - - if (uio->uio_segflg == UIO_USERSPACE) { - KASSERT(uio->uio_td != NULL, - ("bus_dmamap_load_uio: USERSPACE but no proc")); - pmap = vmspace_pmap(uio->uio_td->td_proc->p_vmspace); - } else - pmap = NULL; - nsegs = 0; - error = 0; - first = 1; - lastaddr = (bus_addr_t) 0; - for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { - /* - * Now at the first iovec to load. Load each iovec - * until we have exhausted the residual count. - */ - bus_size_t minlen = - resid < iov[i].iov_len ? resid : iov[i].iov_len; - caddr_t addr = (caddr_t) iov[i].iov_base; - - if (minlen > 0) { - error = _bus_dmamap_load_buffer(dmat, map, - addr, minlen, pmap, flags, &lastaddr, - dmat->segments, &nsegs, first); - first = 0; + sglist_init(&sg, dmat->nsegments, dmat->phys_segments); + error = sglist_append_uio(&sg, uio); + if (error == 0) + error = _bus_dmamap_load_sglist(dmat, map, &sg, flags, + dmat->segments, &segs, &nsegs); - resid -= minlen; - } - } - if (error) { /* force "no valid mappings" in callback */ - (*callback)(callback_arg, dmat->segments, 0, 0, error); + (*callback)(callback_arg, NULL, 0, 0, error); } else { - (*callback)(callback_arg, dmat->segments, - nsegs+1, uio->uio_resid, error); + (*callback)(callback_arg, segs, nsegs, uio->uio_resid, error); } CTR5(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d nsegs %d", - __func__, dmat, dmat->flags, error, nsegs + 1); + __func__, dmat, dmat->flags, error, nsegs); return (error); } @@ -931,6 +1031,7 @@ _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) { struct bounce_page *bpage; + void *datavaddr; if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { /* @@ -941,10 +1042,12 @@ CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " "performing bounce", __func__, op, dmat, dmat->flags); + /* Map the client's page using a temporary per-CPU mapping. */ + datavaddr = pmap_map_dma_page(bpage->datapaddr); if (op & BUS_DMASYNC_PREWRITE) { while (bpage != NULL) { - bcopy((void *)bpage->datavaddr, - (void *)bpage->vaddr, + bcopy(datavaddr, + bpage->vaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } @@ -953,13 +1056,14 @@ if (op & BUS_DMASYNC_POSTREAD) { while (bpage != NULL) { - bcopy((void *)bpage->vaddr, - (void *)bpage->datavaddr, + bcopy(bpage->vaddr, + datavaddr, bpage->datacount); bpage = STAILQ_NEXT(bpage, links); } dmat->bounce_zone->total_bounced++; } + pmap_release_dma_page(); } } @@ -1077,16 +1181,16 @@ if (bpage == NULL) break; - bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, + bpage->vaddr = contigmalloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); - if (bpage->vaddr == 0) { + if (bpage->vaddr == NULL) { free(bpage, M_DEVBUF); break; } - bpage->busaddr = pmap_kextract(bpage->vaddr); + bpage->busaddr = pmap_kextract((vm_offset_t)bpage->vaddr); mtx_lock(&bounce_lock); STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); total_bpages++; @@ -1119,7 +1223,7 @@ } static bus_addr_t -add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, +add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t paddr, bus_size_t size) { struct bounce_zone *bz; @@ -1153,7 +1257,7 @@ bpage->vaddr |= vaddr & PAGE_MASK; bpage->busaddr |= vaddr & PAGE_MASK; } - bpage->datavaddr = vaddr; + bpage->datapaddr = paddr; bpage->datacount = size; STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); return (bpage->busaddr); @@ -1166,7 +1270,7 @@ struct bounce_zone *bz; bz = dmat->bounce_zone; - bpage->datavaddr = 0; + bpage->datapaddr = 0; bpage->datacount = 0; if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { /* @@ -1207,8 +1311,15 @@ mtx_unlock(&bounce_lock); dmat = map->dmat; (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_LOCK); - bus_dmamap_load(map->dmat, map, map->buf, map->buflen, - map->callback, map->callback_arg, /*flags*/0); + if (map->bio != NULL) + bus_dmamap_load_bio(map->dmat, map, map->bio, + map->callback, map->callback_arg, /*flags*/0); + else if (map->sglist != NULL) + bus_dmamap_load_sglist(map->dmat, map, map->sglist, + map->callback, map->callback_arg, /*flags*/0); + else + bus_dmamap_load(map->dmat, map, map->buf, map->buflen, + map->callback, map->callback_arg, /*flags*/0); (dmat->lockfunc)(dmat->lockfuncarg, BUS_DMA_UNLOCK); mtx_lock(&bounce_lock); } --- //depot/vendor/freebsd/src/sys/i386/i386/pmap.c 2009/09/02 16:50:13 +++ //depot/user/jhb/bio/i386/i386/pmap.c 2009/09/02 18:56:14 @@ -238,8 +238,10 @@ struct mtx lock; pt_entry_t *CMAP1; pt_entry_t *CMAP2; + pt_entry_t *CMAPDMA; caddr_t CADDR1; caddr_t CADDR2; + caddr_t CADDRDMA; }; static struct sysmaps sysmaps_pcpu[MAXCPU]; pt_entry_t *CMAP1 = 0; @@ -412,12 +414,14 @@ /* * CMAP1/CMAP2 are used for zeroing and copying pages. * CMAP3 is used for the idle process page zeroing. + * CMAPDMA is used by bus dma to map data pages for bouncing. */ for (i = 0; i < MAXCPU; i++) { sysmaps = &sysmaps_pcpu[i]; mtx_init(&sysmaps->lock, "SYSMAPS", NULL, MTX_DEF); SYSMAP(caddr_t, sysmaps->CMAP1, sysmaps->CADDR1, 1) SYSMAP(caddr_t, sysmaps->CMAP2, sysmaps->CADDR2, 1) + SYSMAP(caddr_t, sysmaps->CMAPDMA, sysmaps->CADDRDMA, 1) } SYSMAP(caddr_t, CMAP1, CADDR1, 1) SYSMAP(caddr_t, CMAP3, CADDR3, 1) @@ -3897,6 +3901,46 @@ } /* + * pmap_map_dma_page() uses CMAPDMA to map a given page + * so it can be copied to a bounce page (or vice versa). + * To avoid locks, this enters a critical section. When + * the copy is complete, the pmap_release_dma_page() must + * be called to release the mapping and exit the critical + * section. + */ +void * +pmap_map_dma_page(vm_paddr_t datapaddr) +{ + struct sysmaps *sysmaps; + vm_offset_t offset; + + offset = datapaddr & PAGE_MASK; + datapaddr &= ~PAGE_MASK; + critical_enter(); + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + if (*sysmaps->CMAPDMA) + panic("pmap_map_dma_page: CMAPDMA busy"); + invlpg((u_int)sysmaps->CADDRDMA); + *sysmaps->CMAPDMA = PG_V | PG_RW | datapaddr | PG_A | PG_M; + return (sysmaps->CADDRDMA + offset); +} + +/* + * pmap_release_dma_page() removes a mapping in CMAPDMA + * and exits the critical section from an earlier call to + * pmap_map_dma_page(). + */ +void +pmap_release_dma_page(void) +{ + struct sysmaps *sysmaps; + + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + *sysmaps->CMAPDMA = 0; + critical_exit(); +} + +/* * Returns true if the pmap's pv is one of the first * 16 pvs linked to from this page. This count may * be changed upwards or downwards in the future; it --- //depot/vendor/freebsd/src/sys/i386/include/pmap.h 2009/08/31 17:45:14 +++ //depot/user/jhb/bio/i386/include/pmap.h 2009/09/02 18:56:14 @@ -486,6 +486,8 @@ void pmap_invalidate_all(pmap_t); void pmap_invalidate_cache(void); void pmap_invalidate_cache_range(vm_offset_t, vm_offset_t); +void *pmap_map_dma_page(vm_paddr_t); +void pmap_release_dma_page(void); #endif /* _KERNEL */ --- //depot/vendor/freebsd/src/sys/kern/kern_physio.c 2005/01/06 23:37:37 +++ //depot/user/jhb/bio/kern/kern_physio.c 2008/10/10 22:14:31 @@ -25,14 +25,25 @@ #include #include #include +#include #include +#include +#include #include +#include + #include #include -int -physio(struct cdev *dev, struct uio *uio, int ioflag) +int sglist_physio; +SYSCTL_INT(_kern, OID_AUTO, physio_sglist, CTLFLAG_RW, &sglist_physio, 0, + "Use S/G lists for physio"); + +#include + +static int +physio_kva(struct cdev *dev, struct uio *uio, int ioflag) { int i; int error; @@ -88,6 +99,8 @@ bp->b_blkno = btodb(bp->b_offset); + CTR2(KTR_DEV, "physio_kva: (%p, %d)", bp->b_data, + bp->b_bcount); if (uio->uio_segflg == UIO_USERSPACE) if (vmapbuf(bp) < 0) { error = EFAULT; @@ -121,3 +134,129 @@ PRELE(curproc); return (error); } + +static int +physio_sglist(struct cdev *dev, struct uio *uio, int ioflag) +{ + struct cdevsw *csw; + struct sglist *sg; + struct bio *bip; + int error, i; + size_t iolen; + + /* Keep the process UPAGES from being swapped. XXX: why ? */ + PHOLD(curproc); + + error = 0; + + /* XXX: sanity check */ + if (dev->si_iosize_max < PAGE_SIZE) { + printf("WARNING: %s si_iosize_max=%d, using DFLTPHYS.\n", + devtoname(dev), dev->si_iosize_max); + dev->si_iosize_max = DFLTPHYS; + } + + /* XXX: Better value? */ + sg = sglist_alloc(MIN(dev->si_iosize_max, uio->uio_resid) / PAGE_SIZE + + 1, M_WAITOK); + for (i = 0; i < uio->uio_iovcnt; i++) { + while (uio->uio_iov[i].iov_len) { + /* Don't exceed drivers iosize limit */ + iolen = uio->uio_iov[i].iov_len; + if (iolen > dev->si_iosize_max) + iolen = dev->si_iosize_max; + + bip = g_alloc_bio(); + if (uio->uio_rw == UIO_READ) + bip->bio_cmd = BIO_READ; + else + bip->bio_cmd = BIO_WRITE; + bip->bio_offset = uio->uio_offset; + bip->bio_length = iolen; + bip->bio_bcount = iolen; /* XXX */ + bip->bio_dev = dev; + + /* + * Wire the pages if this is a user request. Then + * build the scatter/gather list. + */ + if (uio->uio_segflg == UIO_USERSPACE) { + error = vslock(uio->uio_iov[i].iov_base, iolen); + if (error) { + g_destroy_bio(bip); + goto doerror; + } + error = sglist_append_user(sg, + uio->uio_iov[i].iov_base, iolen, + uio->uio_td); + } else + error = sglist_append(sg, + uio->uio_iov[i].iov_base, iolen); + KASSERT(error == 0, ("sglist_append failed")); + bip->bio_sglist = sglist_hold(sg); + { + int j; + + CTR2(KTR_DEV, "physio_sglist: (%p, %d)", + uio->uio_iov[i].iov_base, iolen); + for (j = 0; j < bip->bio_sglist->sg_nseg; j++) + CTR3(KTR_DEV, "sgl[%d]: %08lx:%08x", + j, (long)bip->bio_sglist->sg_segs[j].ss_paddr, + bip->bio_sglist->sg_segs[j].ss_len); + } + + KASSERT(dev->si_refcount > 0, + ("dev_strategy on un-referenced struct cdev *(%s)", + devtoname(dev))); + csw = dev_refthread(dev); + if (csw == NULL) { + vsunlock(uio->uio_iov[i].iov_base, iolen); + g_destroy_bio(bip); + error = ENXIO; + goto doerror; + } + (*csw->d_strategy)(bip); + dev_relthread(dev); + + if (uio->uio_rw == UIO_READ) + biowait(bip, "physrd"); + else + biowait(bip, "physwr"); + + if (uio->uio_segflg == UIO_USERSPACE) + vsunlock(uio->uio_iov[i].iov_base, iolen); + + iolen = bip->bio_completed; + if (iolen == 0 && bip->bio_error == 0) { + g_destroy_bio(bip); + goto doerror; /* EOF */ + } + uio->uio_iov[i].iov_len -= iolen; + uio->uio_iov[i].iov_base = + (char *)uio->uio_iov[i].iov_base + iolen; + uio->uio_resid -= iolen; + uio->uio_offset += iolen; + + error = bip->bio_error; + g_destroy_bio(bip); + if (error) { + goto doerror; + } + sglist_reset(sg); + } + } +doerror: + sglist_free(sg); + PRELE(curproc); + return (error); +} + +int +physio(struct cdev *dev, struct uio *uio, int ioflag) +{ + + if (sglist_physio) + return (physio_sglist(dev, uio, ioflag)); + else + return (physio_kva(dev, uio, ioflag)); +} --- //depot/vendor/freebsd/src/sys/kern/vfs_bio.c 2009/07/19 20:30:14 +++ //depot/user/jhb/bio/kern/vfs_bio.c 2009/09/02 18:56:14 @@ -58,6 +58,7 @@ #include #include #include +#include #include #include #include @@ -3186,8 +3187,17 @@ panic("b_iocmd botch"); for (;;) { bip = g_new_bio(); - if (bip != NULL) + if (bip != NULL) { +#if 0 + bip->bio_sglist = sglist_build(bp->b_data, bp->b_bcount, + M_NOWAIT); + if (bip->bio_sglist != NULL) + break; + g_destroy_bio(bip); +#else break; +#endif + } /* Try again later */ tsleep(&bp, PRIBIO, "dev_strat", hz/10); } --- //depot/vendor/freebsd/src/sys/sys/bio.h 2009/06/11 10:00:15 +++ //depot/user/jhb/bio/sys/bio.h 2009/09/02 18:56:14 @@ -42,6 +42,7 @@ struct disk; struct bio; +struct sglist; /* Empty classifier tag, to prevent further classification. */ #define BIO_NOTCLASSIFIED (void *)(~0UL) @@ -60,7 +61,9 @@ struct disk *bio_disk; /* Valid below geom_disk.c only */ off_t bio_offset; /* Offset into file. */ long bio_bcount; /* Valid bytes in buffer. */ + size_t bio_start; /* Offset into buffer. (XXX sgl only) */ caddr_t bio_data; /* Memory, superblocks, indirect etc. */ + struct sglist *bio_sglist; /* Scatter/gather list. */ int bio_error; /* Errno for BIO_ERROR. */ long bio_resid; /* Remaining I/O in bytes. */ void (*bio_done)(struct bio *); @@ -121,6 +124,8 @@ struct bio *insert_point; }; +extern int sglist_physio; + void biodone(struct bio *bp); void biofinish(struct bio *bp, struct devstat *stat, int error); int biowait(struct bio *bp, const char *wchan); --- //depot/vendor/freebsd/src/sys/sys/bus_dma.h 2009/02/08 23:00:22 +++ //depot/user/jhb/bio/sys/bus_dma.h 2009/09/02 18:56:14 @@ -110,7 +110,9 @@ #define BUS_DMA_KEEP_PG_OFFSET 0x400 /* Forwards needed by prototypes below. */ +struct bio; struct mbuf; +struct sglist; struct uio; /* @@ -231,6 +233,20 @@ void *callback_arg, int flags); /* + * Like bus_dmamap_load but takes a bio instead. + */ +int bus_dmamap_load_bio(bus_dma_tag_t dmat, bus_dmamap_t map, struct bio *bip, + bus_dmamap_callback_t *callback, void *callback_arg, + int flags); + +/* + * Like bus_dmamap_load but takes a sglist instead. + */ +int bus_dmamap_load_sglist(bus_dma_tag_t dmat, bus_dmamap_t map, + struct sglist *sg, bus_dmamap_callback_t *callback, + void *callback_arg, int flags); + +/* * Like bus_dmamap_load but for mbufs. Note the use of the * bus_dmamap_callback2_t interface. */