diff --git a/sys/arm/conf/DB-88F5XXX-SATA b/sys/arm/conf/DB-88F5XXX-SATA index fe0ad02..fab44cc 100644 --- a/sys/arm/conf/DB-88F5XXX-SATA +++ b/sys/arm/conf/DB-88F5XXX-SATA @@ -43,6 +41,12 @@ options DIAGNOSTIC #options INVARIANTS #Enable calls of extra sanity checking #options INVARIANT_SUPPORT #Extra sanity checks of internal structures, required by INVARIANTS options KDB +# options KTR +# options KTR_VERBOSE +# options KTR_ENTRIES=8192 +# options KTR_MASK=KTR_LOCK +# options KTR_COMPILE=KTR_LOCK + options WITNESS #Enable checks to detect deadlocks and cycles options WITNESS_SKIPSPIN #Don't run witness on spinlocks for speed #options WITNESS_KDB diff --git a/sys/arm/mv/mvsata.c b/sys/arm/mv/mvsata.c index a8934e6..7434544 100644 --- a/sys/arm/mv/mvsata.c +++ b/sys/arm/mv/mvsata.c @@ -45,19 +45,25 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include "mvreg.h" +#include "mvvar.h" #include "mvsata_reg.h" /* Identification section. */ struct mvsata_softc { device_t dev; - struct resource *mem_res; - bus_space_tag_t mem_res_bustag; + unsigned int version; + unsigned int edma_qlen; + uint32_t edma_reqis_mask; + uint32_t edma_resos_mask; + struct resource *mem_res; + bus_space_tag_t mem_res_bustag; bus_space_handle_t mem_res_bushdl; - struct resource *irq_res; + struct resource *irq_res; void *irq_cookiep; struct { void (*function)(void *); @@ -65,18 +71,12 @@ struct mvsata_softc { } interrupt[MVSATA_CHAN_NUM]; }; -struct ata_marvell_response { - u_int16_t tag; - u_int8_t edma_status; - u_int8_t dev_status; - u_int32_t timestamp; -}; - +/* Controller functions */ static int mvsata_probe(device_t dev); static int mvsata_attach(device_t dev); static int mvsata_detach(device_t dev); -static void ata_mvsata_intr(void*); -static struct resource * mvsata_alloc_resource(device_t dev, device_t child, +static void mvsata_intr(void*); +static struct resource * mvsata_alloc_resource(device_t dev, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags); static int mvsata_release_resource(device_t dev, device_t child, int type, int rid, struct resource *r); @@ -85,16 +85,26 @@ static int mvsata_setup_intr(device_t dev, device_t child, driver_intr_t *function, void *argument, void **cookiep); static int mvsata_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie); -static int mvsata_edma_begin_transaction(struct ata_request *request); -static int mvsata_edma_end_transaction(struct ata_request *request); -static int mvsata_edma_status(device_t dev); -/* Missing declarations. */ -void ata_sata_phy_check_events(device_t dev); /* From ata-sata.c. */ -void ata_dmainit(device_t); /* From ata-pci.h. */ +/* Channel functions */ +static int mvsata_channel_probe(device_t dev); +static int mvsata_channel_attach(device_t dev); +static int mvsata_channel_detach(device_t dev); +static int mvsata_channel_begin_transaction(struct ata_request *request); +static int mvsata_channel_end_transaction(struct ata_request *request); +static int mvsata_channel_status(device_t dev); +static void mvsata_channel_setmode(device_t parent, device_t dev); +static void mvsata_channel_reset(device_t dev); +static void mvsata_channel_dmasetprd(void *xsc, bus_dma_segment_t *segs, + int nsegs, int error); + +/* EDMA functions */ +static int mvsata_edma_ctrl(device_t dev, int on); +static int mvsata_edma_is_running(device_t); static device_method_t mvsata_methods[] = { - DEVMETHOD(device_probe, mvsata_probe), + /* Device method */ + DEVMETHOD(device_probe, mvsata_probe), DEVMETHOD(device_attach, mvsata_attach), DEVMETHOD(device_detach, mvsata_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), @@ -103,7 +113,7 @@ static device_method_t mvsata_methods[] = { /* ATA bus methods. */ DEVMETHOD(bus_alloc_resource, mvsata_alloc_resource), - DEVMETHOD(bus_release_resource, mvsata_release_resource), + DEVMETHOD(bus_release_resource, mvsata_release_resource), DEVMETHOD(bus_activate_resource, bus_generic_activate_resource), DEVMETHOD(bus_deactivate_resource, bus_generic_deactivate_resource), DEVMETHOD(bus_setup_intr, mvsata_setup_intr), @@ -117,13 +127,6 @@ static driver_t mvsata_driver = { sizeof(struct mvsata_softc), }; -struct ata_marvell_dma_prdentry { - u_int32_t addrlo; - u_int32_t count; - u_int32_t addrhi; - u_int32_t reserved; -}; - devclass_t mvsata_devclass; DRIVER_MODULE(mvsata, mbus, mvsata_driver, mvsata_devclass, 0, 0); @@ -133,83 +136,113 @@ MODULE_DEPEND(mvsata, ata, 1, 1, 1); static int mvsata_probe(device_t dev) { + struct mvsata_softc *sc = device_get_softc(dev); + uint32_t d, r; + + soc_id(&d, &r); + + switch(d) { + case MV_DEV_88F5182: + sc->version = 1; + sc->edma_qlen = 128; + break; - /* No need to enumerate anything. */ - device_set_desc(dev, "Marvell Integrated SATA controller"); + default: + sc->version = 2; + sc->edma_qlen = 32; + break; + } + + sc->edma_reqis_mask = (sc->edma_qlen - 1) << MVSATA_EDMA_REQIS_OFS; + sc->edma_resos_mask = (sc->edma_qlen - 1) << MVSATA_EDMA_RESOS_OFS; + + device_set_desc(dev, "Marvell Integrated SATA Controller"); return (0); } static int mvsata_attach(device_t dev) { - struct mvsata_softc *sc; + struct mvsata_softc *sc = device_get_softc(dev); int mem_id, irq_id, error, i; device_t ata_chan; + uint32_t reg; - sc = device_get_softc(dev); sc->dev = dev; mem_id = 0; irq_id = 0; + /* Allocate resources */ sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &mem_id, RF_ACTIVE); if (sc->mem_res == NULL) { - device_printf(dev, "could not allocate memory\n"); + device_printf(dev, "Could not allocate memory.\n"); return (ENOMEM); } sc->mem_res_bustag = rman_get_bustag(sc->mem_res); sc->mem_res_bushdl = rman_get_bushandle(sc->mem_res); KASSERT(sc->mem_res_bustag && sc_mem_res_bushdl, - "cannot get bus handle or tag"); + "Cannot get bus handle or tag."); sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &irq_id, RF_ACTIVE); if (sc->irq_res == NULL) { - device_printf(dev, "could not allocate IRQ\n"); + device_printf(dev, "Could not allocate IRQ.\n"); error = ENOMEM; goto err; } error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_BIO | INTR_MPSAFE | INTR_ENTROPY, - NULL, ata_mvsata_intr, sc, &sc->irq_cookiep); - if (error != 0) { - device_printf(dev, "could not setup interrupt\n"); - goto err; - } - - /* Attach myself. */ - error = bus_generic_attach(dev); + NULL, mvsata_intr, sc, &sc->irq_cookiep); if (error != 0) { - device_printf(dev, "could not attach to bus\n"); + device_printf(dev, "Could not setup interrupt.\n"); goto err; } - /* We have two SATA ports, so attach two channels. */ - for(i=0; i<2; i++) { + /* Attach channels */ + for (i = 0; i < MVSATA_CHAN_NUM; i++) { ata_chan = device_add_child(dev, "ata", devclass_find_free_unit(ata_devclass, 0)); + if (!ata_chan) { - device_printf(dev, "cannot add channel %d\n", i); - error = 1; + device_printf(dev, "Cannot add channel %d.\n", i); + error = ENOMEM; + goto err; } } + /* Disable interrupt coalescing */ + reg = MVSATA_INL(sc, MVSATA_CR); + for (i = 0; i < MVSATA_CHAN_NUM; i++) + reg |= MVSATA_CR_COALDIS(i); + + /* Disable DMA byte swapping */ + if (sc->version == 2) + reg |= MVSATA_CR_NODMABS | MVSATA_CR_NOEDMABS | + MVSATA_CR_NOPRDPBS; + + MVSATA_OUTL(sc, MVSATA_CR, reg); + + /* Clear and mask all interrupts */ + MVSATA_OUTL(sc, MVSATA_ICR, 0); + MVSATA_OUTL(sc, MVSATA_MIMR, 0); + + return(bus_generic_attach(dev)); + err: - if (error) - mvsata_detach(dev); + mvsata_detach(dev); return (error); } static int mvsata_detach(device_t dev) { - struct mvsata_softc *sc; + struct mvsata_softc *sc = device_get_softc(dev); - sc = device_get_softc(dev); if (device_is_attached(dev)) - bus_generic_detach(sc->dev); + bus_generic_detach(dev); if (sc->mem_res != NULL) { bus_release_resource(dev, SYS_RES_MEMORY, @@ -223,6 +256,7 @@ mvsata_detach(device_t dev) rman_get_rid(sc->irq_res), sc->irq_res); sc->irq_res = NULL; } + return (0); } @@ -230,12 +264,10 @@ static struct resource * mvsata_alloc_resource(device_t dev, device_t child, int type, int *rid, u_long start, u_long end, u_long count, u_int flags) { - struct mvsata_softc *sc; + struct mvsata_softc *sc = device_get_softc(dev); - sc = device_get_softc(dev); - device_printf(dev, "allocating resource"); KASSERT(type == SYS_RES_IRQ && *rid == ATA_IRQ_RID, - ("illegal resource request (type %u rid %u)", + ("Illegal resource request (type %u, rid %u).", type, *rid)); return (sc->irq_res); @@ -247,7 +279,9 @@ mvsata_release_resource(device_t dev, device_t child, int type, int rid, { KASSERT(type == SYS_RES_IRQ && rid == ATA_IRQ_RID, - ("Strange type %u rid %u while releasing resource", type, rid)); + ("Strange type %u and/or rid %u while releasing resource.", type, + rid)); + return (0); } @@ -256,19 +290,18 @@ mvsata_setup_intr(device_t dev, device_t child, struct resource *irq, int flags, driver_filter_t *filt, driver_intr_t *function, void *argument, void **cookiep) { - struct mvsata_softc *sc; - int unit; + struct mvsata_softc *sc = device_get_softc(dev); + struct ata_channel *ch = device_get_softc(child); - sc = device_get_softc(dev); - unit = ((struct ata_channel *)device_get_softc(child))->unit; if (filt != NULL) { - device_printf(dev, "we cannot use a filter here\n"); - return (EINVAL); + device_printf(dev, "Filter interrupts are not supported.\n"); + return (EINVAL); } - sc->interrupt[unit].function = function; - sc->interrupt[unit].argument = argument; + sc->interrupt[ch->unit].function = function; + sc->interrupt[ch->unit].argument = argument; *cookiep = sc; + return (0); } @@ -276,25 +309,21 @@ static int mvsata_teardown_intr(device_t dev, device_t child, struct resource *irq, void *cookie) { - struct mvsata_softc *sc; - int unit; + struct mvsata_softc *sc = device_get_softc(dev); + struct ata_channel *ch = device_get_softc(child); - sc = device_get_softc(dev); - unit = ((struct ata_channel *)device_get_softc(child))->unit; + sc->interrupt[ch->unit].function = NULL; + sc->interrupt[ch->unit].argument = NULL; - sc->interrupt[unit].function = NULL; - sc->interrupt[unit].argument = NULL; return (0); } static void -ata_mvsata_intr(void *xsc) +mvsata_intr(void *xsc) { - struct mvsata_softc *sc; + struct mvsata_softc *sc = xsc; int unit; - sc = xsc; - /* * Behave like ata_generic_intr() for PCI controllers. * Simply invoke ISRs on all channels. @@ -303,355 +332,414 @@ ata_mvsata_intr(void *xsc) if (sc->interrupt[unit].function != NULL) sc->interrupt[unit].function( sc->interrupt[unit].argument); + } static int mvsata_channel_probe(device_t dev) { - device_set_desc_copy(dev, "MVSATA channel"); - /* ata_probe unconditionally returns 0. */ + device_set_desc(dev, "Marvell Integrated SATA Channel"); return (ata_probe(dev)); } -static void -ata_marvell_edma_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, - int error) -{ - struct ata_dmasetprd_args *args = xsc; - struct ata_marvell_dma_prdentry *prd = args->dmatab; - int i; - - KASSERT(nsegs <= ATA_DMA_ENTRIES, ("too many DMA segment entries\n")); - - if ((args->error = error)) - return; - - for (i = 0; i < nsegs; i++) { - prd[i].addrlo = htole32(segs[i].ds_addr); - prd[i].count = htole32(segs[i].ds_len); - prd[i].addrhi = htole32((uintmax_t)segs[i].ds_addr >> 32); - } - prd[nsegs - 1].count |= htole32(ATA_DMA_EOT); - args->nsegs = nsegs; -} - -/* - * Switches EDMA on or off. - * If is_on == TRUE, then EDMA is switched on; - * otherwise off. - * - * Return values: - * 1 if EDMA was switched on/off and timeot was not reached; - * 0 otherwise. - */ -static int -mvsata_edma_on_off(device_t dev, boolean_t is_on) -{ - struct ata_channel *ch; - struct mvsata_softc *sc; - u_int32_t wdog; - u_int32_t val; - - ch = device_get_softc(dev); - sc = device_get_softc(device_get_parent(dev)); - - if (is_on) - val = 0x1; - else - val = 0x2; - - MVSATA_OUTL(MVSATA_EDMA_REG_CMD, val); - - /* - * Ensure write operation will complete before - * we attempt to read from the same localtion. - * We flush whole region of current ata channel. - */ - bus_space_barrier(sc->mem_res_bustag, sc->mem_res_bushdl, - EDMA_REGISTERS_OFFSET + ((ch)->unit - 1) * EDMA_REGISTERS_SIZE, - EDMA_REGISTERS_SIZE, BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE); - wdog = 0; - /* - * Wait until chip switches EDMA on or off. - */ - if (is_on) - while (wdog < MVSATA_WAIT_ATTEMPTS && - !(MVSATA_INL(MVSATA_EDMA_REG_CMD) & 0x1)) { - DELAY(10); - wdog++; - } - else - while (wdog < MVSATA_WAIT_ATTEMPTS && - (MVSATA_INL(MVSATA_EDMA_REG_CMD) & 0x1)) { - DELAY(10); - wdog++; - } - if (wdog == MVSATA_WAIT_ATTEMPTS) { - device_printf(dev, "cannot %s EDMA", - is_on ? "enable" : "disable"); - return (0); - } - - return (1); -} - static int mvsata_channel_attach(device_t dev) { - struct ata_channel *ch; - struct mvsata_softc *sc; - u_int32_t work; - int i; + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); + uint64_t work; + int error, i; - ch = device_get_softc(dev); - sc = device_get_softc(device_get_parent(dev)); + if (ch->attached) + return (0); + ch->dev = dev; ch->unit = device_get_unit(dev); ch->flags |= ATA_USE_16BIT | ATA_NO_SLAVE; - ch->dma.work_bus = 0; - ata_dmainit(dev); - KASSERT(ch->dma.work_bus && ch->dma.load && ch->dma.unload, - "DMA init error!"); - - ch->dma.setprd = ata_marvell_edma_dmasetprd; - - /* Chip does not reliably do 64K DMA transfers. */ - ch->dma.max_iosize = 64 * DEV_BSIZE; - - work = ch->dma.work_bus; - - /* Clear work area. */ - KASSERT(REQ_QUEUE_SIZE + RSP_QUEUE_SIZE <= ch->dma.max_iosize, - ("Insufficient DMA memory for request/response queues\n")); - bzero(ch->dma.work, REQ_QUEUE_SIZE + RSP_QUEUE_SIZE); - /* Set legacy ATA resources. */ for (i = ATA_DATA; i <= ATA_COMMAND; i++) { ch->r_io[i].res = sc->mem_res; - ch->r_io[i].offset = ATA_MV_EDMA_BASE(ch) + - MVSATA_SHADOWR_BASE + (i << 2); + ch->r_io[i].offset = MVSATA_SHADOWR_BASE(ch->unit) + (i << 2); } ch->r_io[ATA_CONTROL].res = sc->mem_res; - ch->r_io[ATA_CONTROL].offset = ATA_MV_EDMA_BASE(ch) + - MVSATA_SHADOWR_BASE + MVSATA_SHADOWR_CONTROL; + ch->r_io[ATA_CONTROL].offset = MVSATA_SHADOWR_CONTROL(ch->unit); ch->r_io[ATA_IDX_ADDR].res = sc->mem_res; ata_default_registers(dev); /* Set SATA resources. */ ch->r_io[ATA_SSTATUS].res = sc->mem_res; - ch->r_io[ATA_SSTATUS].offset = ATA_MV_EDMA_BASE(ch) + - MVSATA_SATA_SSTATUS; + ch->r_io[ATA_SSTATUS].offset = MVSATA_SATA_SSTATUS(ch->unit); ch->r_io[ATA_SERROR].res = sc->mem_res; - ch->r_io[ATA_SERROR].offset = ATA_MV_EDMA_BASE(ch) + - MVSATA_SATA_SERROR; + ch->r_io[ATA_SERROR].offset = MVSATA_SATA_SERROR(ch->unit); ch->r_io[ATA_SCONTROL].res = sc->mem_res; - ch->r_io[ATA_SCONTROL].offset = ATA_MV_EDMA_BASE(ch) + - MVSATA_SATA_SCONTROL; + ch->r_io[ATA_SCONTROL].offset = MVSATA_SATA_SCONTROL(ch->unit); ata_generic_hw(dev); - ch->hw.begin_transaction = mvsata_edma_begin_transaction; - ch->hw.end_transaction = mvsata_edma_end_transaction; - ch->hw.status = mvsata_edma_status; + ch->hw.begin_transaction = mvsata_channel_begin_transaction; + ch->hw.end_transaction = mvsata_channel_end_transaction; + ch->hw.status = mvsata_channel_status; - /* - * EDMA activation magic -- see p.78 of User Manual. - */ - mvsata_edma_on_off(dev, 0); /* Disable EDMA. */ - /* Set configuration to non-queued 128 elements depth queue. */ - MVSATA_OUTL(MVSATA_EDMA_REG_CFG, ( 1<<19 ) ); + /* Set DMA resources */ + ata_dmainit(dev); + ch->dma.setprd = mvsata_channel_dmasetprd; + + /* Clear work area */ + KASSERT(sc->edma_qlen * (sizeof(struct mvsata_crqb) + + sizeof(struct mvsata_crpb)) <= ch->dma.max_iosize, + ("Insufficient DMA memory for request/response queues.\n")); + bzero(ch->dma.work, sc->edma_qlen * (sizeof(struct mvsata_crqb) + + sizeof(struct mvsata_crpb))); + + /* Turn off EDMA engine */ + error = mvsata_edma_ctrl(dev, 0); + if (error) { + ata_dmafini(dev); + return (error); + } /* - * Request queue base high - * Set to zero, we have 32-bit system. + * Initialize EDMA engine: + * - Native Command Queuing off, + * - Non-Queued operation, + * - Host Queue Cache enabled. */ - MVSATA_OUTL(MVSATA_EDMA_REG_REQBA_H, 0x0); - /* Request queue in ptr. */ - MVSATA_OUTL(MVSATA_EDMA_REG_REQIN, work); - /* Request queue out ptr. */ - MVSATA_OUTL(MVSATA_EDMA_REG_REQOUT, 0x0); - /* REQ_QUEUE bytes of DMA work area are now for Request queue. */ - work += REQ_QUEUE_SIZE; - /* Response queue base high. */ - MVSATA_OUTL(MVSATA_EDMA_REG_RSPBA_H, 0x0); - /* Response queue in ptr. */ - MVSATA_OUTL(MVSATA_EDMA_REG_RSPIN, 0x0); - /* Response queue out ptr. */ - MVSATA_OUTL(MVSATA_EDMA_REG_RSPOUT, work); - /* Clear SATA error register. */ + MVSATA_OUTL(sc, MVSATA_EDMA_CFG(ch->unit), MVSATA_EDMA_CFG_HQCACHE | + (sc->version == 1) ? MVSATA_EDMA_CFG_QL128 : 0); + + /* Set request queue pointers */ + work = ch->dma.work_bus; + MVSATA_OUTL(sc, MVSATA_EDMA_REQBAHR(ch->unit), work >> 32); + MVSATA_OUTL(sc, MVSATA_EDMA_REQIPR(ch->unit), work & 0xFFFFFFFF); + MVSATA_OUTL(sc, MVSATA_EDMA_REQOPR(ch->unit), work & 0xFFFFFFFF); + + /* Set response queue pointers */ + work += sc->edma_qlen * sizeof(struct mvsata_crqb); + MVSATA_OUTL(sc, MVSATA_EDMA_RESBAHR(ch->unit), work >> 32); + MVSATA_OUTL(sc, MVSATA_EDMA_RESIPR(ch->unit), work & 0xFFFFFFFF); + MVSATA_OUTL(sc, MVSATA_EDMA_RESOPR(ch->unit), work & 0xFFFFFFFF); + + /* Clear any outstanding interrupts */ ATA_IDX_OUTL(ch, ATA_SERROR, ATA_IDX_INL(ch, ATA_SERROR)); - /* Clear any outstanding error interrupts. */ - MVSATA_OUTL(MVSATA_EDMA_REG_IEC, 0x0); - /* Unmask all error interrupts. */ - MVSATA_OUTL(MVSATA_EDMA_REG_IEM, ~0x0); - /* Enable EDMA machinery. */ - mvsata_edma_on_off(dev, 1); - return (0); + MVSATA_OUTL(sc, MVSATA_SATA_FISICR(ch->unit), 0); + MVSATA_OUTL(sc, MVSATA_EDMA_IECR(ch->unit), 0); + MVSATA_OUTL(sc, MVSATA_ICR, ~(MVSATA_ICR_DEV(ch->unit) | + MVSATA_ICR_DMADONE(ch->unit))); + + /* Umask channel interrupts */ + MVSATA_OUTL(sc, MVSATA_EDMA_IEMR(ch->unit), 0xFFFFFFFF); + MVSATA_OUTL(sc, MVSATA_MIMR, MVSATA_INL(sc, MVSATA_MIMR) | + MVSATA_MICR_DONE(ch->unit) | MVSATA_MICR_DMADONE(ch->unit) | + MVSATA_MICR_ERR(ch->unit)); + + ch->attached = 1; + + return (ata_attach(dev)); } static int -mvsata_edma_begin_transaction(struct ata_request *request) +mvsata_channel_detach(device_t dev) { - struct mvsata_softc *sc; - struct ata_channel *ch; - u_int32_t req_in; - u_int8_t *bytep; + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); int error; - u_int8_t slot; - sc = device_get_softc(GRANDPARENT(request->dev)); - ch = device_get_softc(request->parent); + if (!ch->attached) + return (0); + + /* Turn off EDMA engine */ + mvsata_edma_ctrl(dev, 0); + + /* Mask chanel interrupts */ + MVSATA_OUTL(sc, MVSATA_EDMA_IEMR(ch->unit), 0); + MVSATA_OUTL(sc, MVSATA_MIMR, MVSATA_INL(sc, MVSATA_MIMR) & ~( + MVSATA_MICR_DONE(ch->unit) | MVSATA_MICR_DMADONE(ch->unit) | + MVSATA_MICR_ERR(ch->unit))); + + error = ata_detach(dev); + ata_dmafini(dev); + + ch->attached = 0; + + return (error); +} + +static int +mvsata_channel_begin_transaction(struct ata_request *request) +{ + struct mvsata_softc *sc = device_get_softc(GRANDPARENT(request->dev)); + struct ata_channel *ch = device_get_softc(request->parent); + struct mvsata_crqb *crqb; + uint32_t req_in; + int error, slot; /* Only DMA R/W goes through the EDMA machine. */ if (request->u.ata.command != ATA_READ_DMA && request->u.ata.command != ATA_WRITE_DMA) { - /* Disable the EDMA machinery. */ - if (MVSATA_INL(MVSATA_EDMA_REG_CMD) & 0x00000001) - mvsata_edma_on_off(request->parent, 0); + + /* Disable EDMA before accessing legacy registers */ + if (mvsata_edma_is_running(request->parent)) { + error = mvsata_edma_ctrl(request->parent, 0); + if (error) { + request->result = error; + return (ATA_OP_FINISHED); + } + } + return (ata_begin_transaction(request)); } - /* Check for 48 bit access and convert if needed. */ + /* Check for 48 bit access and convert if needed */ ata_modify_if_48bit(request); - /* Check sanity, setup SG list and DMA engine. */ + /* Prepare data for DMA */ if ((error = ch->dma.load(request, NULL, NULL))) { - device_printf(request->dev, "setting up DMA failed\n"); + device_printf(request->dev, "Setting up DMA failed!\n"); request->result = error; - return (ATA_OP_FINISHED); + return ATA_OP_FINISHED; } - /* Get next free request queue slot. */ - req_in = MVSATA_INL(MVSATA_EDMA_REG_REQIN); - - slot = ((req_in & ~MVSATA_EDMA_REG_REQINP_MASK) >> - MVSATA_EDMA_REG_REQINP_OFS ) & 0x7F; - bytep = (u_int8_t *)(ch->dma.work); - bytep += (slot << 5); - - /* - * Fill in this request. - */ - *(uint32_t *)(bytep) = (u_int32_t)request->dma->sg_bus & 0xffffffff; - *(uint32_t *)(bytep + 32) = 0; - *(uint16_t *)(bytep + 64) = (request->flags & ATA_R_READ ? 0x01 : 0x00) | - (request->tag << 1); - bytep[10] = (request->u.ata.count >> 8) & 0xff; - bytep[11] = 0x10 | ATA_COUNT; - bytep[12] = request->u.ata.count & 0xff; - bytep[13] = 0x10 | ATA_COUNT; - bytep[14] = (request->u.ata.lba >> 24) & 0xff; - bytep[15] = 0x10 | ATA_SECTOR; - bytep[16] = request->u.ata.lba & 0xff; - bytep[17] = 0x10 | ATA_SECTOR; - bytep[18] = (request->u.ata.lba >> 32) & 0xff; - bytep[19] = 0x10 | ATA_CYL_LSB; - bytep[20] = (request->u.ata.lba >> 8) & 0xff; - bytep[21] = 0x10 | ATA_CYL_LSB; - bytep[22] = (request->u.ata.lba >> 40) & 0xff; - bytep[23] = 0x10 | ATA_CYL_MSB; - bytep[24] = (request->u.ata.lba >> 16) & 0xff; - bytep[25] = 0x10 | ATA_CYL_MSB; - bytep[26] = ATA_D_LBA | ATA_D_IBM | ((request->u.ata.lba >> 24) & 0xf); - bytep[27] = 0x10 | ATA_DRIVE; - bytep[28] = request->u.ata.command; - bytep[29] = 0x90 | ATA_COMMAND; - - /* - * Enable EDMA machinery if needed. - */ - error = MVSATA_INL(MVSATA_EDMA_REG_CMD) & 0x00000001; - if (!error) - if (!mvsata_edma_on_off(request->parent, 1)) + /* Get next free queue slot */ + req_in = MVSATA_INL(sc, MVSATA_EDMA_REQIPR(ch->unit)); + slot = (req_in & sc->edma_reqis_mask) >> MVSATA_EDMA_REQIS_OFS; + crqb = (struct mvsata_crqb *)(ch->dma.work + + (slot << MVSATA_EDMA_REQIS_OFS)); + + /* Fill in request */ + crqb->prdlo = htole32((uint64_t)request->dma->sg_bus & 0xFFFFFFFF); + crqb->prdhi = htole32((uint64_t)request->dma->sg_bus >> 32); + crqb->flags = htole32((request->flags & ATA_R_READ ? 0x01 : 0x00) | + (request->tag << 1)); + + crqb->ata_command = request->u.ata.command; + crqb->ata_feature = request->u.ata.feature; + crqb->ata_lba_low = request->u.ata.lba; + crqb->ata_lba_mid = request->u.ata.lba >> 8; + crqb->ata_lba_high = request->u.ata.lba >> 16; + crqb->ata_device = ((request->u.ata.lba >> 24) & 0x0F) | (1 << 6); + crqb->ata_count = request->u.ata.count; + + /* Enable EDMA if disabled */ + if (!mvsata_edma_is_running(request->parent)) { + error = mvsata_edma_ctrl(request->parent, 1); + if (error) { + ch->dma.unload(request); + request->result = error; return (ATA_OP_FINISHED); + } + } - /* - * Tell EDMA it has a new request. - */ - slot++; - req_in &= MVSATA_EDMA_REG_REQINP_MASK; - req_in += (slot << MVSATA_EDMA_REG_REQINP_OFS); - MVSATA_OUTL(MVSATA_EDMA_REG_REQIN, req_in); + /* Tell EDMA about new request */ + req_in = (req_in & ~sc->edma_reqis_mask) | (((slot + 1) << + MVSATA_EDMA_REQIS_OFS) & sc->edma_reqis_mask); + + MVSATA_OUTL(sc, MVSATA_EDMA_REQIPR(ch->unit), req_in); return (ATA_OP_CONTINUES); } -/* - * Must be called with ATA channel locked and state_mtx held. - */ +/* Must be called with ATA channel locked and state_mtx held.*/ static int -mvsata_edma_end_transaction(struct ata_request *request) +mvsata_channel_end_transaction(struct ata_request *request) { - struct mvsata_softc *sc; - struct ata_channel *ch; - struct ata_marvell_response *response; - int mask, res; - u_int32_t icr; - u_int32_t rsp_in, rsp_out, slot_offset; + struct mvsata_softc *sc = device_get_softc(GRANDPARENT(request->dev)); + struct ata_channel *ch = device_get_softc(request->parent); + struct mvsata_crpb *crpb; + uint32_t res_in, res_out, icr; int slot; - sc = device_get_softc(GRANDPARENT(request->dev)); - ch = device_get_softc(request->parent); - mask = (ch->unit == 0 ? 0x1 : 0x2); - - /* Read HC Interrupt cause register. */ - icr = ATA_INL(sc->mem_res, MV_SATAHC_BASE + MVSATA_HC_ICR); - /* EDMA interrupt. */ - if ((icr & mask)) { - /* Stop timeout. */ - callout_stop(&request->callout); - - /* Get response pointers. */ - rsp_in = MVSATA_INL(MVSATA_EDMA_REG_RSPIN); - rsp_out = MVSATA_INL(MVSATA_EDMA_REG_RSPOUT); - slot = (((rsp_in & ~MVSATA_EDMA_REG_RESPINP_MASK) >> - MVSATA_EDMA_REG_RESPINP_OFS)) & MVSATA_MAXSLOT_MASK; - rsp_out &= MVSATA_EDMA_REG_RESPOUT_MASK; - slot_offset = slot << MVSATA_EDMA_REG_RESPOUT_OFS; - rsp_out += slot_offset; - KASSERT(slot_offset < RSP_QUEUE_SIZE, "Too big slot offset!\n"); - response = (struct ata_marvell_response *) - (ch->dma.work + REQ_QUEUE_SIZE + slot_offset); - - /* Record status for this request. */ - request->status = response->dev_status; + icr = MVSATA_INL(sc, MVSATA_ICR); + if (icr & MVSATA_ICR_DMADONE(ch->unit)) { + /* Get current response slot */ + res_out = MVSATA_INL(sc, MVSATA_EDMA_RESOPR(ch->unit)); + slot = (res_out & sc->edma_resos_mask) >> MVSATA_EDMA_RESOS_OFS; + crpb = (struct mvsata_crpb *)(ch->dma.work + + (sc->edma_qlen * sizeof(struct mvsata_crqb)) + + (slot << MVSATA_EDMA_RESOS_OFS)); + + /* Record this request status */ + request->status = crpb->dev_status; request->error = 0; - /* Ack response. */ - MVSATA_OUTL(MVSATA_EDMA_REG_RSPOUT, rsp_out); - /* Update progress. */ + + /* Update response queue pointer */ + res_out = (res_out & ~sc->edma_resos_mask) | (((slot + 1) << + MVSATA_EDMA_RESOS_OFS) & sc->edma_resos_mask); + + MVSATA_OUTL(sc, MVSATA_EDMA_RESOPR(ch->unit), res_out); + + /* Ack DMA interrupt if there is nothing more to do */ + res_in = MVSATA_INL(sc, MVSATA_EDMA_RESIPR(ch->unit)); + res_in &= sc->edma_resos_mask; + res_out &= sc->edma_resos_mask; + + if (res_in == res_out) + MVSATA_OUTL(sc, MVSATA_ICR, + ~MVSATA_ICR_DMADONE(ch->unit)); + + /* Update progress */ if (!(request->status & ATA_S_ERROR) && !(request->flags & ATA_R_TIMEOUT)) request->donecount = request->bytecount; - /* Unload SG list. */ + + /* Unload DMA data */ ch->dma.unload(request); - res = ATA_OP_FINISHED; - } else { /* Legacy ATA interrupt. */ - res = ata_end_transaction(request); + + return(ATA_OP_FINISHED); + } + + /* Legacy ATA interrupt */ + return (ata_end_transaction(request)); +} + +static int +mvsata_channel_status(device_t dev) +{ + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); + uint32_t icr, iecr; + + icr = MVSATA_INL(sc, MVSATA_ICR); + iecr = MVSATA_INL(sc, MVSATA_EDMA_IECR(ch->unit)); + + if ((icr & MVSATA_ICR_DEV(ch->unit)) || iecr) { + /* Disable EDMA before accessing SATA registers */ + mvsata_edma_ctrl(dev, 0); + ata_sata_phy_check_events(dev); + + /* Ack device and error interrupt */ + MVSATA_OUTL(sc, MVSATA_ICR, ~MVSATA_ICR_DEV(ch->unit)); + MVSATA_OUTL(sc, MVSATA_EDMA_IECR(ch->unit), 0); + } + + icr &= MVSATA_ICR_DEV(ch->unit) | MVSATA_ICR_DMADONE(ch->unit); + return (icr); +} + +static void +mvsata_channel_reset(device_t dev) +{ + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); + + /* Disable EDMA before using legacy registers */ + mvsata_edma_ctrl(dev, 0); + + /* Mask all EDMA interrups */ + MVSATA_OUTL(sc, MVSATA_EDMA_IEMR(ch->unit), 0); + + /* Reset EDMA */ + MVSATA_OUTL(sc, MVSATA_EDMA_CMD(ch->unit), MVSATA_EDMA_CMD_RESET); + DELAY(25); + MVSATA_OUTL(sc, MVSATA_EDMA_CMD(ch->unit), 0); + + /* Reset PHY and device */ + if (ata_sata_phy_reset(dev, -1, 1)) + ata_generic_reset(dev); + else + ch->devices = 0; + + /* Clear EDMA errors */ + MVSATA_OUTL(sc, MVSATA_SATA_FISICR(ch->unit), 0); + MVSATA_OUTL(sc, MVSATA_EDMA_IECR(ch->unit), 0); + + /* Unmask all EDMA interrups */ + MVSATA_OUTL(sc, MVSATA_EDMA_IEMR(ch->unit), 0xFFFFFFFF); +} + +static void +mvsata_channel_setmode(device_t parent, device_t dev) +{ + struct ata_device *atadev = device_get_softc(dev); + int mode = atadev->mode; + + /* Disable EDMA before using legacy registers */ + mvsata_edma_ctrl(parent, 0); + + ata_sata_setmode(dev, ATA_PIO_MAX); + if (mode >= ATA_DMA) + ata_sata_setmode(dev, mode); +} + +static void +mvsata_channel_dmasetprd(void *xsc, bus_dma_segment_t *segs, int nsegs, + int error) +{ + struct ata_dmasetprd_args *args = xsc; + struct mvsata_prdentry *prd = args->dmatab; + int i; + + if ((args->error = error)) + return; + + for (i = 0; i < nsegs; i++) { + prd[i].addrlo = htole32(segs[i].ds_addr); + prd[i].addrhi = htole32((uint64_t)segs[i].ds_addr >> 32); + prd[i].count = htole32(segs[i].ds_len); } - /* Ack interrupt. */ - ATA_OUTL(sc->mem_res, MV_SATAHC_BASE + MVSATA_HC_ICR, ~(icr & mask)); - return (res); + + prd[i - 1].count |= htole32(ATA_DMA_EOT); + KASSERT(nsegs <= ATA_DMA_ENTRIES, ("Too many DMA segment entries.\n")); + args->nsegs = nsegs; } static int -mvsata_edma_status(device_t dev) +mvsata_edma_ctrl(device_t dev, int on) { + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); + int bit = on ? MVSATA_EDMA_CMD_ENABLE : MVSATA_EDMA_CMD_DISABLE; + int timeout = EDMA_TIMEOUT; + uint32_t reg; + + MVSATA_OUTL(sc, MVSATA_EDMA_CMD(ch->unit), bit); + + while (1) { + DELAY(1); + + reg = MVSATA_INL(sc, MVSATA_EDMA_CMD(ch->unit)); + + /* Enable bit will be 1 after disable command completion */ + if (on && (reg & MVSATA_EDMA_CMD_ENABLE)) + break; + + /* Disable bit will be 0 after disable command completion */ + if (!on && !(reg & MVSATA_EDMA_CMD_DISABLE)) + break; + + if (timeout-- <= 0) { + device_printf(dev, "EDMA command timeout!\n"); + return (ETIMEDOUT); + } + } - ata_sata_phy_check_events(dev); return (0); } +static int +mvsata_edma_is_running(device_t dev) +{ + struct mvsata_softc *sc = device_get_softc(device_get_parent(dev)); + struct ata_channel *ch = device_get_softc(dev); + + return (MVSATA_INL(sc, MVSATA_EDMA_CMD(ch->unit)) & + MVSATA_EDMA_CMD_ENABLE); +} + static device_method_t mvsata_channel_methods[] = { /* Device interface. */ - DEVMETHOD(device_probe, mvsata_channel_probe), + DEVMETHOD(device_probe, mvsata_channel_probe), DEVMETHOD(device_attach, mvsata_channel_attach), - DEVMETHOD(device_detach, ata_detach), + DEVMETHOD(device_detach, mvsata_channel_detach), DEVMETHOD(device_shutdown, bus_generic_shutdown), DEVMETHOD(device_suspend, ata_suspend), DEVMETHOD(device_resume, ata_resume), + + /* ATA channel interface */ + DEVMETHOD(ata_reset, mvsata_channel_reset), + DEVMETHOD(ata_setmode, mvsata_channel_setmode), { 0, 0 } }; diff --git a/sys/arm/mv/mvsata_reg.h b/sys/arm/mv/mvsata_reg.h index b14f185..447fda4 100644 --- a/sys/arm/mv/mvsata_reg.h +++ b/sys/arm/mv/mvsata_reg.h @@ -25,68 +25,121 @@ * SUCH DAMAGE. */ +#ifndef _MVSATA_REG_H_ +#define _MVSATA_REG_H_ #include __FBSDID("$FreeBSD$"); -#define REQ_QUEUE_SIZE 4096 -#define RSP_QUEUE_SIZE 1024 +/* Useful constants */ +#define MVSATA_CHAN_NUM 2 -#define ATA_MV_EDMA_BASE(ch) (MV_SATAHC_BASE +\ - EDMA_REGISTERS_OFFSET + ((ch)->unit - 1) * EDMA_REGISTERS_SIZE) +#define EDMA_REGISTERS_OFFSET 0x2000 +#define EDMA_REGISTERS_SIZE 0x2000 +#define EDMA_TIMEOUT 100000 /* 100 mS */ -#define MVSATA_INL(reg) ATA_INL(sc->mem_res,\ - ATA_MV_EDMA_BASE(ch) + (reg)) -#define MVSATA_OUTL(reg, val) ATA_OUTL(sc->mem_res,\ - ATA_MV_EDMA_BASE(ch) + (reg), (val)) +#define MVSATA_EDMA_BASE(ch) (EDMA_REGISTERS_OFFSET + \ + ((ch) * EDMA_REGISTERS_SIZE)) -#define MVSATA_CHAN_NUM 2 +#define MVSATA_INL(sc, reg) ATA_INL((sc)->mem_res, reg) +#define MVSATA_OUTL(sc, reg, val) ATA_OUTL((sc)->mem_res, reg, val) -#define MVSATA_WAIT_ATTEMPTS 10 -/* - * Stuff specific for each ATA port. - */ -#define EDMA_REGISTERS_OFFSET 0x2000 -#define EDMA_REGISTERS_SIZE 0x2000 +/* Data structures */ +struct mvsata_prdentry { + uint32_t addrlo; + uint32_t count; + uint32_t addrhi; + uint32_t reserved; +}; -#define MVSATA_MAXSLOT_MASK 0x7F -/* - * Shadow registers. - */ -#define MVSATA_SHADOWR_BASE 0x100 -#define MVSATA_SHADOWR_CONTROL 0x20 -/* - * SATAHC registers. - */ -#define MVSATA_HC_ICR 0x014 /* Interrupt cause */ -/* - * SATA interface registers. - */ -#define MVSATA_SATA_SSTATUS 0x300 /* SStatus */ -#define MVSATA_SATA_SERROR 0x304 /* SError */ -#define MVSATA_SATA_SEIM 0x340 /* SError Interrupt Mask */ -#define MVSATA_SATA_SCONTROL 0x308 /* SControl */ +struct mvsata_crqb { + uint32_t prdlo; + uint32_t prdhi; + uint32_t flags; + uint16_t count; + uint16_t reserved1[2]; + uint8_t ata_command; + uint8_t ata_feature; + uint8_t ata_lba_low; + uint8_t ata_lba_mid; + uint8_t ata_lba_high; + uint8_t ata_device; + uint8_t ata_lba_low_p; + uint8_t ata_lba_mid_p; + uint8_t ata_lba_high_p; + uint8_t ata_feature_p; + uint8_t ata_count; + uint8_t ata_count_p; + uint16_t reserved2; +}; + +struct mvsata_crpb { + uint8_t tag; + uint8_t reserved; + uint8_t edma_status; + uint8_t dev_status; + uint32_t timestamp; +}; + +/* SATAHC registers */ +#define MVSATA_CR 0x000 /* Configuration Reg. */ +#define MVSATA_CR_NODMABS (1 << 8) +#define MVSATA_CR_NOEDMABS (1 << 9) +#define MVSATA_CR_NOPRDPBS (1 << 10) +#define MVSATA_CR_COALDIS(ch) (1 << (24 + ch)) + +#define MVSATA_ICR 0x014 /* Interrupt Cause Reg. */ +#define MVSATA_ICR_DMADONE(ch) (1 << (ch)) +#define MVSATA_ICR_COAL (1 << 4) +#define MVSATA_ICR_DEV(ch) (1 << (8 + ch)) + +#define MVSATA_MICR 0x020 /* Main Interrupt Cause Reg. */ +#define MVSATA_MICR_ERR(ch) (1 << (2 * ch)) +#define MVSATA_MICR_DONE(ch) (1 << ((2 * ch) + 1)) +#define MVSATA_MICR_DMADONE(ch) (1 << (4 + ch)) +#define MVSATA_MICR_COAL (1 << 8) + +#define MVSATA_MIMR 0x024 /* Main Interrupt Mask Reg. */ + +/* Shadow registers */ +#define MVSATA_SHADOWR_BASE(ch) (MVSATA_EDMA_BASE(ch) + 0x100) +#define MVSATA_SHADOWR_CONTROL(ch) (MVSATA_EDMA_BASE(ch) + 0x120) + +/* SATA registers */ +#define MVSATA_SATA_SSTATUS(ch) (MVSATA_EDMA_BASE(ch) + 0x300) +#define MVSATA_SATA_SERROR(ch) (MVSATA_EDMA_BASE(ch) + 0x304) +#define MVSATA_SATA_SEIM(ch) (MVSATA_EDMA_BASE(ch) + 0x340) +#define MVSATA_SATA_SCONTROL(ch) (MVSATA_EDMA_BASE(ch) + 0x308) +#define MVSATA_SATA_FISICR(ch) (MVSATA_EDMA_BASE(ch) + 0x364) + +/* EDMA registers */ +#define MVSATA_EDMA_CFG(ch) (MVSATA_EDMA_BASE(ch) + 0x000) +#define MVSATA_EDMA_CFG_QL128 (1 << 19) +#define MVSATA_EDMA_CFG_HQCACHE (1 << 22) + +#define MVSATA_EDMA_IECR(ch) (MVSATA_EDMA_BASE(ch) + 0x008) +#define MVSATA_EDMA_UNRECOVERABLE 0xFC1E9008 + +#define MVSATA_EDMA_IEMR(ch) (MVSATA_EDMA_BASE(ch) + 0x00C) +#define MVSATA_EDMA_REQBAHR(ch) (MVSATA_EDMA_BASE(ch) + 0x010) +#define MVSATA_EDMA_REQIPR(ch) (MVSATA_EDMA_BASE(ch) + 0x014) +#define MVSATA_EDMA_REQOPR(ch) (MVSATA_EDMA_BASE(ch) + 0x018) +#define MVSATA_EDMA_RESBAHR(ch) (MVSATA_EDMA_BASE(ch) + 0x01C) +#define MVSATA_EDMA_RESIPR(ch) (MVSATA_EDMA_BASE(ch) + 0x020) +#define MVSATA_EDMA_RESOPR(ch) (MVSATA_EDMA_BASE(ch) + 0x024) + +#define MVSATA_EDMA_CMD(ch) (MVSATA_EDMA_BASE(ch) + 0x028) +#define MVSATA_EDMA_CMD_ENABLE (1 << 0) +#define MVSATA_EDMA_CMD_DISABLE (1 << 1) +#define MVSATA_EDMA_CMD_RESET (1 << 2) + +#define MVSATA_EDMA_STATUS(ch) (MVSATA_EDMA_BASE(ch) + 0x030) +#define MVSATA_EDMA_STATUS_IDLE (1 << 7) + +/* Offset to extract input slot from REQIPR register */ +#define MVSATA_EDMA_REQIS_OFS 5 + +/* Offset to extract input slot from RESOPR register */ +#define MVSATA_EDMA_RESOS_OFS 3 + +#endif /* _MVSATA_REG_H_ */ -/* - * EDMA registers. - */ -#define MVSATA_EDMA_REG_CFG 0x000 /* Configuration register*/ -#define MVSATA_EDMA_REG_IEC 0x008 /* Interrupt Error Cause */ -#define MVSATA_EDMA_REG_IEM 0x00C /* Interrupt Error Cause */ -#define MVSATA_EDMA_REG_REQBA_H 0x010 /* Request Queue-In Address H*/ -#define MVSATA_EDMA_REG_REQIN 0x014 /* Request Queue-In pointer */ -#define MVSATA_EDMA_REG_REQOUT 0x018 /* Request Queue-Out pointer */ -#define MVSATA_EDMA_REG_RSPBA_H 0x01C /* Response Queue-In Address H*/ -#define MVSATA_EDMA_REG_RSPIN 0x020 /* Response Queue-In pointer */ -#define MVSATA_EDMA_REG_RSPOUT 0x024 /* Response Queue-Out pointer */ - -#define MVSATA_EDMA_REG_CMD 0x028 /* Command register */ -#define MVSATA_EDMA_REG_STATUS 0x030 /* Status register */ - -#define MVSATA_EDMA_REG_REQINP_OFS 0x5 /* Request Queue-In pointer offset */ -#define MVSATA_EDMA_REG_REQINP_MASK 0xFFFFF000 /* Request Queue-In pointer mask*/ - -#define MVSATA_EDMA_REG_RESPINP_OFS 0x3 /* Response Queue-In pointer offset */ -#define MVSATA_EDMA_REG_RESPINP_MASK 0xFFFFFC00 /* Response Queue-In pointer mask*/ - -#define MVSATA_EDMA_REG_RESPOUT_OFS MVSATA_EDMA_REG_RESPINP_OFS -#define MVSATA_EDMA_REG_RESPOUT_MASK MVSATA_EDMA_REG_RESPINP_MASK diff --git a/sys/dev/ata/ata-dma.c b/sys/dev/ata/ata-dma.c index ab4c3c8..71ee41b 100644 --- a/sys/dev/ata/ata-dma.c +++ b/sys/dev/ata/ata-dma.c @@ -95,8 +95,8 @@ ata_dmainit(device_t dev) 0, NULL, NULL, &ch->dma.work_tag)) goto error; - if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, 0, - &ch->dma.work_map)) + if (bus_dmamem_alloc(ch->dma.work_tag, (void **)&ch->dma.work, + BUS_DMA_COHERENT, &ch->dma.work_map)) goto error; if (bus_dmamap_load(ch->dma.work_tag, ch->dma.work_map, ch->dma.work,