/*-
* SPDX-License-Identifier: BSD-2-Clause
*
* Copyright (c) 2019-2020 Ruslan Bukin
* Copyright (c) 2022 Kyle Evans
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include
#include
#include /* iommu/_task.h */
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include
#include "apple_dartreg.h"
#include "bus_if.h"
#include "iommu_if.h"
MALLOC_DEFINE(M_APLDART, "apple_dart", "Apple DART (IOMMU)");
#define DART_MAXADDR (1ul << 48) - 1
struct dart_cfg {
u_int shift;
uint64_t nsid;
};
static const struct dart_cfg t8103_dart_cfg = {
.shift = 0,
.nsid = DART_STREAM_MAX,
};
static const struct dart_cfg t6000_dart_cfg = {
.shift = 4,
.nsid = DART_STREAM_MAX,
};
static const struct ofw_compat_data compat_data[] = {
{ "apple,t6000-dart", (uintptr_t)&t6000_dart_cfg },
{ "apple,t8103-dart", (uintptr_t)&t8103_dart_cfg },
{ NULL, 0 },
};
static struct resource_spec dart_spec[] = {
{ SYS_RES_MEMORY, 0, RF_ACTIVE },
{ SYS_RES_IRQ, 0, RF_ACTIVE | RF_SHAREABLE},
{ -1, 0 },
};
struct apple_dart_domain;
struct apple_dart_ctx;
struct apple_dart_unit {
struct iommu_unit iommu;
LIST_HEAD(, apple_dart_domain) domain_list;
};
#if 0
struct apple_dart_ttbr {
uintptr_t
};
#endif
struct apple_dart_domain {
struct iommu_domain iodom;
struct apple_dart_ctx *ctx;
LIST_ENTRY(apple_dart_domain) next;
struct apple_dart_softc *sc;
size_t nl1;
uint64_t *dom_l1;
size_t nl2;
uint64_t **dom_l2;
size_t ntte;
uint16_t sid;
bool bypass;
bool ready;
};
struct apple_dart_ctx {
struct iommu_ctx ioctx;
struct apple_dart_domain *domain;
LIST_ENTRY(apple_dart_ctx) next;
device_t dev;
bool bypass;
};
struct apple_dart_softc {
device_t sc_dev;
struct mtx sc_mtx;
struct apple_dart_unit sc_unit;
struct resource *sc_res[2];
phandle_t sc_phandle;
bus_space_tag_t sc_bst;
bus_space_handle_t sc_bsh;
bus_dma_tag_t sc_dmat;
u_int sc_sid_mask;
u_int sc_nsid;
u_int sc_shift;
u_int sc_sid_allocated;
struct mtx sc_sid_mtx;
void *sc_intr_cookie;
};
#define DART_LOCK(_sc) mtx_lock(&(_sc)->sc_mtx)
#define DART_UNLOCK(_sc) mtx_unlock(&(_sc)->sc_mtx)
#define DART_SID_LOCK(_sc) mtx_lock_spin(&(_sc)->sc_sid_mtx)
#define DART_SID_UNLOCK(_sc) mtx_unlock_spin(&(_sc)->sc_sid_mtx)
#define DART_READ(sc, reg) \
bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
#define DART_WRITE(sc, reg, val) \
bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
static void
apple_dart_flush_tlb_sid(struct apple_dart_softc *sc, u_int sid)
{
dsb(sy);
isb();
DART_LOCK(sc);
DART_WRITE(sc, DART_TLB_OP_SIDMASK, sid);
DART_WRITE(sc, DART_TLB_OP, DART_TLB_OP_FLUSH);
while ((DART_READ(sc, DART_TLB_OP) & DART_TLB_OP_BUSY) != 0) {
__asm volatile ("yield" ::: "memory");
}
DART_UNLOCK(sc);
}
static void
apple_dart_flush_tlb(struct apple_dart_softc *sc)
{
return (apple_dart_flush_tlb_sid(sc, sc->sc_sid_mask));
}
#if 0
static struct apple_dart_dma *
apple_dart_dma_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align)
{
struct apple_dart_dma *dma;
int nsegs, error;
dma = malloc(sizeof(*dma), M_APLDART, M_WAITOK | M_ZERO);
dma->dma_size = size;
error = bus_dmamem_alloc(dmat, size, align, 0, &dma->dma_seg, 1,
&nsegs, BUS_DMA_WAITOK);
if (error != 0) {
goto destroy;
}
error = bus_dmamem_map(dmat, &dma->dma_seg, nsegs, size,
&dma->dma_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE);
if (error != 0) {
goto free;
}
error = bus_dmamap_create(dmat, size, 1, size, 0,
BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &dma->dma_map);
if (error != 0) {
goto dmafree;
}
error = bus_dmamap_load(dmat, dma->dma_map, dma->dma_kva, size,
NULL, BUS_DMA_WAITOK);
if (error != 0) {
goto unmap;
}
memset(dma->dma_kva, 0, size);
return dma;
destroy:
bus_dmamap_destroy(dmat, dma->dma_map);
unmap:
bus_dmamem_unmap(dmat, dma->dma_kva, size);
free:
bus_dmamem_free(dmat, &dma->dma_seg, 1);
dmafree:
free(dma, M_APLDART);
return NULL;
}
#endif
static int
apple_dart_intr(void *priv)
{
struct apple_dart_softc * const sc = priv;
uint64_t addr;
uint32_t status;
status = DART_READ(sc, DART_ERR_STATUS);
addr = DART_READ(sc, DART_ERR_ADDRL);
addr |= (uint64_t)DART_READ(sc, DART_ERR_ADDRH) << 32;
DART_WRITE(sc, DART_ERR_STATUS, status);
device_printf(sc->sc_dev, "error addr 0x%016lx status 0x%08x\n",
addr, status);
return 1;
}
static volatile uint64_t *
apple_dart_lookup_tte(struct apple_dart_softc *sc, bus_addr_t dva)
{
#if 0
int idx = dva / DART_PAGE_SIZE;
int l2_idx = idx / (DART_PAGE_SIZE / sizeof(uint64_t));
int tte_idx = idx % (DART_PAGE_SIZE / sizeof(uint64_t));
volatile uint64_t *l2;
l2 = vtophys(sc->sc_l2[l2_idx]);
return &l2[tte_idx];
#endif
return (NULL);
}
static int
apple_dart_probe(device_t dev)
{
if (!ofw_bus_status_okay(dev))
return (ENXIO);
if (ofw_bus_search_compatible(dev, compat_data)->ocd_data == 0)
return (ENXIO);
device_set_desc(dev, "Apple DART");
return (BUS_PROBE_DEFAULT);
}
static int
apple_dart_attach_iommu(struct apple_dart_softc *const sc)
{
struct apple_dart_unit *unit;
struct iommu_unit *iommu;
unit = &sc->sc_unit;
LIST_INIT(&unit->domain_list);
iommu = &unit->iommu;
iommu->dev = sc->sc_dev;
return (iommu_register(iommu));
}
static int
apple_dart_attach(device_t dev)
{
struct apple_dart_softc * const sc = device_get_softc(dev);
const phandle_t phandle = ofw_bus_get_node(dev);
const struct dart_cfg *dcfg;
u_int config, idx, params2, sid;
int error;
bool bypass;
error = 0;
if (bus_alloc_resources(dev, dart_spec, sc->sc_res) != 0) {
device_printf(dev, "cannot allocate resources for device\n");
return (ENXIO);
}
sc->sc_dev = dev;
sc->sc_phandle = phandle;
sc->sc_bst = rman_get_bustag(sc->sc_res[0]);
sc->sc_bsh = rman_get_bushandle(sc->sc_res[0]);
sc->sc_dmat = bus_get_dma_tag(dev);
dcfg = (const struct dart_cfg *)ofw_bus_search_compatible(dev,
compat_data)->ocd_data;
sc->sc_shift = dcfg->shift;
sc->sc_nsid = dcfg->nsid;
config = DART_READ(sc, DART_CONFIG);
if ((config & DART_CONFIG_LOCK) != 0) {
device_printf(dev, "locked\n");
bus_release_resources(dev, dart_spec, sc->sc_res);
return (ENXIO);
}
params2 = DART_READ(sc, DART_PARAMS2);
bypass = ((params2 & DART_PARAMS2_BYPASS_SUPPORT) != 0);
if (bypass) {
for (sid = 0; sid < sc->sc_nsid; sid++) {
DART_WRITE(sc, DART_TCR(sid),
DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF);
for (idx = 0; idx < DART_L1_IDX_MAX; idx++) {
DART_WRITE(sc, DART_TTBR(sid, idx), 0);
}
}
device_printf(dev, "configured in bypass mode\n");
/* XXX */
return (0);
}
mtx_init(&sc->sc_mtx, device_get_nameunit(dev), "apple_dart", MTX_DEF);
mtx_init(&sc->sc_sid_mtx, "asid alloc", NULL, MTX_SPIN);
#if 0
if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
aprint_error(": couldn't map registers\n");
return;
}
#endif
if (OF_getencprop(phandle, "sid-mask", &sc->sc_sid_mask,
sizeof(sc->sc_sid_mask)) <= 0)
sc->sc_sid_mask = DART_STREAM_MASK;
device_printf(dev, "%u SIDs (mask 0x%x)\n", sc->sc_nsid,
sc->sc_sid_mask);
MPASS(sc->sc_nsid == DART_STREAM_MAX);
MPASS(sc->sc_sid_mask == DART_STREAM_MASK); /* For now */
/* Disable translations */
if (!bypass) {
for (sid = 0; sid < sc->sc_nsid; sid++) {
DART_WRITE(sc, DART_TCR(sid), 0);
}
}
/* Remove page tables */
for (sid = 0; sid < sc->sc_nsid; sid++) {
for (idx = 0; idx < DART_L1_IDX_MAX; idx++) {
DART_WRITE(sc, DART_TTBR(sid, idx), 0);
}
}
apple_dart_flush_tlb(sc);
if (bus_setup_intr(dev, sc->sc_res[1], INTR_TYPE_MISC, apple_dart_intr,
NULL, sc, &sc->sc_intr_cookie) != 0) {
device_printf(dev, "Failed to setup interrupt handler\n");
error = ENXIO;
goto out;
}
error = apple_dart_attach_iommu(sc);
if (error != 0) {
device_printf(dev, "Failed to setup iommu contxt.\n");
error = ENXIO;
goto out;
}
OF_device_register_xref(OF_xref_from_node(phandle), dev);
out:
if (error != 0)
bus_release_resources(dev, dart_spec, sc->sc_res);
return (error);
}
static int
apple_dart_find(device_t dev, device_t child)
{
return (ENXIO);
}
static int
apple_dart_sid_acquire(struct apple_dart_domain *dom, int sid, bool bypass)
{
struct apple_dart_softc *sc;
vm_paddr_t pa;
MPASS(!dom->ready);
sc = dom->sc;
DART_SID_LOCK(sc);
MPASS((sc->sc_sid_allocated & (1 << sid)) == 0);
if (sid == 0 || sid > DART_STREAM_MAX) {
DART_SID_UNLOCK(sc);
return (EINVAL);
}
/* Install page tables. */
pa = vtophys(dom->dom_l1);
for (int idx = 0; idx < dom->nl1; idx++) {
DART_WRITE(sc, DART_TTBR(sid, idx),
(pa >> DART_TTBR_SHIFT) | DART_TTBR_VALID);
pa += DART_PAGE_SIZE;
}
if (bypass)
DART_WRITE(sc, DART_TCR(sid),
DART_TCR_BYPASS_DART | DART_TCR_BYPASS_DAPF);
else
DART_WRITE(sc, DART_TCR(sid),
DART_TCR_TXEN);
sc->sc_sid_allocated |= (1 << sid);
dom->sid = sid;
dom->bypass = bypass;
dom->ready = true;
DART_SID_UNLOCK(sc);
apple_dart_flush_tlb_sid(sc, sid);
return (0);
}
static void
apple_dart_sid_release(struct apple_dart_domain *dom)
{
struct apple_dart_softc *sc;
int sid;
MPASS(dom->ready);
sc = dom->sc;
sid = dom->sid;
DART_SID_LOCK(sc);
sc->sc_sid_allocated &= ~(1 << sid);
DART_WRITE(sc, DART_TCR(sid), 0);
for (int idx = 0; idx < DART_L1_IDX_MAX; idx++) {
DART_WRITE(sc, DART_TTBR(sid, idx), 0);
}
DART_SID_UNLOCK(sc);
apple_dart_flush_tlb_sid(sc, sid);
}
static int
apple_dart_map(device_t dev, struct iommu_domain *iodom,
vm_offset_t va, vm_page_t *ma, vm_size_t size,
vm_prot_t prot)
{
struct apple_dart_domain *domain;
struct apple_dart_softc *sc;
vm_paddr_t pa;
int i;
sc = device_get_softc(dev);
domain = (struct apple_dart_domain *)iodom;
#if 0
dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa, size,
domain->asid);
#endif
/*
* XXX Round up to the nearest DART_PAGE_SIZE?
*/
for (i = 0; size > 0; size -= PAGE_SIZE) {
pa = VM_PAGE_TO_PHYS(ma[i++]);
device_printf(dev, "mapping 0x%lx; 16k? %s",
pa, (pa & DART_PAGE_MASK) == 0 ? "Yes" : "No");
#if 0
error = pmap_smmu_enter(&domain->p, va, pa, prot, 0);
if (error)
return (error);
#endif
apple_dart_flush_tlb_sid(sc, domain->sid);
va += PAGE_SIZE;
}
return (0);
}
static int
apple_dart_unmap(device_t dev, struct iommu_domain *iodom,
vm_offset_t va, bus_size_t size)
{
struct apple_dart_domain *domain;
struct apple_dart_softc *sc;
int err;
int i;
sc = device_get_softc(dev);
domain = (struct apple_dart_domain *)iodom;
err = 0;
#if 0
dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->asid);
#endif
for (i = 0; i < size; i += PAGE_SIZE) {
#if 0
if (pmap_smmu_remove(&domain->p, va) == 0) {
/* pmap entry removed, invalidate TLB. */
apple_dart_flush_tlb_sid(sc, domain->asid);
} else {
err = ENOENT;
break;
}
#endif
va += PAGE_SIZE;
}
return (err);
}
static struct iommu_domain *
apple_dart_domain_alloc(device_t dev, struct iommu_unit *iommu)
{
struct apple_dart_domain *domain;
struct apple_dart_unit *unit;
struct apple_dart_softc *sc;
vm_paddr_t pa;
device_printf(dev, "allocating new domain");
sc = device_get_softc(dev);
unit = (struct apple_dart_unit *)iommu;
domain = malloc(sizeof(*domain), M_APLDART, M_WAITOK | M_ZERO);
domain->ntte = howmany(DART_DVA_END, DART_PAGE_SIZE);
domain->nl2 = howmany(domain->ntte,
DART_PAGE_SIZE / sizeof(**domain->dom_l2));
domain->nl1 = howmany(domain->nl1,
DART_PAGE_SIZE / sizeof(*domain->dom_l1));
MPASS(domain->nl1 <= DART_L1_IDX_MAX);
domain->sc = sc;
domain->dom_l1 = (uint64_t *)contigmalloc(domain->nl1 * DART_PAGE_SIZE,
M_APLDART, M_WAITOK, 0, DART_MAXADDR, DART_PAGE_SIZE, 0);
domain->dom_l2 = mallocarray(domain->nl2, sizeof(*domain->dom_l2),
M_APLDART, M_WAITOK);
for (int idx = 0; idx < domain->nl2; idx++) {
domain->dom_l2[idx] = (uint64_t *)contigmalloc(DART_PAGE_SIZE,
M_APLDART, M_WAITOK | M_ZERO, 0, DART_MAXADDR,
DART_PAGE_SIZE, 0);
pa = vtophys(domain->dom_l2[idx]);
/* XXX WRONG */
domain->dom_l1[idx] = (pa >> sc->sc_shift) | DART_L1_TABLE;
}
/*
* Note that the domain isn't usable until the ioctx is created, which
* assigns the domain a sid based on the iommu-map.
*/
IOMMU_LOCK(iommu);
LIST_INSERT_HEAD(&unit->domain_list, domain, next);
IOMMU_UNLOCK(iommu);
return (&domain->iodom);
}
static void
apple_dart_domain_free(device_t dev, struct iommu_domain *iodom)
{
struct apple_dart_domain *domain;
struct apple_dart_softc *sc;
sc = device_get_softc(dev);
domain = (struct apple_dart_domain *)iodom;
LIST_REMOVE(domain, next);
for (int idx = 0; idx < domain->nl2; ++idx) {
free(domain->dom_l2[idx], M_APLDART);
}
free(domain->dom_l2, M_APLDART);
free(domain->dom_l1, M_APLDART);
free(domain, M_APLDART);
}
static struct iommu_ctx *
apple_dart_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child,
bool disabled)
{
struct apple_dart_domain *domain;
struct apple_dart_softc *sc;
struct apple_dart_ctx *ctx;
device_t iommudev;
phandle_t node;
uint16_t rid;
u_int sid;
int err;
sc = device_get_softc(dev);
domain = (struct apple_dart_domain *)iodom;
node = ofw_bus_get_node(device_get_parent(child));
rid = pci_get_rid(child);
err = ofw_bus_iommumap(node, rid, &iommudev, &sid);
if (err != 0)
return (NULL);
if (!apple_dart_sid_acquire(domain, sid, disabled))
return (NULL);
ctx = malloc(sizeof(struct apple_dart_ctx), M_APLDART,
M_WAITOK | M_ZERO);
ctx->dev = child;
ctx->domain = domain;
/*
* dart can only cope with 1:1 domain:ctx. SIDs are allocated by FDT
* data, so this construction is a little awkward.
*/
IOMMU_DOMAIN_LOCK(iodom);
MPASS(domain->ctx == NULL);
domain->ctx = ctx;
IOMMU_DOMAIN_UNLOCK(iodom);
return (&ctx->ioctx);
}
static void
apple_dart_ctx_free(device_t dev, struct iommu_ctx *ioctx)
{
struct apple_dart_softc *sc;
struct apple_dart_ctx *ctx;
IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
sc = device_get_softc(dev);
ctx = (struct apple_dart_ctx *)ioctx;
LIST_REMOVE(ctx, next);
apple_dart_sid_release(ctx->domain);
free(ctx, M_APLDART);
}
static struct apple_dart_ctx *
apple_dart_ctx_lookup_by_sid(device_t dev, u_int sid)
{
struct apple_dart_softc *sc;
struct apple_dart_domain *domain;
struct apple_dart_unit *unit;
sc = device_get_softc(dev);
unit = &sc->sc_unit;
LIST_FOREACH(domain, &unit->domain_list, next) {
if (domain->sid == sid)
return (domain->ctx);
}
return (NULL);
}
static struct iommu_ctx *
apple_dart_ctx_lookup(device_t dev, device_t child)
{
struct iommu_unit *iommu;
struct apple_dart_softc *sc;
struct apple_dart_domain *domain;
struct apple_dart_unit *unit;
sc = device_get_softc(dev);
unit = &sc->sc_unit;
iommu = &unit->iommu;
IOMMU_ASSERT_LOCKED(iommu);
LIST_FOREACH(domain, &unit->domain_list, next) {
IOMMU_DOMAIN_LOCK(&domain->iodom);
if (domain->ctx != NULL && domain->ctx->dev == child) {
IOMMU_DOMAIN_UNLOCK(&domain->iodom);
return (&domain->ctx->ioctx);
}
IOMMU_DOMAIN_UNLOCK(&domain->iodom);
}
return (NULL);
}
static device_method_t apple_dart_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, apple_dart_probe),
DEVMETHOD(device_attach, apple_dart_attach),
/* IOMMU interface */
DEVMETHOD(iommu_find, apple_dart_find),
DEVMETHOD(iommu_map, apple_dart_map),
DEVMETHOD(iommu_unmap, apple_dart_unmap),
DEVMETHOD(iommu_domain_alloc, apple_dart_domain_alloc),
DEVMETHOD(iommu_domain_free, apple_dart_domain_free),
DEVMETHOD(iommu_ctx_alloc, apple_dart_ctx_alloc),
DEVMETHOD(iommu_ctx_free, apple_dart_ctx_free),
DEVMETHOD(iommu_ctx_lookup, apple_dart_ctx_lookup),
DEVMETHOD_END
};
static driver_t apple_dart_driver = {
"dart",
apple_dart_methods,
sizeof(struct apple_dart_softc),
};
static devclass_t apple_dart_devclass;
EARLY_DRIVER_MODULE(apple_dart, simplebus, apple_dart_driver,
apple_dart_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);