X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=blobdiff_plain;f=kernel%2Fdrivers%2Fgpu%2Fdrm%2Fnouveau%2Fnvkm%2Fsubdev%2Finstmem%2Fgk20a.c;fp=kernel%2Fdrivers%2Fgpu%2Fdrm%2Fnouveau%2Fnvkm%2Fsubdev%2Finstmem%2Fgk20a.c;h=14107b5b7811c9d0180e4553adbac77ffd06a94a;hb=e09b41010ba33a20a87472ee821fa407a5b8da36;hp=dd0994d9ebfc3f377141de6662b57130d83e0b2b;hpb=f93b97fd65072de626c074dbe099a1fff05ce060;p=kvmfornfv.git diff --git a/kernel/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c b/kernel/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c index dd0994d9e..14107b5b7 100644 --- a/kernel/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c +++ b/kernel/drivers/gpu/drm/nouveau/nvkm/subdev/instmem/gk20a.c @@ -23,418 +23,607 @@ /* * GK20A does not have dedicated video memory, and to accurately represent this * fact Nouveau will not create a RAM device for it. Therefore its instmem - * implementation must be done directly on top of system memory, while providing - * coherent read and write operations. + * implementation must be done directly on top of system memory, while + * preserving coherency for read and write operations. * * Instmem can be allocated through two means: - * 1) If an IOMMU mapping has been probed, the IOMMU API is used to make memory + * 1) If an IOMMU unit has been probed, the IOMMU API is used to make memory * pages contiguous to the GPU. This is the preferred way. - * 2) If no IOMMU mapping is probed, the DMA API is used to allocate physically + * 2) If no IOMMU unit is probed, the DMA API is used to allocate physically * contiguous memory. * - * In both cases CPU read and writes are performed using PRAMIN (i.e. using the - * GPU path) to ensure these operations are coherent for the GPU. This allows us - * to use more "relaxed" allocation parameters when using the DMA API, since we - * never need a kernel mapping. + * In both cases CPU read and writes are performed by creating a write-combined + * mapping. The GPU L2 cache must thus be flushed/invalidated when required. To + * be conservative we do this every time we acquire or release an instobj, but + * ideally L2 management should be handled at a higher level. + * + * To improve performance, CPU mappings are not removed upon instobj release. + * Instead they are placed into a LRU list to be recycled when the mapped space + * goes beyond a certain threshold. At the moment this limit is 1MB. */ +#include "priv.h" -#include +#include #include -#include - -#ifdef __KERNEL__ -#include -#include -#include -#endif +#include +#include +#include -#include "priv.h" +struct gk20a_instobj { + struct nvkm_memory memory; + struct nvkm_mem mem; + struct gk20a_instmem *imem; -struct gk20a_instobj_priv { - struct nvkm_instobj base; - /* Must be second member here - see nouveau_gpuobj_map_vm() */ - struct nvkm_mem *mem; - /* Pointed by mem */ - struct nvkm_mem _mem; + /* CPU mapping */ + u32 *vaddr; + struct list_head vaddr_node; }; +#define gk20a_instobj(p) container_of((p), struct gk20a_instobj, memory) /* * Used for objects allocated using the DMA API */ struct gk20a_instobj_dma { - struct gk20a_instobj_priv base; + struct gk20a_instobj base; - void *cpuaddr; + u32 *cpuaddr; dma_addr_t handle; struct nvkm_mm_node r; }; +#define gk20a_instobj_dma(p) \ + container_of(gk20a_instobj(p), struct gk20a_instobj_dma, base) /* * Used for objects flattened using the IOMMU API */ struct gk20a_instobj_iommu { - struct gk20a_instobj_priv base; + struct gk20a_instobj base; - /* array of base.mem->size pages */ + /* will point to the higher half of pages */ + dma_addr_t *dma_addrs; + /* array of base.mem->size pages (+ dma_addr_ts) */ struct page *pages[]; }; +#define gk20a_instobj_iommu(p) \ + container_of(gk20a_instobj(p), struct gk20a_instobj_iommu, base) -struct gk20a_instmem_priv { +struct gk20a_instmem { struct nvkm_instmem base; + + /* protects vaddr_* and gk20a_instobj::vaddr* */ spinlock_t lock; - u64 addr; + + /* CPU mappings LRU */ + unsigned int vaddr_use; + unsigned int vaddr_max; + struct list_head vaddr_lru; /* Only used if IOMMU if present */ struct mutex *mm_mutex; struct nvkm_mm *mm; struct iommu_domain *domain; unsigned long iommu_pgshift; + u16 iommu_bit; /* Only used by DMA API */ struct dma_attrs attrs; + + void __iomem * (*cpu_map)(struct nvkm_memory *); }; +#define gk20a_instmem(p) container_of((p), struct gk20a_instmem, base) + +static enum nvkm_memory_target +gk20a_instobj_target(struct nvkm_memory *memory) +{ + return NVKM_MEM_TARGET_HOST; +} + +static u64 +gk20a_instobj_addr(struct nvkm_memory *memory) +{ + return gk20a_instobj(memory)->mem.offset; +} + +static u64 +gk20a_instobj_size(struct nvkm_memory *memory) +{ + return (u64)gk20a_instobj(memory)->mem.size << 12; +} + +static void __iomem * +gk20a_instobj_cpu_map_dma(struct nvkm_memory *memory) +{ +#if defined(CONFIG_ARM) || defined(CONFIG_ARM64) + struct gk20a_instobj_dma *node = gk20a_instobj_dma(memory); + struct device *dev = node->base.imem->base.subdev.device->dev; + int npages = nvkm_memory_size(memory) >> 12; + struct page *pages[npages]; + int i; + + /* we shouldn't see a gk20a on anything but arm/arm64 anyways */ + /* phys_to_page does not exist on all platforms... */ + pages[0] = pfn_to_page(dma_to_phys(dev, node->handle) >> PAGE_SHIFT); + for (i = 1; i < npages; i++) + pages[i] = pages[0] + i; + + return vmap(pages, npages, VM_MAP, pgprot_writecombine(PAGE_KERNEL)); +#else + BUG(); + return NULL; +#endif +} + +static void __iomem * +gk20a_instobj_cpu_map_iommu(struct nvkm_memory *memory) +{ + struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory); + int npages = nvkm_memory_size(memory) >> 12; + + return vmap(node->pages, npages, VM_MAP, + pgprot_writecombine(PAGE_KERNEL)); +} /* - * Use PRAMIN to read/write data and avoid coherency issues. - * PRAMIN uses the GPU path and ensures data will always be coherent. - * - * A dynamic mapping based solution would be desirable in the future, but - * the issue remains of how to maintain coherency efficiently. On ARM it is - * not easy (if possible at all?) to create uncached temporary mappings. + * Must be called while holding gk20a_instmem_lock */ +static void +gk20a_instmem_vaddr_gc(struct gk20a_instmem *imem, const u64 size) +{ + while (imem->vaddr_use + size > imem->vaddr_max) { + struct gk20a_instobj *obj; + + /* no candidate that can be unmapped, abort... */ + if (list_empty(&imem->vaddr_lru)) + break; + + obj = list_first_entry(&imem->vaddr_lru, struct gk20a_instobj, + vaddr_node); + list_del(&obj->vaddr_node); + vunmap(obj->vaddr); + obj->vaddr = NULL; + imem->vaddr_use -= nvkm_memory_size(&obj->memory); + nvkm_debug(&imem->base.subdev, "(GC) vaddr used: %x/%x\n", + imem->vaddr_use, imem->vaddr_max); -static u32 -gk20a_instobj_rd32(struct nvkm_object *object, u64 offset) + } +} + +static void __iomem * +gk20a_instobj_acquire(struct nvkm_memory *memory) { - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(object); - struct gk20a_instobj_priv *node = (void *)object; + struct gk20a_instobj *node = gk20a_instobj(memory); + struct gk20a_instmem *imem = node->imem; + struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; + const u64 size = nvkm_memory_size(memory); unsigned long flags; - u64 base = (node->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (node->mem->offset + offset) & 0x000000fffffULL; - u32 data; - - spin_lock_irqsave(&priv->lock, flags); - if (unlikely(priv->addr != base)) { - nv_wr32(priv, 0x001700, base >> 16); - priv->addr = base; + + nvkm_ltc_flush(ltc); + + spin_lock_irqsave(&imem->lock, flags); + + if (node->vaddr) { + /* remove us from the LRU list since we cannot be unmapped */ + list_del(&node->vaddr_node); + + goto out; + } + + /* try to free some address space if we reached the limit */ + gk20a_instmem_vaddr_gc(imem, size); + + node->vaddr = imem->cpu_map(memory); + + if (!node->vaddr) { + nvkm_error(&imem->base.subdev, "cannot map instobj - " + "this is not going to end well...\n"); + goto out; } - data = nv_rd32(priv, 0x700000 + addr); - spin_unlock_irqrestore(&priv->lock, flags); - return data; + + imem->vaddr_use += size; + nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", + imem->vaddr_use, imem->vaddr_max); + +out: + spin_unlock_irqrestore(&imem->lock, flags); + + return node->vaddr; } static void -gk20a_instobj_wr32(struct nvkm_object *object, u64 offset, u32 data) +gk20a_instobj_release(struct nvkm_memory *memory) { - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(object); - struct gk20a_instobj_priv *node = (void *)object; + struct gk20a_instobj *node = gk20a_instobj(memory); + struct gk20a_instmem *imem = node->imem; + struct nvkm_ltc *ltc = imem->base.subdev.device->ltc; unsigned long flags; - u64 base = (node->mem->offset + offset) & 0xffffff00000ULL; - u64 addr = (node->mem->offset + offset) & 0x000000fffffULL; - spin_lock_irqsave(&priv->lock, flags); - if (unlikely(priv->addr != base)) { - nv_wr32(priv, 0x001700, base >> 16); - priv->addr = base; - } - nv_wr32(priv, 0x700000 + addr, data); - spin_unlock_irqrestore(&priv->lock, flags); + spin_lock_irqsave(&imem->lock, flags); + + /* add ourselves to the LRU list so our CPU mapping can be freed */ + list_add_tail(&node->vaddr_node, &imem->vaddr_lru); + + spin_unlock_irqrestore(&imem->lock, flags); + + wmb(); + nvkm_ltc_invalidate(ltc); +} + +static u32 +gk20a_instobj_rd32(struct nvkm_memory *memory, u64 offset) +{ + struct gk20a_instobj *node = gk20a_instobj(memory); + + return node->vaddr[offset / 4]; } static void -gk20a_instobj_dtor_dma(struct gk20a_instobj_priv *_node) +gk20a_instobj_wr32(struct nvkm_memory *memory, u64 offset, u32 data) { - struct gk20a_instobj_dma *node = (void *)_node; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node); - struct device *dev = nv_device_base(nv_device(priv)); + struct gk20a_instobj *node = gk20a_instobj(memory); - if (unlikely(!node->cpuaddr)) - return; + node->vaddr[offset / 4] = data; +} - dma_free_attrs(dev, _node->mem->size << PAGE_SHIFT, node->cpuaddr, - node->handle, &priv->attrs); +static void +gk20a_instobj_map(struct nvkm_memory *memory, struct nvkm_vma *vma, u64 offset) +{ + struct gk20a_instobj *node = gk20a_instobj(memory); + + nvkm_vm_map_at(vma, offset, &node->mem); } +/* + * Clear the CPU mapping of an instobj if it exists + */ static void -gk20a_instobj_dtor_iommu(struct gk20a_instobj_priv *_node) +gk20a_instobj_dtor(struct gk20a_instobj *node) +{ + struct gk20a_instmem *imem = node->imem; + struct gk20a_instobj *obj; + unsigned long flags; + + spin_lock_irqsave(&imem->lock, flags); + + if (!node->vaddr) + goto out; + + list_for_each_entry(obj, &imem->vaddr_lru, vaddr_node) { + if (obj == node) { + list_del(&obj->vaddr_node); + break; + } + } + vunmap(node->vaddr); + node->vaddr = NULL; + imem->vaddr_use -= nvkm_memory_size(&node->memory); + nvkm_debug(&imem->base.subdev, "vaddr used: %x/%x\n", + imem->vaddr_use, imem->vaddr_max); + +out: + spin_unlock_irqrestore(&imem->lock, flags); +} + +static void * +gk20a_instobj_dtor_dma(struct nvkm_memory *memory) +{ + struct gk20a_instobj_dma *node = gk20a_instobj_dma(memory); + struct gk20a_instmem *imem = node->base.imem; + struct device *dev = imem->base.subdev.device->dev; + + gk20a_instobj_dtor(&node->base); + + if (unlikely(!node->cpuaddr)) + goto out; + + dma_free_attrs(dev, node->base.mem.size << PAGE_SHIFT, node->cpuaddr, + node->handle, &imem->attrs); + +out: + return node; +} + +static void * +gk20a_instobj_dtor_iommu(struct nvkm_memory *memory) { - struct gk20a_instobj_iommu *node = (void *)_node; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node); + struct gk20a_instobj_iommu *node = gk20a_instobj_iommu(memory); + struct gk20a_instmem *imem = node->base.imem; + struct device *dev = imem->base.subdev.device->dev; struct nvkm_mm_node *r; int i; - if (unlikely(list_empty(&_node->mem->regions))) - return; + gk20a_instobj_dtor(&node->base); - r = list_first_entry(&_node->mem->regions, struct nvkm_mm_node, + if (unlikely(list_empty(&node->base.mem.regions))) + goto out; + + r = list_first_entry(&node->base.mem.regions, struct nvkm_mm_node, rl_entry); - /* clear bit 34 to unmap pages */ - r->offset &= ~BIT(34 - priv->iommu_pgshift); + /* clear IOMMU bit to unmap pages */ + r->offset &= ~BIT(imem->iommu_bit - imem->iommu_pgshift); /* Unmap pages from GPU address space and free them */ - for (i = 0; i < _node->mem->size; i++) { - iommu_unmap(priv->domain, - (r->offset + i) << priv->iommu_pgshift, PAGE_SIZE); + for (i = 0; i < node->base.mem.size; i++) { + iommu_unmap(imem->domain, + (r->offset + i) << imem->iommu_pgshift, PAGE_SIZE); + dma_unmap_page(dev, node->dma_addrs[i], PAGE_SIZE, + DMA_BIDIRECTIONAL); __free_page(node->pages[i]); } /* Release area from GPU address space */ - mutex_lock(priv->mm_mutex); - nvkm_mm_free(priv->mm, &r); - mutex_unlock(priv->mm_mutex); -} + mutex_lock(imem->mm_mutex); + nvkm_mm_free(imem->mm, &r); + mutex_unlock(imem->mm_mutex); -static void -gk20a_instobj_dtor(struct nvkm_object *object) -{ - struct gk20a_instobj_priv *node = (void *)object; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(node); +out: + return node; +} - if (priv->domain) - gk20a_instobj_dtor_iommu(node); - else - gk20a_instobj_dtor_dma(node); +static const struct nvkm_memory_func +gk20a_instobj_func_dma = { + .dtor = gk20a_instobj_dtor_dma, + .target = gk20a_instobj_target, + .addr = gk20a_instobj_addr, + .size = gk20a_instobj_size, + .acquire = gk20a_instobj_acquire, + .release = gk20a_instobj_release, + .rd32 = gk20a_instobj_rd32, + .wr32 = gk20a_instobj_wr32, + .map = gk20a_instobj_map, +}; - nvkm_instobj_destroy(&node->base); -} +static const struct nvkm_memory_func +gk20a_instobj_func_iommu = { + .dtor = gk20a_instobj_dtor_iommu, + .target = gk20a_instobj_target, + .addr = gk20a_instobj_addr, + .size = gk20a_instobj_size, + .acquire = gk20a_instobj_acquire, + .release = gk20a_instobj_release, + .rd32 = gk20a_instobj_rd32, + .wr32 = gk20a_instobj_wr32, + .map = gk20a_instobj_map, +}; static int -gk20a_instobj_ctor_dma(struct nvkm_object *parent, struct nvkm_object *engine, - struct nvkm_oclass *oclass, u32 npages, u32 align, - struct gk20a_instobj_priv **_node) +gk20a_instobj_ctor_dma(struct gk20a_instmem *imem, u32 npages, u32 align, + struct gk20a_instobj **_node) { struct gk20a_instobj_dma *node; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent); - struct device *dev = nv_device_base(nv_device(parent)); - int ret; + struct nvkm_subdev *subdev = &imem->base.subdev; + struct device *dev = subdev->device->dev; - ret = nvkm_instobj_create_(parent, engine, oclass, sizeof(*node), - (void **)&node); + if (!(node = kzalloc(sizeof(*node), GFP_KERNEL))) + return -ENOMEM; *_node = &node->base; - if (ret) - return ret; + + nvkm_memory_ctor(&gk20a_instobj_func_dma, &node->base.memory); node->cpuaddr = dma_alloc_attrs(dev, npages << PAGE_SHIFT, &node->handle, GFP_KERNEL, - &priv->attrs); + &imem->attrs); if (!node->cpuaddr) { - nv_error(priv, "cannot allocate DMA memory\n"); + nvkm_error(subdev, "cannot allocate DMA memory\n"); return -ENOMEM; } /* alignment check */ if (unlikely(node->handle & (align - 1))) - nv_warn(priv, "memory not aligned as requested: %pad (0x%x)\n", - &node->handle, align); + nvkm_warn(subdev, + "memory not aligned as requested: %pad (0x%x)\n", + &node->handle, align); /* present memory for being mapped using small pages */ node->r.type = 12; node->r.offset = node->handle >> 12; node->r.length = (npages << PAGE_SHIFT) >> 12; - node->base._mem.offset = node->handle; + node->base.mem.offset = node->handle; - INIT_LIST_HEAD(&node->base._mem.regions); - list_add_tail(&node->r.rl_entry, &node->base._mem.regions); + INIT_LIST_HEAD(&node->base.mem.regions); + list_add_tail(&node->r.rl_entry, &node->base.mem.regions); return 0; } static int -gk20a_instobj_ctor_iommu(struct nvkm_object *parent, struct nvkm_object *engine, - struct nvkm_oclass *oclass, u32 npages, u32 align, - struct gk20a_instobj_priv **_node) +gk20a_instobj_ctor_iommu(struct gk20a_instmem *imem, u32 npages, u32 align, + struct gk20a_instobj **_node) { struct gk20a_instobj_iommu *node; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent); + struct nvkm_subdev *subdev = &imem->base.subdev; + struct device *dev = subdev->device->dev; struct nvkm_mm_node *r; int ret; int i; - ret = nvkm_instobj_create_(parent, engine, oclass, - sizeof(*node) + sizeof(node->pages[0]) * npages, - (void **)&node); + /* + * despite their variable size, instmem allocations are small enough + * (< 1 page) to be handled by kzalloc + */ + if (!(node = kzalloc(sizeof(*node) + ((sizeof(node->pages[0]) + + sizeof(*node->dma_addrs)) * npages), GFP_KERNEL))) + return -ENOMEM; *_node = &node->base; - if (ret) - return ret; + node->dma_addrs = (void *)(node->pages + npages); + + nvkm_memory_ctor(&gk20a_instobj_func_iommu, &node->base.memory); /* Allocate backing memory */ for (i = 0; i < npages; i++) { struct page *p = alloc_page(GFP_KERNEL); + dma_addr_t dma_adr; if (p == NULL) { ret = -ENOMEM; goto free_pages; } node->pages[i] = p; + dma_adr = dma_map_page(dev, p, 0, PAGE_SIZE, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma_adr)) { + nvkm_error(subdev, "DMA mapping error!\n"); + ret = -ENOMEM; + goto free_pages; + } + node->dma_addrs[i] = dma_adr; } - mutex_lock(priv->mm_mutex); + mutex_lock(imem->mm_mutex); /* Reserve area from GPU address space */ - ret = nvkm_mm_head(priv->mm, 0, 1, npages, npages, - align >> priv->iommu_pgshift, &r); - mutex_unlock(priv->mm_mutex); + ret = nvkm_mm_head(imem->mm, 0, 1, npages, npages, + align >> imem->iommu_pgshift, &r); + mutex_unlock(imem->mm_mutex); if (ret) { - nv_error(priv, "virtual space is full!\n"); + nvkm_error(subdev, "IOMMU space is full!\n"); goto free_pages; } /* Map into GPU address space */ for (i = 0; i < npages; i++) { - struct page *p = node->pages[i]; - u32 offset = (r->offset + i) << priv->iommu_pgshift; + u32 offset = (r->offset + i) << imem->iommu_pgshift; - ret = iommu_map(priv->domain, offset, page_to_phys(p), + ret = iommu_map(imem->domain, offset, node->dma_addrs[i], PAGE_SIZE, IOMMU_READ | IOMMU_WRITE); if (ret < 0) { - nv_error(priv, "IOMMU mapping failure: %d\n", ret); + nvkm_error(subdev, "IOMMU mapping failure: %d\n", ret); while (i-- > 0) { offset -= PAGE_SIZE; - iommu_unmap(priv->domain, offset, PAGE_SIZE); + iommu_unmap(imem->domain, offset, PAGE_SIZE); } goto release_area; } } - /* Bit 34 tells that an address is to be resolved through the IOMMU */ - r->offset |= BIT(34 - priv->iommu_pgshift); + /* IOMMU bit tells that an address is to be resolved through the IOMMU */ + r->offset |= BIT(imem->iommu_bit - imem->iommu_pgshift); - node->base._mem.offset = ((u64)r->offset) << priv->iommu_pgshift; + node->base.mem.offset = ((u64)r->offset) << imem->iommu_pgshift; - INIT_LIST_HEAD(&node->base._mem.regions); - list_add_tail(&r->rl_entry, &node->base._mem.regions); + INIT_LIST_HEAD(&node->base.mem.regions); + list_add_tail(&r->rl_entry, &node->base.mem.regions); return 0; release_area: - mutex_lock(priv->mm_mutex); - nvkm_mm_free(priv->mm, &r); - mutex_unlock(priv->mm_mutex); + mutex_lock(imem->mm_mutex); + nvkm_mm_free(imem->mm, &r); + mutex_unlock(imem->mm_mutex); free_pages: - for (i = 0; i < npages && node->pages[i] != NULL; i++) + for (i = 0; i < npages && node->pages[i] != NULL; i++) { + dma_addr_t dma_addr = node->dma_addrs[i]; + if (dma_addr) + dma_unmap_page(dev, dma_addr, PAGE_SIZE, + DMA_BIDIRECTIONAL); __free_page(node->pages[i]); + } return ret; } static int -gk20a_instobj_ctor(struct nvkm_object *parent, struct nvkm_object *engine, - struct nvkm_oclass *oclass, void *data, u32 _size, - struct nvkm_object **pobject) +gk20a_instobj_new(struct nvkm_instmem *base, u32 size, u32 align, bool zero, + struct nvkm_memory **pmemory) { - struct nvkm_instobj_args *args = data; - struct gk20a_instmem_priv *priv = (void *)nvkm_instmem(parent); - struct gk20a_instobj_priv *node; - u32 size, align; + struct gk20a_instmem *imem = gk20a_instmem(base); + struct nvkm_subdev *subdev = &imem->base.subdev; + struct gk20a_instobj *node = NULL; int ret; - nv_debug(parent, "%s (%s): size: %x align: %x\n", __func__, - priv->domain ? "IOMMU" : "DMA", args->size, args->align); + nvkm_debug(subdev, "%s (%s): size: %x align: %x\n", __func__, + imem->domain ? "IOMMU" : "DMA", size, align); /* Round size and align to page bounds */ - size = max(roundup(args->size, PAGE_SIZE), PAGE_SIZE); - align = max(roundup(args->align, PAGE_SIZE), PAGE_SIZE); + size = max(roundup(size, PAGE_SIZE), PAGE_SIZE); + align = max(roundup(align, PAGE_SIZE), PAGE_SIZE); - if (priv->domain) - ret = gk20a_instobj_ctor_iommu(parent, engine, oclass, - size >> PAGE_SHIFT, align, &node); + if (imem->domain) + ret = gk20a_instobj_ctor_iommu(imem, size >> PAGE_SHIFT, + align, &node); else - ret = gk20a_instobj_ctor_dma(parent, engine, oclass, - size >> PAGE_SHIFT, align, &node); - *pobject = nv_object(node); + ret = gk20a_instobj_ctor_dma(imem, size >> PAGE_SHIFT, + align, &node); + *pmemory = node ? &node->memory : NULL; if (ret) return ret; - node->mem = &node->_mem; + node->imem = imem; /* present memory for being mapped using small pages */ - node->mem->size = size >> 12; - node->mem->memtype = 0; - node->mem->page_shift = 12; - - node->base.addr = node->mem->offset; - node->base.size = size; + node->mem.size = size >> 12; + node->mem.memtype = 0; + node->mem.page_shift = 12; - nv_debug(parent, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n", - size, align, node->mem->offset); + nvkm_debug(subdev, "alloc size: 0x%x, align: 0x%x, gaddr: 0x%llx\n", + size, align, node->mem.offset); return 0; } -static struct nvkm_instobj_impl -gk20a_instobj_oclass = { - .base.ofuncs = &(struct nvkm_ofuncs) { - .ctor = gk20a_instobj_ctor, - .dtor = gk20a_instobj_dtor, - .init = _nvkm_instobj_init, - .fini = _nvkm_instobj_fini, - .rd32 = gk20a_instobj_rd32, - .wr32 = gk20a_instobj_wr32, - }, -}; +static void * +gk20a_instmem_dtor(struct nvkm_instmem *base) +{ + struct gk20a_instmem *imem = gk20a_instmem(base); + /* perform some sanity checks... */ + if (!list_empty(&imem->vaddr_lru)) + nvkm_warn(&base->subdev, "instobj LRU not empty!\n"); + if (imem->vaddr_use != 0) + nvkm_warn(&base->subdev, "instobj vmap area not empty! " + "0x%x bytes still mapped\n", imem->vaddr_use); -static int -gk20a_instmem_fini(struct nvkm_object *object, bool suspend) -{ - struct gk20a_instmem_priv *priv = (void *)object; - priv->addr = ~0ULL; - return nvkm_instmem_fini(&priv->base, suspend); + return imem; } -static int -gk20a_instmem_ctor(struct nvkm_object *parent, struct nvkm_object *engine, - struct nvkm_oclass *oclass, void *data, u32 size, - struct nvkm_object **pobject) -{ - struct gk20a_instmem_priv *priv; - struct nouveau_platform_device *plat; - int ret; - - ret = nvkm_instmem_create(parent, engine, oclass, &priv); - *pobject = nv_object(priv); - if (ret) - return ret; - - spin_lock_init(&priv->lock); +static const struct nvkm_instmem_func +gk20a_instmem = { + .dtor = gk20a_instmem_dtor, + .memory_new = gk20a_instobj_new, + .persistent = true, + .zero = false, +}; - plat = nv_device_to_platform(nv_device(parent)); - if (plat->gpu->iommu.domain) { - priv->domain = plat->gpu->iommu.domain; - priv->mm = plat->gpu->iommu.mm; - priv->iommu_pgshift = plat->gpu->iommu.pgshift; - priv->mm_mutex = &plat->gpu->iommu.mutex; +int +gk20a_instmem_new(struct nvkm_device *device, int index, + struct nvkm_instmem **pimem) +{ + struct nvkm_device_tegra *tdev = device->func->tegra(device); + struct gk20a_instmem *imem; - nv_info(priv, "using IOMMU\n"); + if (!(imem = kzalloc(sizeof(*imem), GFP_KERNEL))) + return -ENOMEM; + nvkm_instmem_ctor(&gk20a_instmem, device, index, &imem->base); + spin_lock_init(&imem->lock); + *pimem = &imem->base; + + /* do not allow more than 1MB of CPU-mapped instmem */ + imem->vaddr_use = 0; + imem->vaddr_max = 0x100000; + INIT_LIST_HEAD(&imem->vaddr_lru); + + if (tdev->iommu.domain) { + imem->mm_mutex = &tdev->iommu.mutex; + imem->mm = &tdev->iommu.mm; + imem->domain = tdev->iommu.domain; + imem->iommu_pgshift = tdev->iommu.pgshift; + imem->cpu_map = gk20a_instobj_cpu_map_iommu; + imem->iommu_bit = tdev->func->iommu_bit; + + nvkm_info(&imem->base.subdev, "using IOMMU\n"); } else { - init_dma_attrs(&priv->attrs); - /* - * We will access instmem through PRAMIN and thus do not need a - * consistent CPU pointer or kernel mapping - */ - dma_set_attr(DMA_ATTR_NON_CONSISTENT, &priv->attrs); - dma_set_attr(DMA_ATTR_WEAK_ORDERING, &priv->attrs); - dma_set_attr(DMA_ATTR_WRITE_COMBINE, &priv->attrs); - dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &priv->attrs); - - nv_info(priv, "using DMA API\n"); + init_dma_attrs(&imem->attrs); + /* We will access the memory through our own mapping */ + dma_set_attr(DMA_ATTR_NON_CONSISTENT, &imem->attrs); + dma_set_attr(DMA_ATTR_WEAK_ORDERING, &imem->attrs); + dma_set_attr(DMA_ATTR_WRITE_COMBINE, &imem->attrs); + dma_set_attr(DMA_ATTR_NO_KERNEL_MAPPING, &imem->attrs); + imem->cpu_map = gk20a_instobj_cpu_map_dma; + + nvkm_info(&imem->base.subdev, "using DMA API\n"); } return 0; } - -struct nvkm_oclass * -gk20a_instmem_oclass = &(struct nvkm_instmem_impl) { - .base.handle = NV_SUBDEV(INSTMEM, 0xea), - .base.ofuncs = &(struct nvkm_ofuncs) { - .ctor = gk20a_instmem_ctor, - .dtor = _nvkm_instmem_dtor, - .init = _nvkm_instmem_init, - .fini = gk20a_instmem_fini, - }, - .instobj = &gk20a_instobj_oclass.base, -}.base;