--- /dev/null
+/*
+ * linux/arch/alpha/kernel/pci_iommu.c
+ */
+
+#include <linux/kernel.h>
+#include <linux/mm.h>
+#include <linux/pci.h>
+#include <linux/gfp.h>
+#include <linux/bootmem.h>
+#include <linux/export.h>
+#include <linux/scatterlist.h>
+#include <linux/log2.h>
+#include <linux/dma-mapping.h>
+#include <linux/iommu-helper.h>
+
+#include <asm/io.h>
+#include <asm/hwrpb.h>
+
+#include "proto.h"
+#include "pci_impl.h"
+
+
+#define DEBUG_ALLOC 0
+#if DEBUG_ALLOC > 0
+# define DBGA(args...) printk(KERN_DEBUG args)
+#else
+# define DBGA(args...)
+#endif
+#if DEBUG_ALLOC > 1
+# define DBGA2(args...) printk(KERN_DEBUG args)
+#else
+# define DBGA2(args...)
+#endif
+
+#define DEBUG_NODIRECT 0
+
+#define ISA_DMA_MASK 0x00ffffff
+
+static inline unsigned long
+mk_iommu_pte(unsigned long paddr)
+{
+ return (paddr >> (PAGE_SHIFT-1)) | 1;
+}
+
+/* Return the minimum of MAX or the first power of two larger
+ than main memory. */
+
+unsigned long
+size_for_memory(unsigned long max)
+{
+ unsigned long mem = max_low_pfn << PAGE_SHIFT;
+ if (mem < max)
+ max = roundup_pow_of_two(mem);
+ return max;
+}
+\f
+struct pci_iommu_arena * __init
+iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base,
+ unsigned long window_size, unsigned long align)
+{
+ unsigned long mem_size;
+ struct pci_iommu_arena *arena;
+
+ mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long));
+
+ /* Note that the TLB lookup logic uses bitwise concatenation,
+ not addition, so the required arena alignment is based on
+ the size of the window. Retain the align parameter so that
+ particular systems can over-align the arena. */
+ if (align < mem_size)
+ align = mem_size;
+
+
+#ifdef CONFIG_DISCONTIGMEM
+
+ arena = alloc_bootmem_node(NODE_DATA(nid), sizeof(*arena));
+ if (!NODE_DATA(nid) || !arena) {
+ printk("%s: couldn't allocate arena from node %d\n"
+ " falling back to system-wide allocation\n",
+ __func__, nid);
+ arena = alloc_bootmem(sizeof(*arena));
+ }
+
+ arena->ptes = __alloc_bootmem_node(NODE_DATA(nid), mem_size, align, 0);
+ if (!NODE_DATA(nid) || !arena->ptes) {
+ printk("%s: couldn't allocate arena ptes from node %d\n"
+ " falling back to system-wide allocation\n",
+ __func__, nid);
+ arena->ptes = __alloc_bootmem(mem_size, align, 0);
+ }
+
+#else /* CONFIG_DISCONTIGMEM */
+
+ arena = alloc_bootmem(sizeof(*arena));
+ arena->ptes = __alloc_bootmem(mem_size, align, 0);
+
+#endif /* CONFIG_DISCONTIGMEM */
+
+ spin_lock_init(&arena->lock);
+ arena->hose = hose;
+ arena->dma_base = base;
+ arena->size = window_size;
+ arena->next_entry = 0;
+
+ /* Align allocations to a multiple of a page size. Not needed
+ unless there are chip bugs. */
+ arena->align_entry = 1;
+
+ return arena;
+}
+
+struct pci_iommu_arena * __init
+iommu_arena_new(struct pci_controller *hose, dma_addr_t base,
+ unsigned long window_size, unsigned long align)
+{
+ return iommu_arena_new_node(0, hose, base, window_size, align);
+}
+
+/* Must be called with the arena lock held */
+static long
+iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena,
+ long n, long mask)
+{
+ unsigned long *ptes;
+ long i, p, nent;
+ int pass = 0;
+ unsigned long base;
+ unsigned long boundary_size;
+
+ base = arena->dma_base >> PAGE_SHIFT;
+ if (dev) {
+ boundary_size = dma_get_seg_boundary(dev) + 1;
+ boundary_size >>= PAGE_SHIFT;
+ } else {
+ boundary_size = 1UL << (32 - PAGE_SHIFT);
+ }
+
+ /* Search forward for the first mask-aligned sequence of N free ptes */
+ ptes = arena->ptes;
+ nent = arena->size >> PAGE_SHIFT;
+ p = ALIGN(arena->next_entry, mask + 1);
+ i = 0;
+
+again:
+ while (i < n && p+i < nent) {
+ if (!i && iommu_is_span_boundary(p, n, base, boundary_size)) {
+ p = ALIGN(p + 1, mask + 1);
+ goto again;
+ }
+
+ if (ptes[p+i])
+ p = ALIGN(p + i + 1, mask + 1), i = 0;
+ else
+ i = i + 1;
+ }
+
+ if (i < n) {
+ if (pass < 1) {
+ /*
+ * Reached the end. Flush the TLB and restart
+ * the search from the beginning.
+ */
+ alpha_mv.mv_pci_tbi(arena->hose, 0, -1);
+
+ pass++;
+ p = 0;
+ i = 0;
+ goto again;
+ } else
+ return -1;
+ }
+
+ /* Success. It's the responsibility of the caller to mark them
+ in use before releasing the lock */
+ return p;
+}
+
+static long
+iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n,
+ unsigned int align)
+{
+ unsigned long flags;
+ unsigned long *ptes;
+ long i, p, mask;
+
+ spin_lock_irqsave(&arena->lock, flags);
+
+ /* Search for N empty ptes */
+ ptes = arena->ptes;
+ mask = max(align, arena->align_entry) - 1;
+ p = iommu_arena_find_pages(dev, arena, n, mask);
+ if (p < 0) {
+ spin_unlock_irqrestore(&arena->lock, flags);
+ return -1;
+ }
+
+ /* Success. Mark them all in use, ie not zero and invalid
+ for the iommu tlb that could load them from under us.
+ The chip specific bits will fill this in with something
+ kosher when we return. */
+ for (i = 0; i < n; ++i)
+ ptes[p+i] = IOMMU_INVALID_PTE;
+
+ arena->next_entry = p + n;
+ spin_unlock_irqrestore(&arena->lock, flags);
+
+ return p;
+}
+
+static void
+iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n)
+{
+ unsigned long *p;
+ long i;
+
+ p = arena->ptes + ofs;
+ for (i = 0; i < n; ++i)
+ p[i] = 0;
+}
+
+/*
+ * True if the machine supports DAC addressing, and DEV can
+ * make use of it given MASK.
+ */
+static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask)
+{
+ dma_addr_t dac_offset = alpha_mv.pci_dac_offset;
+ int ok = 1;
+
+ /* If this is not set, the machine doesn't support DAC at all. */
+ if (dac_offset == 0)
+ ok = 0;
+
+ /* The device has to be able to address our DAC bit. */
+ if ((dac_offset & dev->dma_mask) != dac_offset)
+ ok = 0;
+
+ /* If both conditions above are met, we are fine. */
+ DBGA("pci_dac_dma_supported %s from %pf\n",
+ ok ? "yes" : "no", __builtin_return_address(0));
+
+ return ok;
+}
+
+/* Map a single buffer of the indicated size for PCI DMA in streaming
+ mode. The 32-bit PCI bus mastering address to use is returned.
+ Once the device is given the dma address, the device owns this memory
+ until either pci_unmap_single or pci_dma_sync_single is performed. */
+
+static dma_addr_t
+pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size,
+ int dac_allowed)
+{
+ struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
+ dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
+ struct pci_iommu_arena *arena;
+ long npages, dma_ofs, i;
+ unsigned long paddr;
+ dma_addr_t ret;
+ unsigned int align = 0;
+ struct device *dev = pdev ? &pdev->dev : NULL;
+
+ paddr = __pa(cpu_addr);
+
+#if !DEBUG_NODIRECT
+ /* First check to see if we can use the direct map window. */
+ if (paddr + size + __direct_map_base - 1 <= max_dma
+ && paddr + size <= __direct_map_size) {
+ ret = paddr + __direct_map_base;
+
+ DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %pf\n",
+ cpu_addr, size, ret, __builtin_return_address(0));
+
+ return ret;
+ }
+#endif
+
+ /* Next, use DAC if selected earlier. */
+ if (dac_allowed) {
+ ret = paddr + alpha_mv.pci_dac_offset;
+
+ DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %pf\n",
+ cpu_addr, size, ret, __builtin_return_address(0));
+
+ return ret;
+ }
+
+ /* If the machine doesn't define a pci_tbi routine, we have to
+ assume it doesn't support sg mapping, and, since we tried to
+ use direct_map above, it now must be considered an error. */
+ if (! alpha_mv.mv_pci_tbi) {
+ printk_once(KERN_WARNING "pci_map_single: no HW sg\n");
+ return 0;
+ }
+
+ arena = hose->sg_pci;
+ if (!arena || arena->dma_base + arena->size - 1 > max_dma)
+ arena = hose->sg_isa;
+
+ npages = iommu_num_pages(paddr, size, PAGE_SIZE);
+
+ /* Force allocation to 64KB boundary for ISA bridges. */
+ if (pdev && pdev == isa_bridge)
+ align = 8;
+ dma_ofs = iommu_arena_alloc(dev, arena, npages, align);
+ if (dma_ofs < 0) {
+ printk(KERN_WARNING "pci_map_single failed: "
+ "could not allocate dma page tables\n");
+ return 0;
+ }
+
+ paddr &= PAGE_MASK;
+ for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
+ arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr);
+
+ ret = arena->dma_base + dma_ofs * PAGE_SIZE;
+ ret += (unsigned long)cpu_addr & ~PAGE_MASK;
+
+ DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %pf\n",
+ cpu_addr, size, npages, ret, __builtin_return_address(0));
+
+ return ret;
+}
+
+/* Helper for generic DMA-mapping functions. */
+static struct pci_dev *alpha_gendev_to_pci(struct device *dev)
+{
+ if (dev && dev_is_pci(dev))
+ return to_pci_dev(dev);
+
+ /* Assume that non-PCI devices asking for DMA are either ISA or EISA,
+ BUG() otherwise. */
+ BUG_ON(!isa_bridge);
+
+ /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA
+ bridge is bus master then). */
+ if (!dev || !dev->dma_mask || !*dev->dma_mask)
+ return isa_bridge;
+
+ /* For EISA bus masters, return isa_bridge (it might have smaller
+ dma_mask due to wiring limitations). */
+ if (*dev->dma_mask >= isa_bridge->dma_mask)
+ return isa_bridge;
+
+ /* This assumes ISA bus master with dma_mask 0xffffff. */
+ return NULL;
+}
+
+static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page,
+ unsigned long offset, size_t size,
+ enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ int dac_allowed;
+
+ BUG_ON(dir == PCI_DMA_NONE);
+
+ dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
+ return pci_map_single_1(pdev, (char *)page_address(page) + offset,
+ size, dac_allowed);
+}
+
+/* Unmap a single streaming mode DMA translation. The DMA_ADDR and
+ SIZE must match what was provided for in a previous pci_map_single
+ call. All other usages are undefined. After this call, reads by
+ the cpu to the buffer are guaranteed to see whatever the device
+ wrote there. */
+
+static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr,
+ size_t size, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ unsigned long flags;
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose;
+ struct pci_iommu_arena *arena;
+ long dma_ofs, npages;
+
+ BUG_ON(dir == PCI_DMA_NONE);
+
+ if (dma_addr >= __direct_map_base
+ && dma_addr < __direct_map_base + __direct_map_size) {
+ /* Nothing to do. */
+
+ DBGA2("pci_unmap_single: direct [%llx,%zx] from %pf\n",
+ dma_addr, size, __builtin_return_address(0));
+
+ return;
+ }
+
+ if (dma_addr > 0xffffffff) {
+ DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %pf\n",
+ dma_addr, size, __builtin_return_address(0));
+ return;
+ }
+
+ arena = hose->sg_pci;
+ if (!arena || dma_addr < arena->dma_base)
+ arena = hose->sg_isa;
+
+ dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT;
+ if (dma_ofs * PAGE_SIZE >= arena->size) {
+ printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx "
+ " base %llx size %x\n",
+ dma_addr, arena->dma_base, arena->size);
+ return;
+ BUG();
+ }
+
+ npages = iommu_num_pages(dma_addr, size, PAGE_SIZE);
+
+ spin_lock_irqsave(&arena->lock, flags);
+
+ iommu_arena_free(arena, dma_ofs, npages);
+
+ /* If we're freeing ptes above the `next_entry' pointer (they
+ may have snuck back into the TLB since the last wrap flush),
+ we need to flush the TLB before reallocating the latter. */
+ if (dma_ofs >= arena->next_entry)
+ alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1);
+
+ spin_unlock_irqrestore(&arena->lock, flags);
+
+ DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %pf\n",
+ dma_addr, size, npages, __builtin_return_address(0));
+}
+
+/* Allocate and map kernel buffer using consistent mode DMA for PCI
+ device. Returns non-NULL cpu-view pointer to the buffer if
+ successful and sets *DMA_ADDRP to the pci side dma address as well,
+ else DMA_ADDRP is undefined. */
+
+static void *alpha_pci_alloc_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_addrp, gfp_t gfp,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ void *cpu_addr;
+ long order = get_order(size);
+
+ gfp &= ~GFP_DMA;
+
+try_again:
+ cpu_addr = (void *)__get_free_pages(gfp, order);
+ if (! cpu_addr) {
+ printk(KERN_INFO "pci_alloc_consistent: "
+ "get_free_pages failed from %pf\n",
+ __builtin_return_address(0));
+ /* ??? Really atomic allocation? Otherwise we could play
+ with vmalloc and sg if we can't find contiguous memory. */
+ return NULL;
+ }
+ memset(cpu_addr, 0, size);
+
+ *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0);
+ if (*dma_addrp == 0) {
+ free_pages((unsigned long)cpu_addr, order);
+ if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA))
+ return NULL;
+ /* The address doesn't fit required mask and we
+ do not have iommu. Try again with GFP_DMA. */
+ gfp |= GFP_DMA;
+ goto try_again;
+ }
+
+ DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %pf\n",
+ size, cpu_addr, *dma_addrp, __builtin_return_address(0));
+
+ return cpu_addr;
+}
+
+/* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must
+ be values that were returned from pci_alloc_consistent. SIZE must
+ be the same as what as passed into pci_alloc_consistent.
+ References to the memory and mappings associated with CPU_ADDR or
+ DMA_ADDR past this call are illegal. */
+
+static void alpha_pci_free_coherent(struct device *dev, size_t size,
+ void *cpu_addr, dma_addr_t dma_addr,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ pci_unmap_single(pdev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
+ free_pages((unsigned long)cpu_addr, get_order(size));
+
+ DBGA2("pci_free_consistent: [%llx,%zx] from %pf\n",
+ dma_addr, size, __builtin_return_address(0));
+}
+
+/* Classify the elements of the scatterlist. Write dma_address
+ of each element with:
+ 0 : Followers all physically adjacent.
+ 1 : Followers all virtually adjacent.
+ -1 : Not leader, physically adjacent to previous.
+ -2 : Not leader, virtually adjacent to previous.
+ Write dma_length of each leader with the combined lengths of
+ the mergable followers. */
+
+#define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG)))
+#define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG))
+
+static void
+sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end,
+ int virt_ok)
+{
+ unsigned long next_paddr;
+ struct scatterlist *leader;
+ long leader_flag, leader_length;
+ unsigned int max_seg_size;
+
+ leader = sg;
+ leader_flag = 0;
+ leader_length = leader->length;
+ next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length;
+
+ /* we will not marge sg without device. */
+ max_seg_size = dev ? dma_get_max_seg_size(dev) : 0;
+ for (++sg; sg < end; ++sg) {
+ unsigned long addr, len;
+ addr = SG_ENT_PHYS_ADDRESS(sg);
+ len = sg->length;
+
+ if (leader_length + len > max_seg_size)
+ goto new_segment;
+
+ if (next_paddr == addr) {
+ sg->dma_address = -1;
+ leader_length += len;
+ } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) {
+ sg->dma_address = -2;
+ leader_flag = 1;
+ leader_length += len;
+ } else {
+new_segment:
+ leader->dma_address = leader_flag;
+ leader->dma_length = leader_length;
+ leader = sg;
+ leader_flag = 0;
+ leader_length = len;
+ }
+
+ next_paddr = addr + len;
+ }
+
+ leader->dma_address = leader_flag;
+ leader->dma_length = leader_length;
+}
+
+/* Given a scatterlist leader, choose an allocation method and fill
+ in the blanks. */
+
+static int
+sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end,
+ struct scatterlist *out, struct pci_iommu_arena *arena,
+ dma_addr_t max_dma, int dac_allowed)
+{
+ unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader);
+ long size = leader->dma_length;
+ struct scatterlist *sg;
+ unsigned long *ptes;
+ long npages, dma_ofs, i;
+
+#if !DEBUG_NODIRECT
+ /* If everything is physically contiguous, and the addresses
+ fall into the direct-map window, use it. */
+ if (leader->dma_address == 0
+ && paddr + size + __direct_map_base - 1 <= max_dma
+ && paddr + size <= __direct_map_size) {
+ out->dma_address = paddr + __direct_map_base;
+ out->dma_length = size;
+
+ DBGA(" sg_fill: [%p,%lx] -> direct %llx\n",
+ __va(paddr), size, out->dma_address);
+
+ return 0;
+ }
+#endif
+
+ /* If physically contiguous and DAC is available, use it. */
+ if (leader->dma_address == 0 && dac_allowed) {
+ out->dma_address = paddr + alpha_mv.pci_dac_offset;
+ out->dma_length = size;
+
+ DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n",
+ __va(paddr), size, out->dma_address);
+
+ return 0;
+ }
+
+ /* Otherwise, we'll use the iommu to make the pages virtually
+ contiguous. */
+
+ paddr &= ~PAGE_MASK;
+ npages = iommu_num_pages(paddr, size, PAGE_SIZE);
+ dma_ofs = iommu_arena_alloc(dev, arena, npages, 0);
+ if (dma_ofs < 0) {
+ /* If we attempted a direct map above but failed, die. */
+ if (leader->dma_address == 0)
+ return -1;
+
+ /* Otherwise, break up the remaining virtually contiguous
+ hunks into individual direct maps and retry. */
+ sg_classify(dev, leader, end, 0);
+ return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed);
+ }
+
+ out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr;
+ out->dma_length = size;
+
+ DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n",
+ __va(paddr), size, out->dma_address, npages);
+
+ /* All virtually contiguous. We need to find the length of each
+ physically contiguous subsegment to fill in the ptes. */
+ ptes = &arena->ptes[dma_ofs];
+ sg = leader;
+ do {
+#if DEBUG_ALLOC > 0
+ struct scatterlist *last_sg = sg;
+#endif
+
+ size = sg->length;
+ paddr = SG_ENT_PHYS_ADDRESS(sg);
+
+ while (sg+1 < end && (int) sg[1].dma_address == -1) {
+ size += sg[1].length;
+ sg++;
+ }
+
+ npages = iommu_num_pages(paddr, size, PAGE_SIZE);
+
+ paddr &= PAGE_MASK;
+ for (i = 0; i < npages; ++i, paddr += PAGE_SIZE)
+ *ptes++ = mk_iommu_pte(paddr);
+
+#if DEBUG_ALLOC > 0
+ DBGA(" (%ld) [%p,%x] np %ld\n",
+ last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
+ last_sg->length, npages);
+ while (++last_sg <= sg) {
+ DBGA(" (%ld) [%p,%x] cont\n",
+ last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg),
+ last_sg->length);
+ }
+#endif
+ } while (++sg < end && (int) sg->dma_address < 0);
+
+ return 1;
+}
+
+static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ struct scatterlist *start, *end, *out;
+ struct pci_controller *hose;
+ struct pci_iommu_arena *arena;
+ dma_addr_t max_dma;
+ int dac_allowed;
+
+ BUG_ON(dir == PCI_DMA_NONE);
+
+ dac_allowed = dev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0;
+
+ /* Fast path single entry scatterlists. */
+ if (nents == 1) {
+ sg->dma_length = sg->length;
+ sg->dma_address
+ = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg),
+ sg->length, dac_allowed);
+ return sg->dma_address != 0;
+ }
+
+ start = sg;
+ end = sg + nents;
+
+ /* First, prepare information about the entries. */
+ sg_classify(dev, sg, end, alpha_mv.mv_pci_tbi != 0);
+
+ /* Second, figure out where we're going to map things. */
+ if (alpha_mv.mv_pci_tbi) {
+ hose = pdev ? pdev->sysdata : pci_isa_hose;
+ max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
+ arena = hose->sg_pci;
+ if (!arena || arena->dma_base + arena->size - 1 > max_dma)
+ arena = hose->sg_isa;
+ } else {
+ max_dma = -1;
+ arena = NULL;
+ hose = NULL;
+ }
+
+ /* Third, iterate over the scatterlist leaders and allocate
+ dma space as needed. */
+ for (out = sg; sg < end; ++sg) {
+ if ((int) sg->dma_address < 0)
+ continue;
+ if (sg_fill(dev, sg, end, out, arena, max_dma, dac_allowed) < 0)
+ goto error;
+ out++;
+ }
+
+ /* Mark the end of the list for pci_unmap_sg. */
+ if (out < end)
+ out->dma_length = 0;
+
+ if (out - start == 0)
+ printk(KERN_WARNING "pci_map_sg failed: no entries?\n");
+ DBGA("pci_map_sg: %ld entries\n", out - start);
+
+ return out - start;
+
+ error:
+ printk(KERN_WARNING "pci_map_sg failed: "
+ "could not allocate dma page tables\n");
+
+ /* Some allocation failed while mapping the scatterlist
+ entries. Unmap them now. */
+ if (out > start)
+ pci_unmap_sg(pdev, start, out - start, dir);
+ return 0;
+}
+
+/* Unmap a set of streaming mode DMA translations. Again, cpu read
+ rules concerning calls here are the same as for pci_unmap_single()
+ above. */
+
+static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg,
+ int nents, enum dma_data_direction dir,
+ struct dma_attrs *attrs)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ unsigned long flags;
+ struct pci_controller *hose;
+ struct pci_iommu_arena *arena;
+ struct scatterlist *end;
+ dma_addr_t max_dma;
+ dma_addr_t fbeg, fend;
+
+ BUG_ON(dir == PCI_DMA_NONE);
+
+ if (! alpha_mv.mv_pci_tbi)
+ return;
+
+ hose = pdev ? pdev->sysdata : pci_isa_hose;
+ max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK;
+ arena = hose->sg_pci;
+ if (!arena || arena->dma_base + arena->size - 1 > max_dma)
+ arena = hose->sg_isa;
+
+ fbeg = -1, fend = 0;
+
+ spin_lock_irqsave(&arena->lock, flags);
+
+ for (end = sg + nents; sg < end; ++sg) {
+ dma_addr_t addr;
+ size_t size;
+ long npages, ofs;
+ dma_addr_t tend;
+
+ addr = sg->dma_address;
+ size = sg->dma_length;
+ if (!size)
+ break;
+
+ if (addr > 0xffffffff) {
+ /* It's a DAC address -- nothing to do. */
+ DBGA(" (%ld) DAC [%llx,%zx]\n",
+ sg - end + nents, addr, size);
+ continue;
+ }
+
+ if (addr >= __direct_map_base
+ && addr < __direct_map_base + __direct_map_size) {
+ /* Nothing to do. */
+ DBGA(" (%ld) direct [%llx,%zx]\n",
+ sg - end + nents, addr, size);
+ continue;
+ }
+
+ DBGA(" (%ld) sg [%llx,%zx]\n",
+ sg - end + nents, addr, size);
+
+ npages = iommu_num_pages(addr, size, PAGE_SIZE);
+ ofs = (addr - arena->dma_base) >> PAGE_SHIFT;
+ iommu_arena_free(arena, ofs, npages);
+
+ tend = addr + size - 1;
+ if (fbeg > addr) fbeg = addr;
+ if (fend < tend) fend = tend;
+ }
+
+ /* If we're freeing ptes above the `next_entry' pointer (they
+ may have snuck back into the TLB since the last wrap flush),
+ we need to flush the TLB before reallocating the latter. */
+ if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry)
+ alpha_mv.mv_pci_tbi(hose, fbeg, fend);
+
+ spin_unlock_irqrestore(&arena->lock, flags);
+
+ DBGA("pci_unmap_sg: %ld entries\n", nents - (end - sg));
+}
+
+/* Return whether the given PCI device DMA address mask can be
+ supported properly. */
+
+static int alpha_pci_supported(struct device *dev, u64 mask)
+{
+ struct pci_dev *pdev = alpha_gendev_to_pci(dev);
+ struct pci_controller *hose;
+ struct pci_iommu_arena *arena;
+
+ /* If there exists a direct map, and the mask fits either
+ the entire direct mapped space or the total system memory as
+ shifted by the map base */
+ if (__direct_map_size != 0
+ && (__direct_map_base + __direct_map_size - 1 <= mask ||
+ __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask))
+ return 1;
+
+ /* Check that we have a scatter-gather arena that fits. */
+ hose = pdev ? pdev->sysdata : pci_isa_hose;
+ arena = hose->sg_isa;
+ if (arena && arena->dma_base + arena->size - 1 <= mask)
+ return 1;
+ arena = hose->sg_pci;
+ if (arena && arena->dma_base + arena->size - 1 <= mask)
+ return 1;
+
+ /* As last resort try ZONE_DMA. */
+ if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask)
+ return 1;
+
+ return 0;
+}
+
+\f
+/*
+ * AGP GART extensions to the IOMMU
+ */
+int
+iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask)
+{
+ unsigned long flags;
+ unsigned long *ptes;
+ long i, p;
+
+ if (!arena) return -EINVAL;
+
+ spin_lock_irqsave(&arena->lock, flags);
+
+ /* Search for N empty ptes. */
+ ptes = arena->ptes;
+ p = iommu_arena_find_pages(NULL, arena, pg_count, align_mask);
+ if (p < 0) {
+ spin_unlock_irqrestore(&arena->lock, flags);
+ return -1;
+ }
+
+ /* Success. Mark them all reserved (ie not zero and invalid)
+ for the iommu tlb that could load them from under us.
+ They will be filled in with valid bits by _bind() */
+ for (i = 0; i < pg_count; ++i)
+ ptes[p+i] = IOMMU_RESERVED_PTE;
+
+ arena->next_entry = p + pg_count;
+ spin_unlock_irqrestore(&arena->lock, flags);
+
+ return p;
+}
+
+int
+iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count)
+{
+ unsigned long *ptes;
+ long i;
+
+ if (!arena) return -EINVAL;
+
+ ptes = arena->ptes;
+
+ /* Make sure they're all reserved first... */
+ for(i = pg_start; i < pg_start + pg_count; i++)
+ if (ptes[i] != IOMMU_RESERVED_PTE)
+ return -EBUSY;
+
+ iommu_arena_free(arena, pg_start, pg_count);
+ return 0;
+}
+
+int
+iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count,
+ struct page **pages)
+{
+ unsigned long flags;
+ unsigned long *ptes;
+ long i, j;
+
+ if (!arena) return -EINVAL;
+
+ spin_lock_irqsave(&arena->lock, flags);
+
+ ptes = arena->ptes;
+
+ for(j = pg_start; j < pg_start + pg_count; j++) {
+ if (ptes[j] != IOMMU_RESERVED_PTE) {
+ spin_unlock_irqrestore(&arena->lock, flags);
+ return -EBUSY;
+ }
+ }
+
+ for(i = 0, j = pg_start; i < pg_count; i++, j++)
+ ptes[j] = mk_iommu_pte(page_to_phys(pages[i]));
+
+ spin_unlock_irqrestore(&arena->lock, flags);
+
+ return 0;
+}
+
+int
+iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count)
+{
+ unsigned long *p;
+ long i;
+
+ if (!arena) return -EINVAL;
+
+ p = arena->ptes + pg_start;
+ for(i = 0; i < pg_count; i++)
+ p[i] = IOMMU_RESERVED_PTE;
+
+ return 0;
+}
+
+static int alpha_pci_mapping_error(struct device *dev, dma_addr_t dma_addr)
+{
+ return dma_addr == 0;
+}
+
+static int alpha_pci_set_mask(struct device *dev, u64 mask)
+{
+ if (!dev->dma_mask ||
+ !pci_dma_supported(alpha_gendev_to_pci(dev), mask))
+ return -EIO;
+
+ *dev->dma_mask = mask;
+ return 0;
+}
+
+struct dma_map_ops alpha_pci_ops = {
+ .alloc = alpha_pci_alloc_coherent,
+ .free = alpha_pci_free_coherent,
+ .map_page = alpha_pci_map_page,
+ .unmap_page = alpha_pci_unmap_page,
+ .map_sg = alpha_pci_map_sg,
+ .unmap_sg = alpha_pci_unmap_sg,
+ .mapping_error = alpha_pci_mapping_error,
+ .dma_supported = alpha_pci_supported,
+ .set_dma_mask = alpha_pci_set_mask,
+};
+
+struct dma_map_ops *dma_ops = &alpha_pci_ops;
+EXPORT_SYMBOL(dma_ops);
Code Review / kvmfornfv.git / blobdiff