*/
#include <linux/bitops.h>
+#include <linux/debugfs.h>
#include <linux/err.h>
#include <linux/iommu.h>
#include <linux/kernel.h>
struct mutex lock;
struct list_head list;
+
+ struct dentry *debugfs;
};
struct tegra_smmu_as {
struct iommu_domain domain;
struct tegra_smmu *smmu;
unsigned int use_count;
- struct page *count;
+ u32 *count;
+ struct page **pts;
struct page *pd;
+ dma_addr_t pd_dma;
unsigned id;
u32 attr;
};
#define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f)
#define SMMU_PTB_DATA 0x020
-#define SMMU_PTB_DATA_VALUE(page, attr) (page_to_phys(page) >> 12 | (attr))
+#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr))
-#define SMMU_MK_PDE(page, attr) (page_to_phys(page) >> SMMU_PTE_SHIFT | (attr))
+#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr))
#define SMMU_TLB_FLUSH 0x030
#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0)
#define SMMU_PTE_ATTR (SMMU_PTE_READABLE | SMMU_PTE_WRITABLE | \
SMMU_PTE_NONSECURE)
-static inline void smmu_flush_ptc(struct tegra_smmu *smmu, struct page *page,
+static unsigned int iova_pd_index(unsigned long iova)
+{
+ return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);
+}
+
+static unsigned int iova_pt_index(unsigned long iova)
+{
+ return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);
+}
+
+static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)
+{
+ addr >>= 12;
+ return (addr & smmu->pfn_mask) == addr;
+}
+
+static dma_addr_t smmu_pde_to_dma(u32 pde)
+{
+ return pde << 12;
+}
+
+static void smmu_flush_ptc_all(struct tegra_smmu *smmu)
+{
+ smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);
+}
+
+static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,
unsigned long offset)
{
- phys_addr_t phys = page ? page_to_phys(page) : 0;
u32 value;
- if (page) {
- offset &= ~(smmu->mc->soc->atom_size - 1);
+ offset &= ~(smmu->mc->soc->atom_size - 1);
- if (smmu->mc->soc->num_address_bits > 32) {
-#ifdef CONFIG_PHYS_ADDR_T_64BIT
- value = (phys >> 32) & SMMU_PTC_FLUSH_HI_MASK;
+ if (smmu->mc->soc->num_address_bits > 32) {
+#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT
+ value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK;
#else
- value = 0;
+ value = 0;
#endif
- smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
- }
-
- value = (phys + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
- } else {
- value = SMMU_PTC_FLUSH_TYPE_ALL;
+ smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);
}
+ value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR;
smmu_writel(smmu, value, SMMU_PTC_FLUSH);
}
static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type)
{
struct tegra_smmu_as *as;
- unsigned int i;
- uint32_t *pd;
if (type != IOMMU_DOMAIN_UNMANAGED)
return NULL;
as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;
- as->pd = alloc_page(GFP_KERNEL | __GFP_DMA);
+ as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
if (!as->pd) {
kfree(as);
return NULL;
}
- as->count = alloc_page(GFP_KERNEL);
+ as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);
if (!as->count) {
__free_page(as->pd);
kfree(as);
return NULL;
}
- /* clear PDEs */
- pd = page_address(as->pd);
- SetPageReserved(as->pd);
-
- for (i = 0; i < SMMU_NUM_PDE; i++)
- pd[i] = 0;
-
- /* clear PDE usage counters */
- pd = page_address(as->count);
- SetPageReserved(as->count);
-
- for (i = 0; i < SMMU_NUM_PDE; i++)
- pd[i] = 0;
+ as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);
+ if (!as->pts) {
+ kfree(as->count);
+ __free_page(as->pd);
+ kfree(as);
+ return NULL;
+ }
/* setup aperture */
as->domain.geometry.aperture_start = 0;
struct tegra_smmu_as *as = to_smmu_as(domain);
/* TODO: free page directory and page tables */
- ClearPageReserved(as->pd);
kfree(as);
}
return 0;
}
+ as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(smmu->dev, as->pd_dma))
+ return -ENOMEM;
+
+ /* We can't handle 64-bit DMA addresses */
+ if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {
+ err = -ENOMEM;
+ goto err_unmap;
+ }
+
err = tegra_smmu_alloc_asid(smmu, &as->id);
if (err < 0)
- return err;
+ goto err_unmap;
- smmu->soc->ops->flush_dcache(as->pd, 0, SMMU_SIZE_PD);
- smmu_flush_ptc(smmu, as->pd, 0);
+ smmu_flush_ptc(smmu, as->pd_dma, 0);
smmu_flush_tlb_asid(smmu, as->id);
smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);
- value = SMMU_PTB_DATA_VALUE(as->pd, as->attr);
+ value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);
smmu_writel(smmu, value, SMMU_PTB_DATA);
smmu_flush(smmu);
as->use_count++;
return 0;
+
+err_unmap:
+ dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
+ return err;
}
static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,
return;
tegra_smmu_free_asid(smmu, as->id);
+
+ dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);
+
as->smmu = NULL;
}
}
}
+static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova,
+ u32 value)
+{
+ unsigned int pd_index = iova_pd_index(iova);
+ struct tegra_smmu *smmu = as->smmu;
+ u32 *pd = page_address(as->pd);
+ unsigned long offset = pd_index * sizeof(*pd);
+
+ /* Set the page directory entry first */
+ pd[pd_index] = value;
+
+ /* The flush the page directory entry from caches */
+ dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,
+ sizeof(*pd), DMA_TO_DEVICE);
+
+ /* And flush the iommu */
+ smmu_flush_ptc(smmu, as->pd_dma, offset);
+ smmu_flush_tlb_section(smmu, as->id, iova);
+ smmu_flush(smmu);
+}
+
+static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova)
+{
+ u32 *pt = page_address(pt_page);
+
+ return pt + iova_pt_index(iova);
+}
+
+static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova,
+ dma_addr_t *dmap)
+{
+ unsigned int pd_index = iova_pd_index(iova);
+ struct page *pt_page;
+ u32 *pd;
+
+ pt_page = as->pts[pd_index];
+ if (!pt_page)
+ return NULL;
+
+ pd = page_address(as->pd);
+ *dmap = smmu_pde_to_dma(pd[pd_index]);
+
+ return tegra_smmu_pte_offset(pt_page, iova);
+}
+
static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova,
- struct page **pagep)
+ dma_addr_t *dmap)
{
- u32 *pd = page_address(as->pd), *pt, *count;
- u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff;
- u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff;
+ unsigned int pde = iova_pd_index(iova);
struct tegra_smmu *smmu = as->smmu;
- struct page *page;
- unsigned int i;
- if (pd[pde] == 0) {
- page = alloc_page(GFP_KERNEL | __GFP_DMA);
+ if (!as->pts[pde]) {
+ struct page *page;
+ dma_addr_t dma;
+
+ page = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);
if (!page)
return NULL;
- pt = page_address(page);
- SetPageReserved(page);
+ dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(smmu->dev, dma)) {
+ __free_page(page);
+ return NULL;
+ }
- for (i = 0; i < SMMU_NUM_PTE; i++)
- pt[i] = 0;
+ if (!smmu_dma_addr_valid(smmu, dma)) {
+ dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,
+ DMA_TO_DEVICE);
+ __free_page(page);
+ return NULL;
+ }
- smmu->soc->ops->flush_dcache(page, 0, SMMU_SIZE_PT);
+ as->pts[pde] = page;
- pd[pde] = SMMU_MK_PDE(page, SMMU_PDE_ATTR | SMMU_PDE_NEXT);
+ tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR |
+ SMMU_PDE_NEXT));
- smmu->soc->ops->flush_dcache(as->pd, pde << 2, 4);
- smmu_flush_ptc(smmu, as->pd, pde << 2);
- smmu_flush_tlb_section(smmu, as->id, iova);
- smmu_flush(smmu);
+ *dmap = dma;
} else {
- page = pfn_to_page(pd[pde] & smmu->pfn_mask);
- pt = page_address(page);
+ u32 *pd = page_address(as->pd);
+
+ *dmap = smmu_pde_to_dma(pd[pde]);
}
- *pagep = page;
+ return tegra_smmu_pte_offset(as->pts[pde], iova);
+}
- /* Keep track of entries in this page table. */
- count = page_address(as->count);
- if (pt[pte] == 0)
- count[pde]++;
+static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova)
+{
+ unsigned int pd_index = iova_pd_index(iova);
- return &pt[pte];
+ as->count[pd_index]++;
}
-static void as_put_pte(struct tegra_smmu_as *as, dma_addr_t iova)
+static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova)
{
- u32 pde = (iova >> SMMU_PDE_SHIFT) & 0x3ff;
- u32 pte = (iova >> SMMU_PTE_SHIFT) & 0x3ff;
- u32 *count = page_address(as->count);
- u32 *pd = page_address(as->pd), *pt;
- struct page *page;
-
- page = pfn_to_page(pd[pde] & as->smmu->pfn_mask);
- pt = page_address(page);
+ unsigned int pde = iova_pd_index(iova);
+ struct page *page = as->pts[pde];
/*
* When no entries in this page table are used anymore, return the
* memory page to the system.
*/
- if (pt[pte] != 0) {
- if (--count[pde] == 0) {
- ClearPageReserved(page);
- __free_page(page);
- pd[pde] = 0;
- }
+ if (--as->count[pde] == 0) {
+ struct tegra_smmu *smmu = as->smmu;
+ u32 *pd = page_address(as->pd);
+ dma_addr_t pte_dma = smmu_pde_to_dma(pd[pde]);
- pt[pte] = 0;
+ tegra_smmu_set_pde(as, iova, 0);
+
+ dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);
+ __free_page(page);
+ as->pts[pde] = NULL;
}
}
+static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
+ u32 *pte, dma_addr_t pte_dma, u32 val)
+{
+ struct tegra_smmu *smmu = as->smmu;
+ unsigned long offset = offset_in_page(pte);
+
+ *pte = val;
+
+ dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,
+ 4, DMA_TO_DEVICE);
+ smmu_flush_ptc(smmu, pte_dma, offset);
+ smmu_flush_tlb_group(smmu, as->id, iova);
+ smmu_flush(smmu);
+}
+
static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
phys_addr_t paddr, size_t size, int prot)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
- struct tegra_smmu *smmu = as->smmu;
- unsigned long offset;
- struct page *page;
+ dma_addr_t pte_dma;
u32 *pte;
- pte = as_get_pte(as, iova, &page);
+ pte = as_get_pte(as, iova, &pte_dma);
if (!pte)
return -ENOMEM;
- *pte = __phys_to_pfn(paddr) | SMMU_PTE_ATTR;
- offset = offset_in_page(pte);
+ /* If we aren't overwriting a pre-existing entry, increment use */
+ if (*pte == 0)
+ tegra_smmu_pte_get_use(as, iova);
- smmu->soc->ops->flush_dcache(page, offset, 4);
- smmu_flush_ptc(smmu, page, offset);
- smmu_flush_tlb_group(smmu, as->id, iova);
- smmu_flush(smmu);
+ tegra_smmu_set_pte(as, iova, pte, pte_dma,
+ __phys_to_pfn(paddr) | SMMU_PTE_ATTR);
return 0;
}
size_t size)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
- struct tegra_smmu *smmu = as->smmu;
- unsigned long offset;
- struct page *page;
+ dma_addr_t pte_dma;
u32 *pte;
- pte = as_get_pte(as, iova, &page);
- if (!pte)
+ pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
+ if (!pte || !*pte)
return 0;
- offset = offset_in_page(pte);
- as_put_pte(as, iova);
-
- smmu->soc->ops->flush_dcache(page, offset, 4);
- smmu_flush_ptc(smmu, page, offset);
- smmu_flush_tlb_group(smmu, as->id, iova);
- smmu_flush(smmu);
+ tegra_smmu_set_pte(as, iova, pte, pte_dma, 0);
+ tegra_smmu_pte_put_use(as, iova);
return size;
}
dma_addr_t iova)
{
struct tegra_smmu_as *as = to_smmu_as(domain);
- struct page *page;
unsigned long pfn;
+ dma_addr_t pte_dma;
u32 *pte;
- pte = as_get_pte(as, iova, &page);
+ pte = tegra_smmu_pte_lookup(as, iova, &pte_dma);
+ if (!pte || !*pte)
+ return 0;
+
pfn = *pte & as->smmu->pfn_mask;
return PFN_PHYS(pfn);
}
}
+static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)
+{
+ struct tegra_smmu *smmu = s->private;
+ unsigned int i;
+ u32 value;
+
+ seq_printf(s, "swgroup enabled ASID\n");
+ seq_printf(s, "------------------------\n");
+
+ for (i = 0; i < smmu->soc->num_swgroups; i++) {
+ const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];
+ const char *status;
+ unsigned int asid;
+
+ value = smmu_readl(smmu, group->reg);
+
+ if (value & SMMU_ASID_ENABLE)
+ status = "yes";
+ else
+ status = "no";
+
+ asid = value & SMMU_ASID_MASK;
+
+ seq_printf(s, "%-9s %-7s %#04x\n", group->name, status,
+ asid);
+ }
+
+ return 0;
+}
+
+static int tegra_smmu_swgroups_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, tegra_smmu_swgroups_show, inode->i_private);
+}
+
+static const struct file_operations tegra_smmu_swgroups_fops = {
+ .open = tegra_smmu_swgroups_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static int tegra_smmu_clients_show(struct seq_file *s, void *data)
+{
+ struct tegra_smmu *smmu = s->private;
+ unsigned int i;
+ u32 value;
+
+ seq_printf(s, "client enabled\n");
+ seq_printf(s, "--------------------\n");
+
+ for (i = 0; i < smmu->soc->num_clients; i++) {
+ const struct tegra_mc_client *client = &smmu->soc->clients[i];
+ const char *status;
+
+ value = smmu_readl(smmu, client->smmu.reg);
+
+ if (value & BIT(client->smmu.bit))
+ status = "yes";
+ else
+ status = "no";
+
+ seq_printf(s, "%-12s %s\n", client->name, status);
+ }
+
+ return 0;
+}
+
+static int tegra_smmu_clients_open(struct inode *inode, struct file *file)
+{
+ return single_open(file, tegra_smmu_clients_show, inode->i_private);
+}
+
+static const struct file_operations tegra_smmu_clients_fops = {
+ .open = tegra_smmu_clients_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+};
+
+static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)
+{
+ smmu->debugfs = debugfs_create_dir("smmu", NULL);
+ if (!smmu->debugfs)
+ return;
+
+ debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,
+ &tegra_smmu_swgroups_fops);
+ debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,
+ &tegra_smmu_clients_fops);
+}
+
+static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)
+{
+ debugfs_remove_recursive(smmu->debugfs);
+}
+
struct tegra_smmu *tegra_smmu_probe(struct device *dev,
const struct tegra_smmu_soc *soc,
struct tegra_mc *mc)
smmu_writel(smmu, value, SMMU_TLB_CONFIG);
- smmu_flush_ptc(smmu, NULL, 0);
+ smmu_flush_ptc_all(smmu);
smmu_flush_tlb(smmu);
smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);
smmu_flush(smmu);
if (err < 0)
return ERR_PTR(err);
+ if (IS_ENABLED(CONFIG_DEBUG_FS))
+ tegra_smmu_debugfs_init(smmu);
+
return smmu;
}
+
+void tegra_smmu_remove(struct tegra_smmu *smmu)
+{
+ if (IS_ENABLED(CONFIG_DEBUG_FS))
+ tegra_smmu_debugfs_exit(smmu);
+}
Code Review / kvmfornfv.git / blobdiff