kernel/arch/x86/mm/kasan_init_64.c

   1 #include <linux/bootmem.h>
   2 #include <linux/kasan.h>
   3 #include <linux/kdebug.h>
   4 #include <linux/mm.h>
   5 #include <linux/sched.h>
   6 #include <linux/vmalloc.h>
   7
   8 #include <asm/tlbflush.h>
   9 #include <asm/sections.h>
  10
  11 extern pgd_t early_level4_pgt[PTRS_PER_PGD];
  12 extern struct range pfn_mapped[E820_X_MAX];
  13
  14 extern unsigned char kasan_zero_page[PAGE_SIZE];
  15
  16 static int __init map_range(struct range *range)
  17 {
  18         unsigned long start;
  19         unsigned long end;
  20
  21         start = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->start));
  22         end = (unsigned long)kasan_mem_to_shadow(pfn_to_kaddr(range->end));
  23
  24         /*
  25          * end + 1 here is intentional. We check several shadow bytes in advance
  26          * to slightly speed up fastpath. In some rare cases we could cross
  27          * boundary of mapped shadow, so we just map some more here.
  28          */
  29         return vmemmap_populate(start, end + 1, NUMA_NO_NODE);
  30 }
  31
  32 static void __init clear_pgds(unsigned long start,
  33                         unsigned long end)
  34 {
  35         for (; start < end; start += PGDIR_SIZE)
  36                 pgd_clear(pgd_offset_k(start));
  37 }
  38
  39 void __init kasan_map_early_shadow(pgd_t *pgd)
  40 {
  41         int i;
  42         unsigned long start = KASAN_SHADOW_START;
  43         unsigned long end = KASAN_SHADOW_END;
  44
  45         for (i = pgd_index(start); start < end; i++) {
  46                 pgd[i] = __pgd(__pa_nodebug(kasan_zero_pud)
  47                                 | _KERNPG_TABLE);
  48                 start += PGDIR_SIZE;
  49         }
  50 }
  51
  52 static int __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
  53                                 unsigned long end)
  54 {
  55         pte_t *pte = pte_offset_kernel(pmd, addr);
  56
  57         while (addr + PAGE_SIZE <= end) {
  58                 WARN_ON(!pte_none(*pte));
  59                 set_pte(pte, __pte(__pa_nodebug(kasan_zero_page)
  60                                         | __PAGE_KERNEL_RO));
  61                 addr += PAGE_SIZE;
  62                 pte = pte_offset_kernel(pmd, addr);
  63         }
  64         return 0;
  65 }
  66
  67 static int __init zero_pmd_populate(pud_t *pud, unsigned long addr,
  68                                 unsigned long end)
  69 {
  70         int ret = 0;
  71         pmd_t *pmd = pmd_offset(pud, addr);
  72
  73         while (IS_ALIGNED(addr, PMD_SIZE) && addr + PMD_SIZE <= end) {
  74                 WARN_ON(!pmd_none(*pmd));
  75                 set_pmd(pmd, __pmd(__pa_nodebug(kasan_zero_pte)
  76                                         | __PAGE_KERNEL_RO));
  77                 addr += PMD_SIZE;
  78                 pmd = pmd_offset(pud, addr);
  79         }
  80         if (addr < end) {
  81                 if (pmd_none(*pmd)) {
  82                         void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE);
  83                         if (!p)
  84                                 return -ENOMEM;
  85                         set_pmd(pmd, __pmd(__pa_nodebug(p) | _KERNPG_TABLE));
  86                 }
  87                 ret = zero_pte_populate(pmd, addr, end);
  88         }
  89         return ret;
  90 }
  91
  92
  93 static int __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
  94                                 unsigned long end)
  95 {
  96         int ret = 0;
  97         pud_t *pud = pud_offset(pgd, addr);
  98
  99         while (IS_ALIGNED(addr, PUD_SIZE) && addr + PUD_SIZE <= end) {
 100                 WARN_ON(!pud_none(*pud));
 101                 set_pud(pud, __pud(__pa_nodebug(kasan_zero_pmd)
 102                                         | __PAGE_KERNEL_RO));
 103                 addr += PUD_SIZE;
 104                 pud = pud_offset(pgd, addr);
 105         }
 106
 107         if (addr < end) {
 108                 if (pud_none(*pud)) {
 109                         void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE);
 110                         if (!p)
 111                                 return -ENOMEM;
 112                         set_pud(pud, __pud(__pa_nodebug(p) | _KERNPG_TABLE));
 113                 }
 114                 ret = zero_pmd_populate(pud, addr, end);
 115         }
 116         return ret;
 117 }
 118
 119 static int __init zero_pgd_populate(unsigned long addr, unsigned long end)
 120 {
 121         int ret = 0;
 122         pgd_t *pgd = pgd_offset_k(addr);
 123
 124         while (IS_ALIGNED(addr, PGDIR_SIZE) && addr + PGDIR_SIZE <= end) {
 125                 WARN_ON(!pgd_none(*pgd));
 126                 set_pgd(pgd, __pgd(__pa_nodebug(kasan_zero_pud)
 127                                         | __PAGE_KERNEL_RO));
 128                 addr += PGDIR_SIZE;
 129                 pgd = pgd_offset_k(addr);
 130         }
 131
 132         if (addr < end) {
 133                 if (pgd_none(*pgd)) {
 134                         void *p = vmemmap_alloc_block(PAGE_SIZE, NUMA_NO_NODE);
 135                         if (!p)
 136                                 return -ENOMEM;
 137                         set_pgd(pgd, __pgd(__pa_nodebug(p) | _KERNPG_TABLE));
 138                 }
 139                 ret = zero_pud_populate(pgd, addr, end);
 140         }
 141         return ret;
 142 }
 143
 144
 145 static void __init populate_zero_shadow(const void *start, const void *end)
 146 {
 147         if (zero_pgd_populate((unsigned long)start, (unsigned long)end))
 148                 panic("kasan: unable to map zero shadow!");
 149 }
 150
 151
 152 #ifdef CONFIG_KASAN_INLINE
 153 static int kasan_die_handler(struct notifier_block *self,
 154                              unsigned long val,
 155                              void *data)
 156 {
 157         if (val == DIE_GPF) {
 158                 pr_emerg("CONFIG_KASAN_INLINE enabled");
 159                 pr_emerg("GPF could be caused by NULL-ptr deref or user memory access");
 160         }
 161         return NOTIFY_OK;
 162 }
 163
 164 static struct notifier_block kasan_die_notifier = {
 165         .notifier_call = kasan_die_handler,
 166 };
 167 #endif
 168
 169 void __init kasan_init(void)
 170 {
 171         int i;
 172
 173 #ifdef CONFIG_KASAN_INLINE
 174         register_die_notifier(&kasan_die_notifier);
 175 #endif
 176
 177         memcpy(early_level4_pgt, init_level4_pgt, sizeof(early_level4_pgt));
 178         load_cr3(early_level4_pgt);
 179
 180         clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
 181
 182         populate_zero_shadow((void *)KASAN_SHADOW_START,
 183                         kasan_mem_to_shadow((void *)PAGE_OFFSET));
 184
 185         for (i = 0; i < E820_X_MAX; i++) {
 186                 if (pfn_mapped[i].end == 0)
 187                         break;
 188
 189                 if (map_range(&pfn_mapped[i]))
 190                         panic("kasan: unable to allocate shadow!");
 191         }
 192         populate_zero_shadow(kasan_mem_to_shadow((void *)PAGE_OFFSET + MAXMEM),
 193                         kasan_mem_to_shadow((void *)__START_KERNEL_map));
 194
 195         vmemmap_populate((unsigned long)kasan_mem_to_shadow(_stext),
 196                         (unsigned long)kasan_mem_to_shadow(_end),
 197                         NUMA_NO_NODE);
 198
 199         populate_zero_shadow(kasan_mem_to_shadow((void *)MODULES_END),
 200                         (void *)KASAN_SHADOW_END);
 201
 202         memset(kasan_zero_page, 0, PAGE_SIZE);
 203
 204         load_cr3(init_level4_pgt);
 205         init_task.kasan_depth = 0;
 206 }