kernel/arch/s390/mm/vmem.c

   1 /*
   2  *    Copyright IBM Corp. 2006
   3  *    Author(s): Heiko Carstens <heiko.carstens@de.ibm.com>
   4  */
   5
   6 #include <linux/bootmem.h>
   7 #include <linux/pfn.h>
   8 #include <linux/mm.h>
   9 #include <linux/module.h>
  10 #include <linux/list.h>
  11 #include <linux/hugetlb.h>
  12 #include <linux/slab.h>
  13 #include <linux/memblock.h>
  14 #include <asm/pgalloc.h>
  15 #include <asm/pgtable.h>
  16 #include <asm/setup.h>
  17 #include <asm/tlbflush.h>
  18 #include <asm/sections.h>
  19
  20 static DEFINE_MUTEX(vmem_mutex);
  21
  22 struct memory_segment {
  23         struct list_head list;
  24         unsigned long start;
  25         unsigned long size;
  26 };
  27
  28 static LIST_HEAD(mem_segs);
  29
  30 static void __ref *vmem_alloc_pages(unsigned int order)
  31 {
  32         if (slab_is_available())
  33                 return (void *)__get_free_pages(GFP_KERNEL, order);
  34         return alloc_bootmem_pages((1 << order) * PAGE_SIZE);
  35 }
  36
  37 static inline pud_t *vmem_pud_alloc(void)
  38 {
  39         pud_t *pud = NULL;
  40
  41         pud = vmem_alloc_pages(2);
  42         if (!pud)
  43                 return NULL;
  44         clear_table((unsigned long *) pud, _REGION3_ENTRY_EMPTY, PAGE_SIZE * 4);
  45         return pud;
  46 }
  47
  48 static inline pmd_t *vmem_pmd_alloc(void)
  49 {
  50         pmd_t *pmd = NULL;
  51
  52         pmd = vmem_alloc_pages(2);
  53         if (!pmd)
  54                 return NULL;
  55         clear_table((unsigned long *) pmd, _SEGMENT_ENTRY_EMPTY, PAGE_SIZE * 4);
  56         return pmd;
  57 }
  58
  59 static pte_t __ref *vmem_pte_alloc(unsigned long address)
  60 {
  61         pte_t *pte;
  62
  63         if (slab_is_available())
  64                 pte = (pte_t *) page_table_alloc(&init_mm);
  65         else
  66                 pte = alloc_bootmem_align(PTRS_PER_PTE * sizeof(pte_t),
  67                                           PTRS_PER_PTE * sizeof(pte_t));
  68         if (!pte)
  69                 return NULL;
  70         clear_table((unsigned long *) pte, _PAGE_INVALID,
  71                     PTRS_PER_PTE * sizeof(pte_t));
  72         return pte;
  73 }
  74
  75 /*
  76  * Add a physical memory range to the 1:1 mapping.
  77  */
  78 static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
  79 {
  80         unsigned long end = start + size;
  81         unsigned long address = start;
  82         pgd_t *pg_dir;
  83         pud_t *pu_dir;
  84         pmd_t *pm_dir;
  85         pte_t *pt_dir;
  86         int ret = -ENOMEM;
  87
  88         while (address < end) {
  89                 pg_dir = pgd_offset_k(address);
  90                 if (pgd_none(*pg_dir)) {
  91                         pu_dir = vmem_pud_alloc();
  92                         if (!pu_dir)
  93                                 goto out;
  94                         pgd_populate(&init_mm, pg_dir, pu_dir);
  95                 }
  96                 pu_dir = pud_offset(pg_dir, address);
  97 #ifndef CONFIG_DEBUG_PAGEALLOC
  98                 if (MACHINE_HAS_EDAT2 && pud_none(*pu_dir) && address &&
  99                     !(address & ~PUD_MASK) && (address + PUD_SIZE <= end)) {
 100                         pud_val(*pu_dir) = __pa(address) |
 101                                 _REGION_ENTRY_TYPE_R3 | _REGION3_ENTRY_LARGE |
 102                                 (ro ? _REGION_ENTRY_PROTECT : 0);
 103                         address += PUD_SIZE;
 104                         continue;
 105                 }
 106 #endif
 107                 if (pud_none(*pu_dir)) {
 108                         pm_dir = vmem_pmd_alloc();
 109                         if (!pm_dir)
 110                                 goto out;
 111                         pud_populate(&init_mm, pu_dir, pm_dir);
 112                 }
 113                 pm_dir = pmd_offset(pu_dir, address);
 114 #ifndef CONFIG_DEBUG_PAGEALLOC
 115                 if (MACHINE_HAS_EDAT1 && pmd_none(*pm_dir) && address &&
 116                     !(address & ~PMD_MASK) && (address + PMD_SIZE <= end)) {
 117                         pmd_val(*pm_dir) = __pa(address) |
 118                                 _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE |
 119                                 _SEGMENT_ENTRY_YOUNG |
 120                                 (ro ? _SEGMENT_ENTRY_PROTECT : 0);
 121                         address += PMD_SIZE;
 122                         continue;
 123                 }
 124 #endif
 125                 if (pmd_none(*pm_dir)) {
 126                         pt_dir = vmem_pte_alloc(address);
 127                         if (!pt_dir)
 128                                 goto out;
 129                         pmd_populate(&init_mm, pm_dir, pt_dir);
 130                 }
 131
 132                 pt_dir = pte_offset_kernel(pm_dir, address);
 133                 pte_val(*pt_dir) = __pa(address) |
 134                         pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL);
 135                 address += PAGE_SIZE;
 136         }
 137         ret = 0;
 138 out:
 139         return ret;
 140 }
 141
 142 /*
 143  * Remove a physical memory range from the 1:1 mapping.
 144  * Currently only invalidates page table entries.
 145  */
 146 static void vmem_remove_range(unsigned long start, unsigned long size)
 147 {
 148         unsigned long end = start + size;
 149         unsigned long address = start;
 150         pgd_t *pg_dir;
 151         pud_t *pu_dir;
 152         pmd_t *pm_dir;
 153         pte_t *pt_dir;
 154         pte_t  pte;
 155
 156         pte_val(pte) = _PAGE_INVALID;
 157         while (address < end) {
 158                 pg_dir = pgd_offset_k(address);
 159                 if (pgd_none(*pg_dir)) {
 160                         address += PGDIR_SIZE;
 161                         continue;
 162                 }
 163                 pu_dir = pud_offset(pg_dir, address);
 164                 if (pud_none(*pu_dir)) {
 165                         address += PUD_SIZE;
 166                         continue;
 167                 }
 168                 if (pud_large(*pu_dir)) {
 169                         pud_clear(pu_dir);
 170                         address += PUD_SIZE;
 171                         continue;
 172                 }
 173                 pm_dir = pmd_offset(pu_dir, address);
 174                 if (pmd_none(*pm_dir)) {
 175                         address += PMD_SIZE;
 176                         continue;
 177                 }
 178                 if (pmd_large(*pm_dir)) {
 179                         pmd_clear(pm_dir);
 180                         address += PMD_SIZE;
 181                         continue;
 182                 }
 183                 pt_dir = pte_offset_kernel(pm_dir, address);
 184                 *pt_dir = pte;
 185                 address += PAGE_SIZE;
 186         }
 187         flush_tlb_kernel_range(start, end);
 188 }
 189
 190 /*
 191  * Add a backed mem_map array to the virtual mem_map array.
 192  */
 193 int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
 194 {
 195         unsigned long address = start;
 196         pgd_t *pg_dir;
 197         pud_t *pu_dir;
 198         pmd_t *pm_dir;
 199         pte_t *pt_dir;
 200         int ret = -ENOMEM;
 201
 202         for (address = start; address < end;) {
 203                 pg_dir = pgd_offset_k(address);
 204                 if (pgd_none(*pg_dir)) {
 205                         pu_dir = vmem_pud_alloc();
 206                         if (!pu_dir)
 207                                 goto out;
 208                         pgd_populate(&init_mm, pg_dir, pu_dir);
 209                 }
 210
 211                 pu_dir = pud_offset(pg_dir, address);
 212                 if (pud_none(*pu_dir)) {
 213                         pm_dir = vmem_pmd_alloc();
 214                         if (!pm_dir)
 215                                 goto out;
 216                         pud_populate(&init_mm, pu_dir, pm_dir);
 217                 }
 218
 219                 pm_dir = pmd_offset(pu_dir, address);
 220                 if (pmd_none(*pm_dir)) {
 221                         /* Use 1MB frames for vmemmap if available. We always
 222                          * use large frames even if they are only partially
 223                          * used.
 224                          * Otherwise we would have also page tables since
 225                          * vmemmap_populate gets called for each section
 226                          * separately. */
 227                         if (MACHINE_HAS_EDAT1) {
 228                                 void *new_page;
 229
 230                                 new_page = vmemmap_alloc_block(PMD_SIZE, node);
 231                                 if (!new_page)
 232                                         goto out;
 233                                 pmd_val(*pm_dir) = __pa(new_page) |
 234                                         _SEGMENT_ENTRY | _SEGMENT_ENTRY_LARGE;
 235                                 address = (address + PMD_SIZE) & PMD_MASK;
 236                                 continue;
 237                         }
 238                         pt_dir = vmem_pte_alloc(address);
 239                         if (!pt_dir)
 240                                 goto out;
 241                         pmd_populate(&init_mm, pm_dir, pt_dir);
 242                 } else if (pmd_large(*pm_dir)) {
 243                         address = (address + PMD_SIZE) & PMD_MASK;
 244                         continue;
 245                 }
 246
 247                 pt_dir = pte_offset_kernel(pm_dir, address);
 248                 if (pte_none(*pt_dir)) {
 249                         void *new_page;
 250
 251                         new_page = vmemmap_alloc_block(PAGE_SIZE, node);
 252                         if (!new_page)
 253                                 goto out;
 254                         pte_val(*pt_dir) =
 255                                 __pa(new_page) | pgprot_val(PAGE_KERNEL);
 256                 }
 257                 address += PAGE_SIZE;
 258         }
 259         ret = 0;
 260 out:
 261         return ret;
 262 }
 263
 264 void vmemmap_free(unsigned long start, unsigned long end)
 265 {
 266 }
 267
 268 /*
 269  * Add memory segment to the segment list if it doesn't overlap with
 270  * an already present segment.
 271  */
 272 static int insert_memory_segment(struct memory_segment *seg)
 273 {
 274         struct memory_segment *tmp;
 275
 276         if (seg->start + seg->size > VMEM_MAX_PHYS ||
 277             seg->start + seg->size < seg->start)
 278                 return -ERANGE;
 279
 280         list_for_each_entry(tmp, &mem_segs, list) {
 281                 if (seg->start >= tmp->start + tmp->size)
 282                         continue;
 283                 if (seg->start + seg->size <= tmp->start)
 284                         continue;
 285                 return -ENOSPC;
 286         }
 287         list_add(&seg->list, &mem_segs);
 288         return 0;
 289 }
 290
 291 /*
 292  * Remove memory segment from the segment list.
 293  */
 294 static void remove_memory_segment(struct memory_segment *seg)
 295 {
 296         list_del(&seg->list);
 297 }
 298
 299 static void __remove_shared_memory(struct memory_segment *seg)
 300 {
 301         remove_memory_segment(seg);
 302         vmem_remove_range(seg->start, seg->size);
 303 }
 304
 305 int vmem_remove_mapping(unsigned long start, unsigned long size)
 306 {
 307         struct memory_segment *seg;
 308         int ret;
 309
 310         mutex_lock(&vmem_mutex);
 311
 312         ret = -ENOENT;
 313         list_for_each_entry(seg, &mem_segs, list) {
 314                 if (seg->start == start && seg->size == size)
 315                         break;
 316         }
 317
 318         if (seg->start != start || seg->size != size)
 319                 goto out;
 320
 321         ret = 0;
 322         __remove_shared_memory(seg);
 323         kfree(seg);
 324 out:
 325         mutex_unlock(&vmem_mutex);
 326         return ret;
 327 }
 328
 329 int vmem_add_mapping(unsigned long start, unsigned long size)
 330 {
 331         struct memory_segment *seg;
 332         int ret;
 333
 334         mutex_lock(&vmem_mutex);
 335         ret = -ENOMEM;
 336         seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 337         if (!seg)
 338                 goto out;
 339         seg->start = start;
 340         seg->size = size;
 341
 342         ret = insert_memory_segment(seg);
 343         if (ret)
 344                 goto out_free;
 345
 346         ret = vmem_add_mem(start, size, 0);
 347         if (ret)
 348                 goto out_remove;
 349         goto out;
 350
 351 out_remove:
 352         __remove_shared_memory(seg);
 353 out_free:
 354         kfree(seg);
 355 out:
 356         mutex_unlock(&vmem_mutex);
 357         return ret;
 358 }
 359
 360 /*
 361  * map whole physical memory to virtual memory (identity mapping)
 362  * we reserve enough space in the vmalloc area for vmemmap to hotplug
 363  * additional memory segments.
 364  */
 365 void __init vmem_map_init(void)
 366 {
 367         unsigned long ro_start, ro_end;
 368         struct memblock_region *reg;
 369         phys_addr_t start, end;
 370
 371         ro_start = PFN_ALIGN((unsigned long)&_stext);
 372         ro_end = (unsigned long)&_eshared & PAGE_MASK;
 373         for_each_memblock(memory, reg) {
 374                 start = reg->base;
 375                 end = reg->base + reg->size - 1;
 376                 if (start >= ro_end || end <= ro_start)
 377                         vmem_add_mem(start, end - start, 0);
 378                 else if (start >= ro_start && end <= ro_end)
 379                         vmem_add_mem(start, end - start, 1);
 380                 else if (start >= ro_start) {
 381                         vmem_add_mem(start, ro_end - start, 1);
 382                         vmem_add_mem(ro_end, end - ro_end, 0);
 383                 } else if (end < ro_end) {
 384                         vmem_add_mem(start, ro_start - start, 0);
 385                         vmem_add_mem(ro_start, end - ro_start, 1);
 386                 } else {
 387                         vmem_add_mem(start, ro_start - start, 0);
 388                         vmem_add_mem(ro_start, ro_end - ro_start, 1);
 389                         vmem_add_mem(ro_end, end - ro_end, 0);
 390                 }
 391         }
 392 }
 393
 394 /*
 395  * Convert memblock.memory  to a memory segment list so there is a single
 396  * list that contains all memory segments.
 397  */
 398 static int __init vmem_convert_memory_chunk(void)
 399 {
 400         struct memblock_region *reg;
 401         struct memory_segment *seg;
 402
 403         mutex_lock(&vmem_mutex);
 404         for_each_memblock(memory, reg) {
 405                 seg = kzalloc(sizeof(*seg), GFP_KERNEL);
 406                 if (!seg)
 407                         panic("Out of memory...\n");
 408                 seg->start = reg->base;
 409                 seg->size = reg->size;
 410                 insert_memory_segment(seg);
 411         }
 412         mutex_unlock(&vmem_mutex);
 413         return 0;
 414 }
 415
 416 core_initcall(vmem_convert_memory_chunk);