ion_system_heap.c revision 8898227ed555b477e2989a2a9b984fa37e7a9b42
1/* 2 * drivers/staging/android/ion/ion_system_heap.c 3 * 4 * Copyright (C) 2011 Google, Inc. 5 * 6 * This software is licensed under the terms of the GNU General Public 7 * License version 2, as published by the Free Software Foundation, and 8 * may be copied, distributed, and modified under those terms. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 */ 16 17#include <asm/page.h> 18#include <linux/dma-mapping.h> 19#include <linux/err.h> 20#include <linux/highmem.h> 21#include <linux/mm.h> 22#include <linux/scatterlist.h> 23#include <linux/seq_file.h> 24#include <linux/slab.h> 25#include <linux/vmalloc.h> 26#include "ion.h" 27#include "ion_priv.h" 28 29static unsigned int high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | 30 __GFP_NOWARN | __GFP_NORETRY | 31 __GFP_NO_KSWAPD) & ~__GFP_WAIT; 32static unsigned int low_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | 33 __GFP_NOWARN); 34static const unsigned int orders[] = {8, 4, 0}; 35static const int num_orders = ARRAY_SIZE(orders); 36static int order_to_index(unsigned int order) 37{ 38 int i; 39 for (i = 0; i < num_orders; i++) 40 if (order == orders[i]) 41 return i; 42 BUG(); 43 return -1; 44} 45 46static unsigned int order_to_size(int order) 47{ 48 return PAGE_SIZE << order; 49} 50 51struct ion_system_heap { 52 struct ion_heap heap; 53 struct ion_page_pool **pools; 54}; 55 56struct page_info { 57 struct page *page; 58 unsigned int order; 59 struct list_head list; 60}; 61 62static struct page *alloc_buffer_page(struct ion_system_heap *heap, 63 struct ion_buffer *buffer, 64 unsigned long order) 65{ 66 bool cached = ion_buffer_cached(buffer); 67 bool split_pages = ion_buffer_fault_user_mappings(buffer); 68 struct ion_page_pool *pool = heap->pools[order_to_index(order)]; 69 struct page *page; 70 71 if (!cached) { 72 page = ion_page_pool_alloc(pool); 73 } else { 74 gfp_t gfp_flags = low_order_gfp_flags; 75 76 if (order > 4) 77 gfp_flags = high_order_gfp_flags; 78 page = alloc_pages(gfp_flags, order); 79 if (!page) 80 return 0; 81 __dma_page_cpu_to_dev(page, 0, PAGE_SIZE << order, 82 DMA_BIDIRECTIONAL); 83 } 84 if (!page) 85 return 0; 86 87 if (split_pages) 88 split_page(page, order); 89 return page; 90} 91 92static void free_buffer_page(struct ion_system_heap *heap, 93 struct ion_buffer *buffer, struct page *page, 94 unsigned int order) 95{ 96 bool cached = ion_buffer_cached(buffer); 97 bool split_pages = ion_buffer_fault_user_mappings(buffer); 98 int i; 99 100 if (!cached) { 101 struct ion_page_pool *pool = heap->pools[order_to_index(order)]; 102 /* zero the pages before returning them to the pool for 103 security. This uses vmap as we want to set the pgprot so 104 the writes to occur to noncached mappings, as the pool's 105 purpose is to keep the pages out of the cache */ 106 for (i = 0; i < (1 << order); i++) { 107 struct page *sub_page = page + i; 108 void *addr = vmap(&sub_page, 1, VM_MAP, 109 pgprot_writecombine(PAGE_KERNEL)); 110 memset(addr, 0, PAGE_SIZE); 111 vunmap(addr); 112 } 113 ion_page_pool_free(pool, page); 114 } else if (split_pages) { 115 for (i = 0; i < (1 << order); i++) 116 __free_page(page + i); 117 } else { 118 __free_pages(page, order); 119 } 120} 121 122 123static struct page_info *alloc_largest_available(struct ion_system_heap *heap, 124 struct ion_buffer *buffer, 125 unsigned long size, 126 unsigned int max_order) 127{ 128 struct page *page; 129 struct page_info *info; 130 int i; 131 132 for (i = 0; i < num_orders; i++) { 133 if (size < order_to_size(orders[i])) 134 continue; 135 if (max_order < orders[i]) 136 continue; 137 138 page = alloc_buffer_page(heap, buffer, orders[i]); 139 if (!page) 140 continue; 141 142 info = kmalloc(sizeof(struct page_info), GFP_KERNEL); 143 info->page = page; 144 info->order = orders[i]; 145 return info; 146 } 147 return NULL; 148} 149 150static int ion_system_heap_allocate(struct ion_heap *heap, 151 struct ion_buffer *buffer, 152 unsigned long size, unsigned long align, 153 unsigned long flags) 154{ 155 struct ion_system_heap *sys_heap = container_of(heap, 156 struct ion_system_heap, 157 heap); 158 struct sg_table *table; 159 struct scatterlist *sg; 160 int ret; 161 struct list_head pages; 162 struct page_info *info, *tmp_info; 163 int i = 0; 164 long size_remaining = PAGE_ALIGN(size); 165 unsigned int max_order = orders[0]; 166 bool split_pages = ion_buffer_fault_user_mappings(buffer); 167 168 INIT_LIST_HEAD(&pages); 169 while (size_remaining > 0) { 170 info = alloc_largest_available(sys_heap, buffer, size_remaining, max_order); 171 if (!info) 172 goto err; 173 list_add_tail(&info->list, &pages); 174 size_remaining -= (1 << info->order) * PAGE_SIZE; 175 max_order = info->order; 176 i++; 177 } 178 179 table = kmalloc(sizeof(struct sg_table), GFP_KERNEL); 180 if (!table) 181 goto err; 182 183 if (split_pages) 184 ret = sg_alloc_table(table, PAGE_ALIGN(size) / PAGE_SIZE, 185 GFP_KERNEL); 186 else 187 ret = sg_alloc_table(table, i, GFP_KERNEL); 188 189 if (ret) 190 goto err1; 191 192 sg = table->sgl; 193 list_for_each_entry_safe(info, tmp_info, &pages, list) { 194 struct page *page = info->page; 195 if (split_pages) { 196 for (i = 0; i < (1 << info->order); i++) { 197 sg_set_page(sg, page + i, PAGE_SIZE, 0); 198 sg = sg_next(sg); 199 } 200 } else { 201 sg_set_page(sg, page, (1 << info->order) * PAGE_SIZE, 202 0); 203 sg = sg_next(sg); 204 } 205 list_del(&info->list); 206 kfree(info); 207 } 208 209 buffer->priv_virt = table; 210 return 0; 211err1: 212 kfree(table); 213err: 214 list_for_each_entry(info, &pages, list) { 215 free_buffer_page(sys_heap, buffer, info->page, info->order); 216 kfree(info); 217 } 218 return -ENOMEM; 219} 220 221void ion_system_heap_free(struct ion_buffer *buffer) 222{ 223 struct ion_heap *heap = buffer->heap; 224 struct ion_system_heap *sys_heap = container_of(heap, 225 struct ion_system_heap, 226 heap); 227 struct sg_table *table = buffer->sg_table; 228 struct scatterlist *sg; 229 LIST_HEAD(pages); 230 int i; 231 232 for_each_sg(table->sgl, sg, table->nents, i) 233 free_buffer_page(sys_heap, buffer, sg_page(sg), get_order(sg_dma_len(sg))); 234 sg_free_table(table); 235 kfree(table); 236} 237 238struct sg_table *ion_system_heap_map_dma(struct ion_heap *heap, 239 struct ion_buffer *buffer) 240{ 241 return buffer->priv_virt; 242} 243 244void ion_system_heap_unmap_dma(struct ion_heap *heap, 245 struct ion_buffer *buffer) 246{ 247 return; 248} 249 250static struct ion_heap_ops system_heap_ops = { 251 .allocate = ion_system_heap_allocate, 252 .free = ion_system_heap_free, 253 .map_dma = ion_system_heap_map_dma, 254 .unmap_dma = ion_system_heap_unmap_dma, 255 .map_kernel = ion_heap_map_kernel, 256 .unmap_kernel = ion_heap_unmap_kernel, 257 .map_user = ion_heap_map_user, 258}; 259 260static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s, 261 void *unused) 262{ 263 264 struct ion_system_heap *sys_heap = container_of(heap, 265 struct ion_system_heap, 266 heap); 267 int i; 268 for (i = 0; i < num_orders; i++) { 269 struct ion_page_pool *pool = sys_heap->pools[i]; 270 seq_printf(s, "%d order %u highmem pages in pool = %lu total\n", 271 pool->high_count, pool->order, 272 (1 << pool->order) * PAGE_SIZE * pool->high_count); 273 seq_printf(s, "%d order %u lowmem pages in pool = %lu total\n", 274 pool->low_count, pool->order, 275 (1 << pool->order) * PAGE_SIZE * pool->low_count); 276 } 277 return 0; 278} 279 280struct ion_heap *ion_system_heap_create(struct ion_platform_heap *unused) 281{ 282 struct ion_system_heap *heap; 283 int i; 284 285 heap = kzalloc(sizeof(struct ion_system_heap), GFP_KERNEL); 286 if (!heap) 287 return ERR_PTR(-ENOMEM); 288 heap->heap.ops = &system_heap_ops; 289 heap->heap.type = ION_HEAP_TYPE_SYSTEM; 290 heap->pools = kzalloc(sizeof(struct ion_page_pool *) * num_orders, 291 GFP_KERNEL); 292 if (!heap->pools) 293 goto err_alloc_pools; 294 for (i = 0; i < num_orders; i++) { 295 struct ion_page_pool *pool; 296 gfp_t gfp_flags = low_order_gfp_flags; 297 298 if (orders[i] > 4) 299 gfp_flags = high_order_gfp_flags; 300 pool = ion_page_pool_create(gfp_flags, orders[i]); 301 if (!pool) 302 goto err_create_pool; 303 heap->pools[i] = pool; 304 } 305 heap->heap.debug_show = ion_system_heap_debug_show; 306 return &heap->heap; 307err_create_pool: 308 for (i = 0; i < num_orders; i++) 309 if (heap->pools[i]) 310 ion_page_pool_destroy(heap->pools[i]); 311 kfree(heap->pools); 312err_alloc_pools: 313 kfree(heap); 314 return ERR_PTR(-ENOMEM); 315} 316 317void ion_system_heap_destroy(struct ion_heap *heap) 318{ 319 struct ion_system_heap *sys_heap = container_of(heap, 320 struct ion_system_heap, 321 heap); 322 int i; 323 324 for (i = 0; i < num_orders; i++) 325 ion_page_pool_destroy(sys_heap->pools[i]); 326 kfree(sys_heap->pools); 327 kfree(sys_heap); 328} 329 330static int ion_system_contig_heap_allocate(struct ion_heap *heap, 331 struct ion_buffer *buffer, 332 unsigned long len, 333 unsigned long align, 334 unsigned long flags) 335{ 336 buffer->priv_virt = kzalloc(len, GFP_KERNEL); 337 if (!buffer->priv_virt) 338 return -ENOMEM; 339 return 0; 340} 341 342void ion_system_contig_heap_free(struct ion_buffer *buffer) 343{ 344 kfree(buffer->priv_virt); 345} 346 347static int ion_system_contig_heap_phys(struct ion_heap *heap, 348 struct ion_buffer *buffer, 349 ion_phys_addr_t *addr, size_t *len) 350{ 351 *addr = virt_to_phys(buffer->priv_virt); 352 *len = buffer->size; 353 return 0; 354} 355 356struct sg_table *ion_system_contig_heap_map_dma(struct ion_heap *heap, 357 struct ion_buffer *buffer) 358{ 359 struct sg_table *table; 360 int ret; 361 362 table = kzalloc(sizeof(struct sg_table), GFP_KERNEL); 363 if (!table) 364 return ERR_PTR(-ENOMEM); 365 ret = sg_alloc_table(table, 1, GFP_KERNEL); 366 if (ret) { 367 kfree(table); 368 return ERR_PTR(ret); 369 } 370 sg_set_page(table->sgl, virt_to_page(buffer->priv_virt), buffer->size, 371 0); 372 return table; 373} 374 375void ion_system_contig_heap_unmap_dma(struct ion_heap *heap, 376 struct ion_buffer *buffer) 377{ 378 sg_free_table(buffer->sg_table); 379 kfree(buffer->sg_table); 380} 381 382int ion_system_contig_heap_map_user(struct ion_heap *heap, 383 struct ion_buffer *buffer, 384 struct vm_area_struct *vma) 385{ 386 unsigned long pfn = __phys_to_pfn(virt_to_phys(buffer->priv_virt)); 387 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, 388 vma->vm_end - vma->vm_start, 389 vma->vm_page_prot); 390 391} 392 393static struct ion_heap_ops kmalloc_ops = { 394 .allocate = ion_system_contig_heap_allocate, 395 .free = ion_system_contig_heap_free, 396 .phys = ion_system_contig_heap_phys, 397 .map_dma = ion_system_contig_heap_map_dma, 398 .unmap_dma = ion_system_contig_heap_unmap_dma, 399 .map_kernel = ion_heap_map_kernel, 400 .unmap_kernel = ion_heap_unmap_kernel, 401 .map_user = ion_system_contig_heap_map_user, 402}; 403 404struct ion_heap *ion_system_contig_heap_create(struct ion_platform_heap *unused) 405{ 406 struct ion_heap *heap; 407 408 heap = kzalloc(sizeof(struct ion_heap), GFP_KERNEL); 409 if (!heap) 410 return ERR_PTR(-ENOMEM); 411 heap->ops = &kmalloc_ops; 412 heap->type = ION_HEAP_TYPE_SYSTEM_CONTIG; 413 return heap; 414} 415 416void ion_system_contig_heap_destroy(struct ion_heap *heap) 417{ 418 kfree(heap); 419} 420 421