|
@@ -277,3 +277,134 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot,
|
|
|
static void __dma_free_remap(void *cpu_addr, size_t size)
|
|
|
{
|
|
|
unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP;
|
|
|
+ struct vm_struct *area = find_vm_area(cpu_addr);
|
|
|
+ if (!area || (area->flags & flags) != flags) {
|
|
|
+ WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ unmap_kernel_range((unsigned long)cpu_addr, size);
|
|
|
+ vunmap(cpu_addr);
|
|
|
+}
|
|
|
+
|
|
|
+#define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K
|
|
|
+
|
|
|
+struct dma_pool {
|
|
|
+ size_t size;
|
|
|
+ spinlock_t lock;
|
|
|
+ unsigned long *bitmap;
|
|
|
+ unsigned long nr_pages;
|
|
|
+ void *vaddr;
|
|
|
+ struct page **pages;
|
|
|
+};
|
|
|
+
|
|
|
+static struct dma_pool atomic_pool = {
|
|
|
+ .size = DEFAULT_DMA_COHERENT_POOL_SIZE,
|
|
|
+};
|
|
|
+
|
|
|
+static int __init early_coherent_pool(char *p)
|
|
|
+{
|
|
|
+ atomic_pool.size = memparse(p, &p);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+early_param("coherent_pool", early_coherent_pool);
|
|
|
+
|
|
|
+void __init init_dma_coherent_pool_size(unsigned long size)
|
|
|
+{
|
|
|
+ /*
|
|
|
+ * Catch any attempt to set the pool size too late.
|
|
|
+ */
|
|
|
+ BUG_ON(atomic_pool.vaddr);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Set architecture specific coherent pool size only if
|
|
|
+ * it has not been changed by kernel command line parameter.
|
|
|
+ */
|
|
|
+ if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE)
|
|
|
+ atomic_pool.size = size;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Initialise the coherent pool for atomic allocations.
|
|
|
+ */
|
|
|
+static int __init atomic_pool_init(void)
|
|
|
+{
|
|
|
+ struct dma_pool *pool = &atomic_pool;
|
|
|
+ pgprot_t prot = pgprot_dmacoherent(pgprot_kernel);
|
|
|
+ gfp_t gfp = GFP_KERNEL | GFP_DMA;
|
|
|
+ unsigned long nr_pages = pool->size >> PAGE_SHIFT;
|
|
|
+ unsigned long *bitmap;
|
|
|
+ struct page *page;
|
|
|
+ struct page **pages;
|
|
|
+ void *ptr;
|
|
|
+ int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long);
|
|
|
+
|
|
|
+ bitmap = kzalloc(bitmap_size, GFP_KERNEL);
|
|
|
+ if (!bitmap)
|
|
|
+ goto no_bitmap;
|
|
|
+
|
|
|
+ pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL);
|
|
|
+ if (!pages)
|
|
|
+ goto no_pages;
|
|
|
+
|
|
|
+ if (IS_ENABLED(CONFIG_CMA))
|
|
|
+ ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page);
|
|
|
+ else
|
|
|
+ ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page,
|
|
|
+ NULL);
|
|
|
+ if (ptr) {
|
|
|
+ int i;
|
|
|
+
|
|
|
+ for (i = 0; i < nr_pages; i++)
|
|
|
+ pages[i] = page + i;
|
|
|
+
|
|
|
+ spin_lock_init(&pool->lock);
|
|
|
+ pool->vaddr = ptr;
|
|
|
+ pool->pages = pages;
|
|
|
+ pool->bitmap = bitmap;
|
|
|
+ pool->nr_pages = nr_pages;
|
|
|
+ pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n",
|
|
|
+ (unsigned)pool->size / 1024);
|
|
|
+ return 0;
|
|
|
+ }
|
|
|
+
|
|
|
+ kfree(pages);
|
|
|
+no_pages:
|
|
|
+ kfree(bitmap);
|
|
|
+no_bitmap:
|
|
|
+ pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n",
|
|
|
+ (unsigned)pool->size / 1024);
|
|
|
+ return -ENOMEM;
|
|
|
+}
|
|
|
+/*
|
|
|
+ * CMA is activated by core_initcall, so we must be called after it.
|
|
|
+ */
|
|
|
+postcore_initcall(atomic_pool_init);
|
|
|
+
|
|
|
+struct dma_contig_early_reserve {
|
|
|
+ phys_addr_t base;
|
|
|
+ unsigned long size;
|
|
|
+};
|
|
|
+
|
|
|
+static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata;
|
|
|
+
|
|
|
+static int dma_mmu_remap_num __initdata;
|
|
|
+
|
|
|
+void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size)
|
|
|
+{
|
|
|
+ dma_mmu_remap[dma_mmu_remap_num].base = base;
|
|
|
+ dma_mmu_remap[dma_mmu_remap_num].size = size;
|
|
|
+ dma_mmu_remap_num++;
|
|
|
+}
|
|
|
+
|
|
|
+void __init dma_contiguous_remap(void)
|
|
|
+{
|
|
|
+ int i;
|
|
|
+ for (i = 0; i < dma_mmu_remap_num; i++) {
|
|
|
+ phys_addr_t start = dma_mmu_remap[i].base;
|
|
|
+ phys_addr_t end = start + dma_mmu_remap[i].size;
|
|
|
+ struct map_desc map;
|
|
|
+ unsigned long addr;
|
|
|
+
|
|
|
+ if (end > arm_lowmem_limit)
|
|
|
+ end = arm_lowmem_limit;
|
|
|
+ if (start >= end)
|