| 
					
				 | 
			
			
				@@ -277,3 +277,134 @@ __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 static void __dma_free_remap(void *cpu_addr, size_t size) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				 	unsigned int flags = VM_ARM_DMA_CONSISTENT | VM_USERMAP; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct vm_struct *area = find_vm_area(cpu_addr); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!area || (area->flags & flags) != flags) { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unmap_kernel_range((unsigned long)cpu_addr, size); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	vunmap(cpu_addr); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+#define DEFAULT_DMA_COHERENT_POOL_SIZE	SZ_256K 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+struct dma_pool { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	size_t size; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	spinlock_t lock; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unsigned long *bitmap; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unsigned long nr_pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	void *vaddr; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct page **pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+}; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static struct dma_pool atomic_pool = { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	.size = DEFAULT_DMA_COHERENT_POOL_SIZE, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+}; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static int __init early_coherent_pool(char *p) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	atomic_pool.size = memparse(p, &p); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return 0; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+early_param("coherent_pool", early_coherent_pool); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+void __init init_dma_coherent_pool_size(unsigned long size) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * Catch any attempt to set the pool size too late. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	BUG_ON(atomic_pool.vaddr); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * Set architecture specific coherent pool size only if 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 * it has not been changed by kernel command line parameter. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	 */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (atomic_pool.size == DEFAULT_DMA_COHERENT_POOL_SIZE) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		atomic_pool.size = size; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * Initialise the coherent pool for atomic allocations. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static int __init atomic_pool_init(void) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct dma_pool *pool = &atomic_pool; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	pgprot_t prot = pgprot_dmacoherent(pgprot_kernel); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	gfp_t gfp = GFP_KERNEL | GFP_DMA; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unsigned long nr_pages = pool->size >> PAGE_SHIFT; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unsigned long *bitmap; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct page *page; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	struct page **pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	void *ptr; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	int bitmap_size = BITS_TO_LONGS(nr_pages) * sizeof(long); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	bitmap = kzalloc(bitmap_size, GFP_KERNEL); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!bitmap) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		goto no_bitmap; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	pages = kzalloc(nr_pages * sizeof(struct page *), GFP_KERNEL); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (!pages) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		goto no_pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (IS_ENABLED(CONFIG_CMA)) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		ptr = __alloc_from_contiguous(NULL, pool->size, prot, &page); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	else 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		ptr = __alloc_remap_buffer(NULL, pool->size, gfp, prot, &page, 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+					   NULL); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	if (ptr) { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		int i; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		for (i = 0; i < nr_pages; i++) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+			pages[i] = page + i; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		spin_lock_init(&pool->lock); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		pool->vaddr = ptr; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		pool->pages = pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		pool->bitmap = bitmap; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		pool->nr_pages = nr_pages; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		pr_info("DMA: preallocated %u KiB pool for atomic coherent allocations\n", 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		       (unsigned)pool->size / 1024); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		return 0; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	kfree(pages); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+no_pages: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	kfree(bitmap); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+no_bitmap: 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	pr_err("DMA: failed to allocate %u KiB pool for atomic coherent allocation\n", 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	       (unsigned)pool->size / 1024); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	return -ENOMEM; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+/* 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ * CMA is activated by core_initcall, so we must be called after it. 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ */ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+postcore_initcall(atomic_pool_init); 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+struct dma_contig_early_reserve { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	phys_addr_t base; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	unsigned long size; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+}; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+static int dma_mmu_remap_num __initdata; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	dma_mmu_remap[dma_mmu_remap_num].base = base; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	dma_mmu_remap[dma_mmu_remap_num].size = size; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	dma_mmu_remap_num++; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+} 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+void __init dma_contiguous_remap(void) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+{ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	int i; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+	for (i = 0; i < dma_mmu_remap_num; i++) { 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		phys_addr_t start = dma_mmu_remap[i].base; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		phys_addr_t end = start + dma_mmu_remap[i].size; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		struct map_desc map; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		unsigned long addr; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+ 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		if (end > arm_lowmem_limit) 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+			end = arm_lowmem_limit; 
			 | 
		
	
		
			
				 | 
				 | 
			
			
				+		if (start >= end) 
			 |