|
@@ -650,3 +650,155 @@ static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
|
|
|
|
|
|
if (addr)
|
|
|
*handle = pfn_to_dma(dev, page_to_pfn(page));
|
|
|
+
|
|
|
+ return addr;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Allocate DMA-coherent memory space and return both the kernel remapped
|
|
|
+ * virtual and bus address for that space.
|
|
|
+ */
|
|
|
+void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
|
|
|
+ gfp_t gfp, struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
|
|
|
+ void *memory;
|
|
|
+
|
|
|
+ if (dma_alloc_from_coherent(dev, size, handle, &memory))
|
|
|
+ return memory;
|
|
|
+
|
|
|
+ return __dma_alloc(dev, size, handle, gfp, prot, false,
|
|
|
+ __builtin_return_address(0));
|
|
|
+}
|
|
|
+
|
|
|
+static void *arm_coherent_dma_alloc(struct device *dev, size_t size,
|
|
|
+ dma_addr_t *handle, gfp_t gfp, struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ pgprot_t prot = __get_dma_pgprot(attrs, pgprot_kernel);
|
|
|
+ void *memory;
|
|
|
+
|
|
|
+ if (dma_alloc_from_coherent(dev, size, handle, &memory))
|
|
|
+ return memory;
|
|
|
+
|
|
|
+ return __dma_alloc(dev, size, handle, gfp, prot, true,
|
|
|
+ __builtin_return_address(0));
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Create userspace mapping for the DMA-coherent memory.
|
|
|
+ */
|
|
|
+int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma,
|
|
|
+ void *cpu_addr, dma_addr_t dma_addr, size_t size,
|
|
|
+ struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ int ret = -ENXIO;
|
|
|
+#ifdef CONFIG_MMU
|
|
|
+ unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
|
|
|
+ unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
|
|
|
+ unsigned long pfn = dma_to_pfn(dev, dma_addr);
|
|
|
+ unsigned long off = vma->vm_pgoff;
|
|
|
+
|
|
|
+ vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot);
|
|
|
+
|
|
|
+ if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
|
|
|
+ return ret;
|
|
|
+
|
|
|
+ if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) {
|
|
|
+ ret = remap_pfn_range(vma, vma->vm_start,
|
|
|
+ pfn + off,
|
|
|
+ vma->vm_end - vma->vm_start,
|
|
|
+ vma->vm_page_prot);
|
|
|
+ }
|
|
|
+#endif /* CONFIG_MMU */
|
|
|
+
|
|
|
+ return ret;
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Free a buffer as defined by the above mapping.
|
|
|
+ */
|
|
|
+static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
|
|
+ dma_addr_t handle, struct dma_attrs *attrs,
|
|
|
+ bool is_coherent)
|
|
|
+{
|
|
|
+ struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
|
|
|
+
|
|
|
+ if (dma_release_from_coherent(dev, get_order(size), cpu_addr))
|
|
|
+ return;
|
|
|
+
|
|
|
+ size = PAGE_ALIGN(size);
|
|
|
+
|
|
|
+ if (is_coherent || nommu()) {
|
|
|
+ __dma_free_buffer(page, size);
|
|
|
+ } else if (__free_from_pool(cpu_addr, size)) {
|
|
|
+ return;
|
|
|
+ } else if (!IS_ENABLED(CONFIG_CMA)) {
|
|
|
+ __dma_free_remap(cpu_addr, size);
|
|
|
+ __dma_free_buffer(page, size);
|
|
|
+ } else {
|
|
|
+ /*
|
|
|
+ * Non-atomic allocations cannot be freed with IRQs disabled
|
|
|
+ */
|
|
|
+ WARN_ON(irqs_disabled());
|
|
|
+ __free_from_contiguous(dev, page, size);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+void arm_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
|
|
+ dma_addr_t handle, struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ __arm_dma_free(dev, size, cpu_addr, handle, attrs, false);
|
|
|
+}
|
|
|
+
|
|
|
+static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr,
|
|
|
+ dma_addr_t handle, struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ __arm_dma_free(dev, size, cpu_addr, handle, attrs, true);
|
|
|
+}
|
|
|
+
|
|
|
+int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
|
|
|
+ void *cpu_addr, dma_addr_t handle, size_t size,
|
|
|
+ struct dma_attrs *attrs)
|
|
|
+{
|
|
|
+ struct page *page = pfn_to_page(dma_to_pfn(dev, handle));
|
|
|
+ int ret;
|
|
|
+
|
|
|
+ ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
|
|
|
+ if (unlikely(ret))
|
|
|
+ return ret;
|
|
|
+
|
|
|
+ sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
|
|
|
+ return 0;
|
|
|
+}
|
|
|
+
|
|
|
+static void dma_cache_maint_page(struct page *page, unsigned long offset,
|
|
|
+ size_t size, enum dma_data_direction dir,
|
|
|
+ void (*op)(const void *, size_t, int))
|
|
|
+{
|
|
|
+ unsigned long pfn;
|
|
|
+ size_t left = size;
|
|
|
+
|
|
|
+ pfn = page_to_pfn(page) + offset / PAGE_SIZE;
|
|
|
+ offset %= PAGE_SIZE;
|
|
|
+
|
|
|
+ /*
|
|
|
+ * A single sg entry may refer to multiple physically contiguous
|
|
|
+ * pages. But we still need to process highmem pages individually.
|
|
|
+ * If highmem is not configured then the bulk of this loop gets
|
|
|
+ * optimized out.
|
|
|
+ */
|
|
|
+ do {
|
|
|
+ size_t len = left;
|
|
|
+ void *vaddr;
|
|
|
+
|
|
|
+ page = pfn_to_page(pfn);
|
|
|
+
|
|
|
+ if (PageHighMem(page)) {
|
|
|
+ if (len + offset > PAGE_SIZE)
|
|
|
+ len = PAGE_SIZE - offset;
|
|
|
+ vaddr = kmap_high_get(page);
|
|
|
+ if (vaddr) {
|
|
|
+ vaddr += offset;
|
|
|
+ op(vaddr, len, dir);
|
|
|
+ kunmap_high(page);
|
|
|
+ } else if (cache_is_vipt()) {
|