ion: Rewrite to improve clarity and performance

The ION driver suffers from massive code bloat caused by excessive
debug features, as well as poor lock usage as a result of that. Multiple
locks in ION exist to make the debug features thread-safe, which hurts
ION's actual performance when doing its job.

There are numerous code paths in ION that hold mutexes for no reason and
hold them for longer than necessary. This results in not only unwanted
lock contention, but also long delays when a mutex lock results in the
calling thread getting preempted for a while. All lock usage in ION
follows this pattern, which causes poor performance across the board.
Furthermore, a big mutex lock is used mostly everywhere, which causes
performance degradation due to unnecessary lock overhead.

Instead of having a big mutex lock, multiple fine-grained locks are now
used, improving performance.

Additionally, ion_dup_sg_table is called very frequently, and lies
within the rendering path for the display. Speed it up by copying
scatterlists in page-sized chunks rather than iterating one at a time.
Note that sg_alloc_table zeroes out `table`, so there's no need to zero
it out using the memory allocator.

Overall, just rewrite ION entirely to fix its deficiencies. This
optimizes ION for excellent performance and discards its debug cruft.

Signed-off-by: Sultan Alsawaf <sultan@kerneltoast.com>
Signed-off-by: Zile995 <stefan.zivkovic995@gmail.com>

Author: kerneltoast

drivers/staging/android/ion/ion.c

@@ -19,6 +19,8 @@
 #define _LINUX_ION_H
 
 #include <linux/err.h>
+#include <linux/memblock.h>
+#include <linux/msm_dma_iommu_mapping.h>
 #include "../uapi/ion.h"
 
 struct ion_handle;
@@ -28,6 +30,21 @@ struct ion_mapper;
 struct ion_client;
 struct ion_buffer;
 
+struct ion_buffer {
+	struct ion_heap *heap;
+	struct sg_table *sg_table;
+	struct mutex kmap_lock;
+	struct work_struct free;
+	atomic_t refcount;
+	void *priv_virt;
+	void *vaddr;
+	unsigned int flags;
+	unsigned int private_flags;
+	size_t size;
+	int kmap_refcount;
+	struct msm_iommu_data iommu_data;
+};
+
 /* This should be removed some day when phys_addr_t's are fully
    plumbed in the kernel, and all instances of ion_phys_addr_t should
    be converted to phys_addr_t.  For the time being many kernel interfaces
@@ -88,15 +105,31 @@ struct ion_platform_data {
  * located at specific memory addresses or of specific sizes not
  * managed by the kernel
  */
-void ion_reserve(struct ion_platform_data *data);
+static inline void ion_reserve(struct ion_platform_data *data)
+{
+	int i;
+
+	for (i = 0; i < data->nr; i++) {
+		struct ion_platform_heap *pheap = &data->heaps[i];
+
+		if (pheap->size) {
+			if (pheap->base)
+				memblock_reserve(pheap->base, pheap->size);
+			else
+				pheap->base = memblock_alloc_base(pheap->size,
+					pheap->align, MEMBLOCK_ALLOC_ANYWHERE);
+		}
+	}
+}
 
 /**
  * ion_client_create() -  allocate a client and returns it
  * @dev:		the global ion device
- * @name:		used for debugging
  */
-struct ion_client *ion_client_create(struct ion_device *dev,
-				     const char *name);
+static inline struct ion_client *ion_client_create(void *idev)
+{
+	return idev;
+}
 
 /**
  * ion_client_destroy() -  free's a client and all it's handles
@@ -105,7 +138,9 @@ struct ion_client *ion_client_create(struct ion_device *dev,
  * Free the provided client and all it's resources including
  * any handles it is holding.
  */
-void ion_client_destroy(struct ion_client *client);
+static inline void ion_client_destroy(struct ion_client *client)
+{
+}
 
 /**
  * ion_alloc - allocate ion memory
@@ -123,9 +158,14 @@ void ion_client_destroy(struct ion_client *client);
  * Allocate memory in one of the heaps provided in heap mask and return
  * an opaque handle to it.
  */
-struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
-			     size_t align, unsigned int heap_id_mask,
-			     unsigned int flags);
+struct ion_buffer *__ion_alloc(struct ion_device *idev, size_t len,
+			       size_t align, unsigned int heap_id_mask,
+			       unsigned int flags);
+static inline void *ion_alloc(void *client, size_t len, size_t align,
+			      unsigned int heap_id_mask, unsigned int flags)
+{
+	return __ion_alloc(client, len, align, heap_id_mask, flags);
+}
 
 /**
  * ion_free - free a handle
@@ -134,7 +174,11 @@ struct ion_handle *ion_alloc(struct ion_client *client, size_t len,
  *
  * Free the provided handle.
  */
-void ion_free(struct ion_client *client, struct ion_handle *handle);
+void ion_buffer_put(struct ion_buffer *buffer);
+static inline void ion_free(struct ion_client *client, void *handle)
+{
+	ion_buffer_put(handle);
+}
 
 /**
  * ion_phys - returns the physical address and len of a handle
@@ -152,19 +196,12 @@ void ion_free(struct ion_client *client, struct ion_handle *handle);
  * the returned value may not be valid if the caller is not
  * holding a reference.
  */
-int ion_phys(struct ion_client *client, struct ion_handle *handle,
-	     ion_phys_addr_t *addr, size_t *len);
-
-/**
- * ion_map_dma - return an sg_table describing a handle
- * @client:	the client
- * @handle:	the handle
- *
- * This function returns the sg_table describing
- * a particular ion handle.
- */
-struct sg_table *ion_sg_table(struct ion_client *client,
-			      struct ion_handle *handle);
+int __ion_phys(struct ion_buffer *buffer, ion_phys_addr_t *addr, size_t *len);
+static inline int ion_phys(struct ion_client *client, void *handle,
+			   ion_phys_addr_t *addr, size_t *len)
+{
+	return __ion_phys(handle, addr, len);
+}
 
 /**
  * ion_map_kernel - create mapping for the given handle
@@ -174,29 +211,45 @@ struct sg_table *ion_sg_table(struct ion_client *client,
  * Map the given handle into the kernel and return a kernel address that
  * can be used to access this address.
  */
-void *ion_map_kernel(struct ion_client *client, struct ion_handle *handle);
+void *__ion_map_kernel(struct ion_buffer *buffer);
+static inline void *ion_map_kernel(struct ion_client *client, void *handle)
+{
+	return __ion_map_kernel(handle);
+}
 
 /**
  * ion_unmap_kernel() - destroy a kernel mapping for a handle
  * @client:	the client
  * @handle:	handle to unmap
  */
-void ion_unmap_kernel(struct ion_client *client, struct ion_handle *handle);
+void __ion_unmap_kernel(struct ion_buffer *buffer);
+static inline void ion_unmap_kernel(struct ion_client *client, void *handle)
+{
+	__ion_unmap_kernel(handle);
+}
 
 /**
  * ion_share_dma_buf() - share buffer as dma-buf
  * @client:	the client
  * @handle:	the handle
  */
-struct dma_buf *ion_share_dma_buf(struct ion_client *client,
-						struct ion_handle *handle);
+struct dma_buf *__ion_share_dma_buf(struct ion_buffer *buffer);
+static inline struct dma_buf *ion_share_dma_buf(struct ion_client *client,
+						void *handle)
+{
+	return __ion_share_dma_buf(handle);
+}
 
 /**
  * ion_share_dma_buf_fd() - given an ion client, create a dma-buf fd
  * @client:	the client
  * @handle:	the handle
  */
-int ion_share_dma_buf_fd(struct ion_client *client, struct ion_handle *handle);
+int __ion_share_dma_buf_fd(struct ion_buffer *buffer);
+static inline int ion_share_dma_buf_fd(struct ion_client *client, void *handle)
+{
+	return __ion_share_dma_buf_fd(handle);
+}
 
 /**
  * ion_import_dma_buf() - given an dma-buf fd from the ion exporter get handle
@@ -207,7 +260,11 @@ int ion_share_dma_buf_fd(struct ion_client *client, struct ion_handle *handle);
  * import that fd and return a handle representing it.  If a dma-buf from
  * another exporter is passed in this function will return ERR_PTR(-EINVAL)
  */
-struct ion_handle *ion_import_dma_buf(struct ion_client *client, int fd);
+struct ion_buffer *__ion_import_dma_buf(int fd);
+static inline void *ion_import_dma_buf(struct ion_client *client, int fd)
+{
+	return __ion_import_dma_buf(fd);
+}
 
 #else
 static inline void ion_reserve(struct ion_platform_data *data)

drivers/staging/android/ion/ion.h

@@ -30,7 +30,7 @@ struct ion_chunk_heap {
 	ion_phys_addr_t base;
 	unsigned long chunk_size;
 	unsigned long size;
-	unsigned long allocated;
+	atomic_long_t allocated;
 };
 
 static int ion_chunk_heap_allocate(struct ion_heap *heap,
@@ -52,7 +52,8 @@ static int ion_chunk_heap_allocate(struct ion_heap *heap,
 	allocated_size = ALIGN(size, chunk_heap->chunk_size);
 	num_chunks = allocated_size / chunk_heap->chunk_size;
 
-	if (allocated_size > chunk_heap->size - chunk_heap->allocated)
+	if (allocated_size >
+	    chunk_heap->size - atomic_long_read(&chunk_heap->allocated))
 		return -ENOMEM;
 
 	table = kmalloc(sizeof(struct sg_table), GFP_KERNEL);
@@ -76,7 +77,7 @@ static int ion_chunk_heap_allocate(struct ion_heap *heap,
 	}
 
 	buffer->priv_virt = table;
-	chunk_heap->allocated += allocated_size;
+	atomic_long_add(allocated_size, &chunk_heap->allocated);
 	return 0;
 err:
 	sg = table->sgl;
@@ -113,7 +114,7 @@ static void ion_chunk_heap_free(struct ion_buffer *buffer)
 		gen_pool_free(chunk_heap->pool, page_to_phys(sg_page(sg)),
 			      sg->length);
 	}
-	chunk_heap->allocated -= allocated_size;
+	atomic_long_sub(allocated_size, &chunk_heap->allocated);
 	sg_free_table(table);
 	kfree(table);
 }
@@ -169,7 +170,6 @@ struct ion_heap *ion_chunk_heap_create(struct ion_platform_heap *heap_data)
 	}
 	chunk_heap->base = heap_data->base;
 	chunk_heap->size = heap_data->size;
-	chunk_heap->allocated = 0;
 
 	gen_pool_add(chunk_heap->pool, chunk_heap->base, heap_data->size, -1);
 	chunk_heap->heap.ops = &chunk_heap_ops;