net: mana: Fall back to scattered pages for GDMA queues

Each GDMA queue ring is one dma_alloc_coherent() of the whole ring size.
Such high-order allocations fail first under memory fragmentation, so
queue setup can fail with memory still free.

The hardware does not need the ring physically contiguous:
mana_gd_create_dma_region() already maps it as a list of MANA_PAGE_SIZE
(4K) device addresses. Only the driver's linear CPU view needs
contiguity, and it goes through mana_gd_ring_ptr() and
mana_gd_ring_contig_avail(); change both to map offsets onto
scattered pages.

Add a fallback in mana_gd_alloc_memory(): data-path queues pass
allow_scatter=true, so when the contiguous allocation fails the ring is
backed by a vector of scattered PAGE_SIZE (order-0) coherent pages,
presenting the same DMA page-list layout to the device. The HW channel
bootstrap keeps allow_scatter=false, and the debugfs ring dumper reads
scattered rings through the same helpers.

Signed-off-by: Aditya Garg <gargaditya@linux.microsoft.com>
Link: https://patch.msgid.link/20260807210002.1695263-3-gargaditya@linux.microsoft.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>

drivers/net/ethernet/microsoft/mana/gdma_main.c

@@ -11,6 +11,7 @@
 #include <linux/msi.h>
 #include <linux/irqdomain.h>
 #include <linux/export.h>
+#include <linux/uaccess.h>
 
 #include <net/mana/mana.h>
 #include <net/mana/hw_channel.h>
@@ -374,28 +375,96 @@ int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
 EXPORT_SYMBOL_NS(mana_gd_send_request, "NET_MANA");
 
 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
-			 struct gdma_mem_info *gmi)
+			 struct gdma_mem_info *gmi, bool allow_scatter)
 {
+	unsigned int npages, i;
 	dma_addr_t dma_handle;
+	bool can_fallback;
 	void *buf;
 
 	if (length < MANA_PAGE_SIZE || !is_power_of_2(length))
 		return -EINVAL;
 
 	gmi->dev = gc->dev;
-	buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, GFP_KERNEL);
-	if (!buf)
+
+	/* An allocation that fits in one page does not benefit from
+	 * fallback.
+	 */
+	can_fallback = allow_scatter && length > PAGE_SIZE;
+
+	/* Warn only when there is no fallback to rescue the failure. */
+	buf = dma_alloc_coherent(gmi->dev, length, &dma_handle,
+				 GFP_KERNEL |
+				 (can_fallback ? __GFP_NOWARN : 0));
+	if (buf) {
+		gmi->dma_handle = dma_handle;
+		gmi->virt_addr = buf;
+		gmi->length = length;
+		gmi->nr_pages = 0;
+		return 0;
+	}
+
+	if (!can_fallback)
 		return -ENOMEM;
 
-	gmi->dma_handle = dma_handle;
-	gmi->virt_addr = buf;
+	/* length is a power of 2 above PAGE_SIZE, so this divides exactly. */
+	npages = length / PAGE_SIZE;
+
+	gmi->pages_va = kvcalloc(npages, sizeof(*gmi->pages_va), GFP_KERNEL);
+	if (!gmi->pages_va)
+		return -ENOMEM;
+
+	gmi->pages_dma = kvcalloc(npages, sizeof(*gmi->pages_dma), GFP_KERNEL);
+	if (!gmi->pages_dma)
+		goto free_va;
+
+	for (i = 0; i < npages; i++) {
+		gmi->pages_va[i] = dma_alloc_coherent(gmi->dev, PAGE_SIZE,
+						      &gmi->pages_dma[i],
+						      GFP_KERNEL);
+		if (!gmi->pages_va[i])
+			goto free_pages;
+	}
+
+	dev_info_ratelimited(gmi->dev,
+			     "contiguous %u-byte DMA alloc failed; using %u scattered pages\n",
+			     length, npages);
+
+	gmi->virt_addr = NULL;
+	gmi->dma_handle = 0;
 	gmi->length = length;
+	gmi->nr_pages = npages;
 
 	return 0;
+
+free_pages:
+	while (i--)
+		dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i],
+				  gmi->pages_dma[i]);
+	kvfree(gmi->pages_dma);
+	gmi->pages_dma = NULL;
+free_va:
+	kvfree(gmi->pages_va);
+	gmi->pages_va = NULL;
+	return -ENOMEM;
 }
 
 void mana_gd_free_memory(struct gdma_mem_info *gmi)
 {
+	unsigned int i;
+
+	if (gmi->nr_pages > 0) {
+		for (i = 0; i < gmi->nr_pages; i++)
+			dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i],
+					  gmi->pages_dma[i]);
+		kvfree(gmi->pages_va);
+		kvfree(gmi->pages_dma);
+		gmi->pages_va = NULL;
+		gmi->pages_dma = NULL;
+		gmi->nr_pages = 0;
+		return;
+	}
+
 	dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr,
 			  gmi->dma_handle);
 }
@@ -756,17 +825,66 @@ int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type)
 /* Return the CPU address of byte @offset within a queue's ring buffer. */
 static void *mana_gd_ring_ptr(const struct gdma_queue *q, u32 offset)
 {
+	const struct gdma_mem_info *gmi = &q->mem_info;
+
+	if (gmi->nr_pages > 0)
+		return (u8 *)gmi->pages_va[offset / PAGE_SIZE] +
+		       (offset & (PAGE_SIZE - 1));
+
 	return q->queue_mem_ptr + offset;
 }
 
-/* Number of bytes from @offset to the end of the ring buffer, i.e. the point
- * at which ring access wraps back to the start.
+/* Number of bytes from @offset to the end of the CPU-contiguous region: the
+ * rest of the ring, or the rest of the current page when scattered.
  */
 static u32 mana_gd_ring_contig_avail(const struct gdma_queue *q, u32 offset)
 {
+	if (q->mem_info.nr_pages > 0)
+		return PAGE_SIZE - (offset & (PAGE_SIZE - 1));
+
 	return q->queue_size - offset;
 }
 
+/* Copy up to @count bytes from ring offset *@pos of @q into user buffer @buf,
+ * so a scattered ring reads back as if it were contiguous. Returns bytes
+ * copied, 0 at end of ring, or a negative errno.
+ */
+ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf,
+			  size_t count, loff_t *pos)
+{
+	u32 size = q->queue_size;
+	loff_t off = *pos;
+	size_t copied = 0;
+
+	if (off < 0)
+		return -EINVAL;
+	if (off >= size || !count)
+		return 0;
+	count = min_t(size_t, count, size - off);
+
+	while (count) {
+		u32 offset = off;
+		u32 avail = mana_gd_ring_contig_avail(q, offset);
+		size_t chunk = min_t(size_t, count, avail);
+		size_t left = copy_to_user(buf, mana_gd_ring_ptr(q, offset),
+					   chunk);
+
+		chunk -= left;
+		buf += chunk;
+		off += chunk;
+		copied += chunk;
+		count -= chunk;
+		if (left)
+			break;
+	}
+
+	if (!copied)
+		return -EFAULT;
+
+	*pos = off;
+	return copied;
+}
+
 static void mana_gd_process_eqe(struct gdma_queue *eq)
 {
 	u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE);
@@ -1118,7 +1236,7 @@ int mana_gd_create_hwc_queue(struct gdma_dev *gd,
 		return -ENOMEM;
 
 	gmi = &queue->mem_info;
-	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
+	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, false);
 	if (err) {
 		dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n",
 			spec->type, spec->queue_size, err);
@@ -1193,7 +1311,7 @@ static int mana_gd_create_dma_region(struct gdma_dev *gd,
 	if (length < MANA_PAGE_SIZE || !is_power_of_2(length))
 		return -EINVAL;
 
-	if (!MANA_PAGE_ALIGNED(gmi->virt_addr))
+	if (gmi->nr_pages == 0 && !MANA_PAGE_ALIGNED(gmi->virt_addr))
 		return -EINVAL;
 
 	hwc = gc->hwc.driver_data;
@@ -1213,8 +1331,24 @@ static int mana_gd_create_dma_region(struct gdma_dev *gd,
 	req->page_count = num_page;
 	req->page_addr_list_len = num_page;
 
-	for (i = 0; i < num_page; i++)
-		req->page_addr_list[i] = gmi->dma_handle +  i * MANA_PAGE_SIZE;
+	if (gmi->nr_pages > 0) {
+		unsigned int subpages = PAGE_SIZE / MANA_PAGE_SIZE;
+		unsigned int idx = 0;
+		unsigned int pg, sub;
+
+		/* Each PAGE_SIZE chunk is physically contiguous and contains
+		 * PAGE_SIZE / MANA_PAGE_SIZE consecutive device pages.
+		 */
+		for (pg = 0; pg < gmi->nr_pages; pg++)
+			for (sub = 0; sub < subpages; sub++)
+				req->page_addr_list[idx++] =
+					gmi->pages_dma[pg] +
+					sub * MANA_PAGE_SIZE;
+	} else {
+		for (i = 0; i < num_page; i++)
+			req->page_addr_list[i] =
+				gmi->dma_handle + i * MANA_PAGE_SIZE;
+	}
 
 	err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp);
 	if (err)
@@ -1257,7 +1391,7 @@ int mana_gd_create_mana_eq(struct gdma_dev *gd,
 		return -ENOMEM;
 
 	gmi = &queue->mem_info;
-	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
+	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true);
 	if (err) {
 		dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n",
 			spec->type, spec->queue_size, err);
@@ -1312,7 +1446,7 @@ int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
 	queue->id = INVALID_QUEUE_ID;
 
 	gmi = &queue->mem_info;
-	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);
+	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true);
 	if (err) {
 		dev_err(gc->dev, "GDMA queue type: %d, size: %u, memory allocation err: %d\n",
 			spec->type, spec->queue_size, err);

drivers/net/ethernet/microsoft/mana/hw_channel.c

@@ -479,7 +479,7 @@ static int mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, u16 q_depth,
 	buf_size = MANA_PAGE_ALIGN(q_depth * max_msg_size);
 
 	gmi = &dma_buf->mem_info;
-	err = mana_gd_alloc_memory(gc, buf_size, gmi);
+	err = mana_gd_alloc_memory(gc, buf_size, gmi, false);
 	if (err) {
 		dev_err(hwc->dev, "Failed to allocate DMA buffer size: %u, err %d\n",
 			buf_size, err);

drivers/net/ethernet/microsoft/mana/mana_en.c

@@ -40,6 +40,9 @@ static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count
 {
 	struct gdma_queue *gdma_q = filp->private_data;
 
+	if (gdma_q->mem_info.nr_pages)
+		return mana_gd_read_ring(gdma_q, buf, count, pos);
+
 	return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr,
 				       gdma_q->queue_size);
 }

include/net/mana/gdma.h

@@ -241,6 +241,14 @@ struct gdma_mem_info {
 	void *virt_addr;
 	u64 length;
 
+	/* Scattered fallback: when @nr_pages > 0 the ring is that many
+	 * PAGE_SIZE coherent allocations in @pages_va/@pages_dma, not
+	 * @virt_addr/@dma_handle.
+	 */
+	void **pages_va;
+	dma_addr_t *pages_dma;
+	unsigned int nr_pages;
+
 	/* Allocated by the PF driver */
 	u64 dma_region_handle;
 };
@@ -516,6 +524,9 @@ int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
 
 void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
 
+ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf,
+			  size_t count, loff_t *pos);
+
 int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type);
 
 void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld,
@@ -672,6 +683,9 @@ enum {
 /* Driver supports dynamic interrupt moderation - DIM */
 #define GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION BIT(28)
 
+/* Driver supports non-contiguous queue buffers */
+#define GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS BIT(30)
+
 #define GDMA_DRV_CAP_FLAGS1 \
 	(GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT | \
 	 GDMA_DRV_CAP_FLAG_1_NAPI_WKDONE_FIX | \
@@ -688,7 +702,8 @@ enum {
 	 GDMA_DRV_CAP_FLAG_1_HANDLE_STALL_SQ_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECOVERY | \
 	 GDMA_DRV_CAP_FLAG_1_EQ_MSI_UNSHARE_MULTI_VPORT | \
-	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)
+	 GDMA_DRV_CAP_FLAG_1_DYN_INTERRUPT_MODERATION | \
+	 GDMA_DRV_CAP_FLAG_1_NON_CONTIGUOUS_BUFFERS)
 
 #define GDMA_DRV_CAP_FLAGS2 0
 
@@ -1049,7 +1064,7 @@ void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
 			      struct gdma_queue *queue);
 
 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
-			 struct gdma_mem_info *gmi);
+			 struct gdma_mem_info *gmi, bool allow_scatter);
 
 void mana_gd_free_memory(struct gdma_mem_info *gmi);