From 796bdb33e86aec8504bf8868e0665f120638ac72 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:47 -0400 Subject: [PATCH 01/73] dmaengine: Add API to combine configuration and preparation (sg and single) Previously, configuration and preparation required two separate calls. This works well when configuration is done only once during initialization. However, in cases where the burst length or source/destination address must be adjusted for each transfer, calling two functions is verbose and requires additional locking to ensure both steps complete atomically. Add a new API dmaengine_prep_config_single() and dmaengine_prep_config_sg() and callback device_prep_config_sg() that combines configuration and preparation into a single operation. If the configuration argument is passed as NULL, fall back to the existing implementation. Tested-by: Niklas Cassel Acked-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-1-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- Documentation/driver-api/dmaengine/client.rst | 9 +++ include/linux/dmaengine.h | 63 ++++++++++++++++--- 2 files changed, 64 insertions(+), 8 deletions(-) diff --git a/Documentation/driver-api/dmaengine/client.rst b/Documentation/driver-api/dmaengine/client.rst index d491e385d61a..5ee5d4a3596d 100644 --- a/Documentation/driver-api/dmaengine/client.rst +++ b/Documentation/driver-api/dmaengine/client.rst @@ -80,6 +80,10 @@ The details of these operations are: - slave_sg: DMA a list of scatter gather buffers from/to a peripheral + - config_sg: Similar with slave_sg, just pass down dma_slave_config + struct to avoid calling dmaengine_slave_config() every time adjusting the + burst length or the FIFO address is needed. + - peripheral_dma_vec: DMA an array of scatter gather buffers from/to a peripheral. Similar to slave_sg, but uses an array of dma_vec structures instead of a scatterlist. @@ -106,6 +110,11 @@ The details of these operations are: unsigned int sg_len, enum dma_data_direction direction, unsigned long flags); + struct dma_async_tx_descriptor *dmaengine_prep_config_sg( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, struct dma_slave_config *config); + struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( struct dma_chan *chan, const struct dma_vec *vecs, size_t nents, enum dma_data_direction direction, diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index b3d251c9734e..defa377d2ef5 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -835,6 +835,7 @@ struct dma_filter { * where the address and size of each segment is located in one entry of * the dma_vec array. * @device_prep_slave_sg: prepares a slave dma operation + * @device_prep_config_sg: prepares a slave DMA operation with dma_slave_config * @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio. * The function takes a buffer of size buf_len. The callback function will * be called after period_len bytes have been transferred. @@ -934,6 +935,10 @@ struct dma_device { struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, enum dma_transfer_direction direction, unsigned long flags, void *context); + struct dma_async_tx_descriptor *(*device_prep_config_sg)( + struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction direction, + unsigned long flags, struct dma_slave_config *config); struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)( struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len, size_t period_len, enum dma_transfer_direction direction, @@ -974,22 +979,44 @@ static inline bool is_slave_direction(enum dma_transfer_direction direction) (direction == DMA_DEV_TO_DEV); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single( - struct dma_chan *chan, dma_addr_t buf, size_t len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) { struct scatterlist sg; + + if (!chan || !chan->device) + return NULL; + sg_init_table(&sg, 1); sg_dma_address(&sg) = buf; sg_dma_len(&sg) = len; - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, &sg, 1, dir, + flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, &sg, 1, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_single(struct dma_chan *chan, dma_addr_t buf, size_t len, + enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_single(chan, buf, len, dir, flags, NULL); +} + /** * dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor * @chan: The channel to be used for this descriptor @@ -1010,17 +1037,37 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec( dir, flags); } -static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg( - struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len, - enum dma_transfer_direction dir, unsigned long flags) +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags, struct dma_slave_config *config) { - if (!chan || !chan->device || !chan->device->device_prep_slave_sg) + if (!chan || !chan->device) + return NULL; + + if (chan->device->device_prep_config_sg) + return chan->device->device_prep_config_sg(chan, sgl, sg_len, + dir, flags, config); + + if (config) + if (dmaengine_slave_config(chan, config)) + return NULL; + + if (!chan->device->device_prep_slave_sg) return NULL; return chan->device->device_prep_slave_sg(chan, sgl, sg_len, dir, flags, NULL); } +static inline struct dma_async_tx_descriptor * +dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, enum dma_transfer_direction dir, + unsigned long flags) +{ + return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL); +} + #ifdef CONFIG_RAPIDIO_DMA_ENGINE struct rio_dma_ext; static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg( From af900b7dc1e1cdac571ac38e7fee80f1a1776a62 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:48 -0400 Subject: [PATCH 02/73] dmaengine: Add safe API to combine configuration and preparation Introduce dmaengine_prep_config_single_safe() and dmaengine_prep_config_sg_safe() to provide a reentrant-safe way to combine slave configuration and transfer preparation. Drivers may implement the new device_prep_config_sg() callback to perform both steps atomically. If the callback is not provided, the helpers fall back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg() under per-channel spinlock protection. Tested-by: Niklas Cassel Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-2-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.c | 2 + include/linux/dmaengine.h | 86 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 9049171df857..23e3bb18c166 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -1100,6 +1100,8 @@ static int __dma_async_device_channel_register(struct dma_device *device, chan->dev->device.parent = device->dev; chan->dev->chan = chan; chan->dev->dev_id = device->dev_id; + spin_lock_init(&chan->lock); + if (!name) dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id); else diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index defa377d2ef5..6fe46c0c9452 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -322,6 +322,8 @@ struct dma_router { * @slave: ptr to the device using this channel * @cookie: last cookie value returned to client * @completed_cookie: last completed cookie for this channel + * @lock: protect between config and prepare transfer when driver have not + * implemented callback device_prep_config_sg(). * @chan_id: channel ID for sysfs * @dev: class device for sysfs * @name: backlink name for sysfs @@ -341,6 +343,12 @@ struct dma_chan { dma_cookie_t cookie; dma_cookie_t completed_cookie; + /* + * protect between config and prepare transfer because *_prep() may be + * called from complete callback, which is in GFP_NOSLEEP context. + */ + spinlock_t lock; + /* sysfs */ int chan_id; struct dma_chan_dev *dev; @@ -1068,6 +1076,84 @@ dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL); } +/** + * dmaengine_prep_config_sg_safe - prepare a scatter-gather DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @sgl: scatterlist for the transfer + * @sg_len: number of entries in @sgl + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Prepare a DMA scatter-gather transfer together with a corresponding slave + * configuration update in a re-entrant and race-safe manner. + * + * DMA engine drivers may implement the optional + * device_prep_config_sg() callback to perform both the slave configuration + * and descriptor preparation atomically. In this case, the operation is + * fully handled by the DMA engine driver. + * + * If the DMA engine driver does not implement device_prep_config_sg(), falls + * back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg(). + * The fallback path is protected by a per-channel spinlock to ensure that + * concurrent callers cannot interleave configuration and descriptor preparation + * on the same DMA channel. + * + * Return: Pointer to a prepared DMA async transaction descriptor on success, + * or %NULL if the transfer could not be prepared. + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_sg_safe(struct dma_chan *chan, struct scatterlist *sgl, + unsigned int sg_len, + enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct dma_async_tx_descriptor *tx; + unsigned long spinlock_flags; + + if (!chan || !chan->device) + return NULL; + + if (!chan->device->device_prep_config_sg) + spin_lock_irqsave(&chan->lock, spinlock_flags); + + tx = dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, config); + + if (!chan->device->device_prep_config_sg) + spin_unlock_irqrestore(&chan->lock, spinlock_flags); + + return tx; +} + +/** + * dmaengine_prep_config_single_safe - prepare a single-buffer DMA transfer + * with atomic slave configuration update + * @chan: DMA channel + * @buf: DMA buffer address + * @len: length of the transfer in bytes + * @dir: DMA transfer direction + * @flags: transfer preparation flags + * @config: DMA slave configuration for this transfer + * + * Detail see dmaengine_prep_config_sg_safe(). + */ +static inline struct dma_async_tx_descriptor * +dmaengine_prep_config_single_safe(struct dma_chan *chan, dma_addr_t buf, + size_t len, enum dma_transfer_direction dir, + unsigned long flags, + struct dma_slave_config *config) +{ + struct scatterlist sg; + + sg_init_table(&sg, 1); + sg_dma_address(&sg) = buf; + sg_dma_len(&sg) = len; + + return dmaengine_prep_config_sg_safe(chan, &sg, 1, dir, flags, config); +} + #ifdef CONFIG_RAPIDIO_DMA_ENGINE struct rio_dma_ext; static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg( From 9605825841061bbdd2fc2a0218098373c539173e Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:49 -0400 Subject: [PATCH 03/73] PCI: endpoint: pci-epf-test: Use dmaengine_prep_config_single() to simplify code Use dmaengine_prep_config_single() to simplify code. No functional change. Tested-by: Niklas Cassel Reviewed-by: Damien Le Moal Acked-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-3-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/pci/endpoint/functions/pci-epf-test.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 4802d4f80f78..d4905aa8e4c0 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -183,12 +183,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test, else sconf.src_addr = dma_remote; - if (dmaengine_slave_config(chan, &sconf)) { - dev_err(dev, "DMA slave config fail\n"); - return -EIO; - } - tx = dmaengine_prep_slave_single(chan, dma_local, len, dir, - flags); + tx = dmaengine_prep_config_single(chan, dma_local, len, + dir, flags, &sconf); } else { tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len, flags); From bfb66d8098dbbaaada3ab877eda21cd447115c95 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:50 -0400 Subject: [PATCH 04/73] dmaengine: dw-edma: Use new .device_prep_config_sg() callback Use the new .device_prep_config_sg() callback to combine configuration and descriptor preparation. No functional changes. Tested-by: Niklas Cassel Reviewed-by: Damien Le Moal Reviewed-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-4-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 89a4c498a17b..8c192d63113c 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -577,10 +577,11 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) } static struct dma_async_tx_descriptor * -dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, - unsigned int len, - enum dma_transfer_direction direction, - unsigned long flags, void *context) +dw_edma_device_prep_config_sg(struct dma_chan *dchan, struct scatterlist *sgl, + unsigned int len, + enum dma_transfer_direction direction, + unsigned long flags, + struct dma_slave_config *config) { struct dw_edma_transfer xfer; @@ -591,6 +592,9 @@ dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl, xfer.flags = flags; xfer.type = EDMA_XFER_SCATTER_GATHER; + if (config && dw_edma_device_config(dchan, config)) + return NULL; + return dw_edma_device_transfer(&xfer); } @@ -970,7 +974,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_terminate_all = dw_edma_device_terminate_all; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; - dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg; + dma->device_prep_config_sg = dw_edma_device_prep_config_sg; dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic; dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma; From 1af246e9d222f93aee59f3b47ff3bd59d08725e7 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:51 -0400 Subject: [PATCH 05/73] dmaengine: dw-edma: Pass dma_slave_config to dw_edma_device_transfer() Pass dma_slave_config to dw_edma_device_transfer() to support atomic configuration and descriptor preparation when a non-NULL config is provided to device_prep_config_sg(). Tested-by: Niklas Cassel Reviewed-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-5-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 8c192d63113c..1fec1b52e3d4 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -267,6 +267,20 @@ static int dw_edma_device_config(struct dma_chan *dchan, return 0; } +static struct dma_slave_config * +dw_edma_device_get_config(struct dma_chan *dchan, + struct dma_slave_config *config) +{ + struct dw_edma_chan *chan; + + if (config) + return config; + + chan = dchan2dw_edma_chan(dchan); + + return &chan->config; +} + static int dw_edma_device_pause(struct dma_chan *dchan) { struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); @@ -385,7 +399,8 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, } static struct dma_async_tx_descriptor * -dw_edma_device_transfer(struct dw_edma_transfer *xfer) +dw_edma_device_transfer(struct dw_edma_transfer *xfer, + struct dma_slave_config *config) { struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); enum dma_transfer_direction dir = xfer->direction; @@ -472,8 +487,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer) src_addr = xfer->xfer.il->src_start; dst_addr = xfer->xfer.il->dst_start; } else { - src_addr = chan->config.src_addr; - dst_addr = chan->config.dst_addr; + src_addr = config->src_addr; + dst_addr = config->dst_addr; } if (dir == DMA_DEV_TO_MEM) @@ -595,7 +610,7 @@ dw_edma_device_prep_config_sg(struct dma_chan *dchan, struct scatterlist *sgl, if (config && dw_edma_device_config(dchan, config)) return NULL; - return dw_edma_device_transfer(&xfer); + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, config)); } static struct dma_async_tx_descriptor * @@ -614,7 +629,7 @@ dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr, xfer.flags = flags; xfer.type = EDMA_XFER_CYCLIC; - return dw_edma_device_transfer(&xfer); + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL)); } static struct dma_async_tx_descriptor * @@ -630,7 +645,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan, xfer.flags = flags; xfer.type = EDMA_XFER_INTERLEAVED; - return dw_edma_device_transfer(&xfer); + return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL)); } static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, From bd00d2c4a1b2a1e7ad3060d3d606fb4c9db6a064 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:52 -0400 Subject: [PATCH 06/73] nvmet: pci-epf: Remove unnecessary dmaengine_terminate_sync() on each DMA transfer dmaengine_terminate_sync() cancels all pending requests. Calling it for every DMA transfer is unnecessary and counterproductive. This function is generally intended for cleanup paths such as module removal, device close, or unbind operations. Remove the redundant calls for success path and keep it only at error path. Tested-by: Niklas Cassel Reviewed-by: Damien Le Moal Acked-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-6-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/nvme/target/pci-epf.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 4e9db96ebfec..2afe8f4d0e46 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -420,10 +420,9 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf, if (dma_sync_wait(chan, cookie) != DMA_COMPLETE) { dev_err(dev, "DMA transfer failed\n"); ret = -EIO; + dmaengine_terminate_sync(chan); } - dmaengine_terminate_sync(chan); - unmap: dma_unmap_single(dma_dev, dma_addr, seg->length, dir); From a0fba0a49f77effd3962723b1fa14c766fbc0ec4 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:53 -0400 Subject: [PATCH 07/73] nvmet: pci-epf: Use dmaengine_prep_config_single_safe() API Use the new dmaengine_prep_config_single_safe() API to combine the configuration and descriptor preparation into a single call. Since dmaengine_prep_config_single_safe() performs the configuration and preparation atomically and the mutex can be removed. Tested-by: Niklas Cassel Acked-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-7-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/nvme/target/pci-epf.c | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/drivers/nvme/target/pci-epf.c b/drivers/nvme/target/pci-epf.c index 2afe8f4d0e46..b1ba2d0bea6d 100644 --- a/drivers/nvme/target/pci-epf.c +++ b/drivers/nvme/target/pci-epf.c @@ -210,9 +210,7 @@ struct nvmet_pci_epf { bool dma_enabled; struct dma_chan *dma_tx_chan; - struct mutex dma_tx_lock; struct dma_chan *dma_rx_chan; - struct mutex dma_rx_lock; struct mutex mmio_lock; @@ -295,9 +293,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf) struct dma_chan *chan; dma_cap_mask_t mask; - mutex_init(&nvme_epf->dma_rx_lock); - mutex_init(&nvme_epf->dma_tx_lock); - dma_cap_zero(mask); dma_cap_set(DMA_SLAVE, mask); @@ -336,8 +331,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf) nvme_epf->dma_rx_chan = NULL; out_dma_no_rx: - mutex_destroy(&nvme_epf->dma_rx_lock); - mutex_destroy(&nvme_epf->dma_tx_lock); nvme_epf->dma_enabled = false; dev_info(&epf->dev, "DMA not supported, falling back to MMIO\n"); @@ -352,8 +345,6 @@ static void nvmet_pci_epf_deinit_dma(struct nvmet_pci_epf *nvme_epf) nvme_epf->dma_tx_chan = NULL; dma_release_channel(nvme_epf->dma_rx_chan); nvme_epf->dma_rx_chan = NULL; - mutex_destroy(&nvme_epf->dma_rx_lock); - mutex_destroy(&nvme_epf->dma_tx_lock); nvme_epf->dma_enabled = false; } @@ -368,18 +359,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf, struct dma_chan *chan; dma_cookie_t cookie; dma_addr_t dma_addr; - struct mutex *lock; int ret; switch (dir) { case DMA_FROM_DEVICE: - lock = &nvme_epf->dma_rx_lock; chan = nvme_epf->dma_rx_chan; sconf.direction = DMA_DEV_TO_MEM; sconf.src_addr = seg->pci_addr; break; case DMA_TO_DEVICE: - lock = &nvme_epf->dma_tx_lock; chan = nvme_epf->dma_tx_chan; sconf.direction = DMA_MEM_TO_DEV; sconf.dst_addr = seg->pci_addr; @@ -388,22 +376,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf, return -EINVAL; } - mutex_lock(lock); - dma_dev = dmaengine_get_dma_device(chan); dma_addr = dma_map_single(dma_dev, seg->buf, seg->length, dir); ret = dma_mapping_error(dma_dev, dma_addr); if (ret) - goto unlock; + return ret; - ret = dmaengine_slave_config(chan, &sconf); - if (ret) { - dev_err(dev, "Failed to configure DMA channel\n"); - goto unmap; - } - - desc = dmaengine_prep_slave_single(chan, dma_addr, seg->length, - sconf.direction, DMA_CTRL_ACK); + desc = dmaengine_prep_config_single_safe(chan, dma_addr, seg->length, + sconf.direction, + DMA_CTRL_ACK, &sconf); if (!desc) { dev_err(dev, "Failed to prepare DMA\n"); ret = -EIO; @@ -426,9 +407,6 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf, unmap: dma_unmap_single(dma_dev, dma_addr, seg->length, dir); -unlock: - mutex_unlock(lock); - return ret; } From 53191cc449db1cf4f25db275978c61b1c6aaeba9 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:54 -0400 Subject: [PATCH 08/73] PCI: epf-mhi: Use dmaengine_prep_config_single() to simplify code Use dmaengine_prep_config_single() to simplify pci_epf_mhi_edma_read[_sync]() and pci_epf_mhi_edma_write[_sync](). No functional change. Tested-by: Niklas Cassel Acked-by: Manivannan Sadhasivam Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-8-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/pci/endpoint/functions/pci-epf-mhi.c | 52 ++++++-------------- 1 file changed, 16 insertions(+), 36 deletions(-) diff --git a/drivers/pci/endpoint/functions/pci-epf-mhi.c b/drivers/pci/endpoint/functions/pci-epf-mhi.c index 7f5326925ed5..c3e3b58fb86c 100644 --- a/drivers/pci/endpoint/functions/pci-epf-mhi.c +++ b/drivers/pci/endpoint/functions/pci-epf-mhi.c @@ -328,12 +328,6 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl, config.direction = DMA_DEV_TO_MEM; config.src_addr = buf_info->host_addr; - ret = dmaengine_slave_config(chan, &config); - if (ret) { - dev_err(dev, "Failed to configure DMA channel\n"); - goto err_unlock; - } - dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size, DMA_FROM_DEVICE); ret = dma_mapping_error(dma_dev, dst_addr); @@ -342,9 +336,10 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl, goto err_unlock; } - desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size, - DMA_DEV_TO_MEM, - DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size, + DMA_DEV_TO_MEM, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT, + &config); if (!desc) { dev_err(dev, "Failed to prepare DMA\n"); ret = -EIO; @@ -401,12 +396,6 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl, config.direction = DMA_MEM_TO_DEV; config.dst_addr = buf_info->host_addr; - ret = dmaengine_slave_config(chan, &config); - if (ret) { - dev_err(dev, "Failed to configure DMA channel\n"); - goto err_unlock; - } - src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size, DMA_TO_DEVICE); ret = dma_mapping_error(dma_dev, src_addr); @@ -415,9 +404,10 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl, goto err_unlock; } - desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size, - DMA_MEM_TO_DEV, - DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size, + DMA_MEM_TO_DEV, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT, + &config); if (!desc) { dev_err(dev, "Failed to prepare DMA\n"); ret = -EIO; @@ -506,12 +496,6 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl, config.direction = DMA_DEV_TO_MEM; config.src_addr = buf_info->host_addr; - ret = dmaengine_slave_config(chan, &config); - if (ret) { - dev_err(dev, "Failed to configure DMA channel\n"); - goto err_unlock; - } - dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size, DMA_FROM_DEVICE); ret = dma_mapping_error(dma_dev, dst_addr); @@ -520,9 +504,10 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl, goto err_unlock; } - desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size, - DMA_DEV_TO_MEM, - DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size, + DMA_DEV_TO_MEM, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT, + &config); if (!desc) { dev_err(dev, "Failed to prepare DMA\n"); ret = -EIO; @@ -585,12 +570,6 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl, config.direction = DMA_MEM_TO_DEV; config.dst_addr = buf_info->host_addr; - ret = dmaengine_slave_config(chan, &config); - if (ret) { - dev_err(dev, "Failed to configure DMA channel\n"); - goto err_unlock; - } - src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size, DMA_TO_DEVICE); ret = dma_mapping_error(dma_dev, src_addr); @@ -599,9 +578,10 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl, goto err_unlock; } - desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size, - DMA_MEM_TO_DEV, - DMA_CTRL_ACK | DMA_PREP_INTERRUPT); + desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size, + DMA_MEM_TO_DEV, + DMA_CTRL_ACK | DMA_PREP_INTERRUPT, + &config); if (!desc) { dev_err(dev, "Failed to prepare DMA\n"); ret = -EIO; From c9e9927c6d8346cdf6555a8f97da093980172e4b Mon Sep 17 00:00:00 2001 From: Frank Li Date: Thu, 21 May 2026 11:32:55 -0400 Subject: [PATCH 09/73] crypto: atmel: Use dmaengine_prep_config_sg() API Using new API dmaengine_prep_config_sg() to simple code. dmaengine_prep_config_sg() does not distinguish between configuration failures and descriptor preparation failures, as both are reported through a NULL return value. Converting both cases to -ENOMEM is therefore acceptable and consistent with the helper's abstraction. In practice, most users only care whether the operation succeeds or fails, and do not depend on the exact errno value returned from this path. Tested-by: Niklas Cassel Acked-by: Nicolas Ferre Signed-off-by: Frank Li Link: https://patch.msgid.link/20260521-dma_prep_config-v7-9-1f73f4899883@nxp.com Signed-off-by: Vinod Koul --- drivers/crypto/atmel-aes.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index b393689400b4..d890b5a277b9 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -795,7 +795,6 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, struct dma_slave_config config; dma_async_tx_callback callback; struct atmel_aes_dma *dma; - int err; memset(&config, 0, sizeof(config)); config.src_addr_width = addr_width; @@ -820,12 +819,9 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, return -EINVAL; } - err = dmaengine_slave_config(dma->chan, &config); - if (err) - return err; - - desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + desc = dmaengine_prep_config_sg(dma->chan, dma->sg, dma->sg_len, dir, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK, + &config); if (!desc) return -ENOMEM; From 287bdea77529e6abac5fe15461d93c1acdcb07e9 Mon Sep 17 00:00:00 2001 From: Suraj Gupta Date: Thu, 25 Jun 2026 21:40:16 +0530 Subject: [PATCH 10/73] dt-bindings: dma: xlnx,axi-dma: Restore xlnx,flush-fsync as u32 The DT schema conversion incorrectly changed xlnx,flush-fsync from a u32 property to a boolean. The original binding documented values 1, 2, and 3 to select which VDMA channel(s) flush on frame sync. Restore the uint32 type with the documented enum values and fix the example accordingly. Fixes: 2d5c2952b972 ("dt-bindings: dma: xlnx,axi-dma: Convert to DT schema") Signed-off-by: Suraj Gupta Acked-by: Conor Dooley Link: https://patch.msgid.link/20260625161016.1249570-1-suraj.gupta2@amd.com Signed-off-by: Vinod Koul --- .../devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml index 340ae9e91cb0..95b951eea1b7 100644 --- a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml +++ b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml @@ -81,8 +81,13 @@ properties: description: Should be the number of framebuffers as configured in h/w. xlnx,flush-fsync: - type: boolean - description: Tells which channel to Flush on Frame sync. + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2, 3] + description: + Tells which channel to flush on frame sync. + 1 - flush both channels + 2 - flush mm2s channel + 3 - flush s2mm channel xlnx,sg-length-width: $ref: /schemas/types.yaml#/definitions/uint32 @@ -251,7 +256,7 @@ examples: "m_axi_s2mm_aclk", "m_axis_mm2s_aclk", "s_axis_s2mm_aclk"; xlnx,num-fstores = <8>; - xlnx,flush-fsync; + xlnx,flush-fsync = <1>; xlnx,addrwidth = <32>; dma-channel-mm2s { From fa9cb11584851414b25fd8bf9f59518424b5917c Mon Sep 17 00:00:00 2001 From: "Rob Herring (Arm)" Date: Fri, 12 Jun 2026 16:52:25 -0500 Subject: [PATCH 11/73] dt-bindings: dma: xilinx: Fix "xlnx,irq-delay" type "xlnx,irq-delay" programs an 8-bit delay field in the DMA control register, and the driver stores and reads it as a byte. The binding described the property as a uint32 cell, which made the helper type check report the driver as wrong. Document "xlnx,irq-delay" as uint8 so the generated schema reflects the hardware field width and the existing driver access. Assisted-by: Codex:gpt-5-5 Signed-off-by: Rob Herring (Arm) Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260612215226.1887726-1-robh@kernel.org Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml index 95b951eea1b7..2c88e927d9d3 100644 --- a/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml +++ b/Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml @@ -98,7 +98,7 @@ properties: Width in bits of the length register as configured in hardware. xlnx,irq-delay: - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint8 minimum: 0 maximum: 255 description: From 467265c750edd7ab43803deeafe7d3120a791d32 Mon Sep 17 00:00:00 2001 From: Vladimir Zapolskiy Date: Wed, 1 Jul 2026 23:07:03 +0300 Subject: [PATCH 12/73] dmaengine: mediatek: mtk-uart-apdma: Return -ENOMEM on memory allocation failure If dynamic memory allocation in driver's probe function execution fails, it should be reported to the driver's framework with -ENOMEM error code. Fixes: 9135408c3ace ("dmaengine: mediatek: Add MediaTek UART APDMA support") Signed-off-by: Vladimir Zapolskiy Reviewed-by: AngeloGioacchino Del Regno Reviewed-by: Frank Li Reviewed-by: Matthias Brugger Link: https://patch.msgid.link/20260701200703.117929-1-vz@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/mediatek/mtk-uart-apdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/mediatek/mtk-uart-apdma.c b/drivers/dma/mediatek/mtk-uart-apdma.c index c269d84d7bd2..f74e9a328588 100644 --- a/drivers/dma/mediatek/mtk-uart-apdma.c +++ b/drivers/dma/mediatek/mtk-uart-apdma.c @@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev) for (i = 0; i < mtkd->dma_requests; i++) { c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL); if (!c) { - rc = -ENODEV; + rc = -ENOMEM; goto err_no_dma; } From 0d2b094b1c10be619a63f53a610587bcabbee06b Mon Sep 17 00:00:00 2001 From: Louis-Alexis Eyraud Date: Wed, 1 Jul 2026 17:47:20 +0200 Subject: [PATCH 13/73] dt-bindings: dma: mediatek,uart-dma: add support for MT8189 SoC Add the compatible string for the APDMA IP found in MT8189 SoC, that supports 35-bits addressing as MT6985 SoC. Signed-off-by: Louis-Alexis Eyraud Reviewed-by: AngeloGioacchino Del Regno Acked-by: Conor Dooley Link: https://patch.msgid.link/20260701-mt8189-dt-bindings-uart-dma-v1-1-c7106216a40d@collabora.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml index 3708518fe7fc..bba20e88a6dc 100644 --- a/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml +++ b/Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml @@ -40,6 +40,7 @@ properties: - items: - enum: - mediatek,mt6991-uart-dma + - mediatek,mt8189-uart-dma - mediatek,mt8196-uart-dma - const: mediatek,mt6985-uart-dma - enum: From 95cf38ae309f21b651fb7b8afe267eb3c84017a3 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Tue, 9 Jun 2026 14:25:31 -0700 Subject: [PATCH 14/73] dmaengine: tegra210-adma: use platform to ioremap Simpler to call devm_platform_ioremap_resource() as it returns multiple error messages for whichever part fails. Signed-off-by: Rosen Penev Reviewed-by: Jon Hunter Reviewed-by: Frank Li Link: https://patch.msgid.link/20260609212531.22044-1-rosenp@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/tegra210-adma.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/drivers/dma/tegra210-adma.c b/drivers/dma/tegra210-adma.c index ceaee1e33e68..21a381d022cf 100644 --- a/drivers/dma/tegra210-adma.c +++ b/drivers/dma/tegra210-adma.c @@ -1087,15 +1087,9 @@ static int tegra_adma_probe(struct platform_device *pdev) } } else { /* If no 'page' property found, then reg DT binding would be legacy */ - res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (res_base) { - tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base); - if (IS_ERR(tdma->base_addr)) - return PTR_ERR(tdma->base_addr); - } else { - return dev_err_probe(&pdev->dev, -ENODEV, - "failed to get memory resource\n"); - } + tdma->base_addr = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(tdma->base_addr)) + return PTR_ERR(tdma->base_addr); tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset; } From 0b6d055edb55ecadadf54e930c2b4fab76fa9a5a Mon Sep 17 00:00:00 2001 From: Suraj Gupta Date: Fri, 26 Jun 2026 14:56:54 +0530 Subject: [PATCH 15/73] dmaengine: xilinx_dma: Fix channel idle state management in AXIDMA and MCDMA interrupt handlers Fix a race condition in AXIDMA and MCDMA irq handlers where the channel could be incorrectly marked as idle and attempt spurious transfers when descriptors are still being processed. The issue occurs when: 1. Multiple descriptors are queued and active. 2. An interrupt fires after completing some descriptors. 3. xilinx_dma_complete_descriptor() moves completed descriptors to done_list. 4. Channel is marked idle and start_transfer() is called even though active_list still contains unprocessed descriptors. 5. This leads to premature transfer attempts and potential descriptor corruption or missed completions. Only mark the channel as idle and start new transfers when the active list is actually empty, ensuring proper channel state management and avoiding spurious transfer attempts. Fixes: c0bba3a99f07 ("dmaengine: vdma: Add Support for Xilinx AXI Direct Memory Access Engine") Tested-by: Folker Schwesinger Signed-off-by: Suraj Gupta Co-developed-by: Srinivas Neeli Signed-off-by: Srinivas Neeli Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260626092656.1563871-2-suraj.gupta2@amd.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 404235c17353..ca396b709742 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1893,8 +1893,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data) if (status & XILINX_MCDMA_IRQ_IOC_MASK) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } @@ -1950,8 +1952,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data) XILINX_DMA_DMASR_DLY_CNT_IRQ)) { spin_lock(&chan->lock); xilinx_dma_complete_descriptor(chan); - chan->idle = true; - chan->start_transfer(chan); + if (list_empty(&chan->active_list)) { + chan->idle = true; + chan->start_transfer(chan); + } spin_unlock(&chan->lock); } From 6078690034790131b9a59081bdf30e26de2254af Mon Sep 17 00:00:00 2001 From: Suraj Gupta Date: Fri, 26 Jun 2026 14:56:55 +0530 Subject: [PATCH 16/73] dmaengine: xilinx_dma: Enable transfer chaining for AXIDMA and MCDMA by removing idle restriction Relax the idle check in xilinx_dma_start_transfer() and xilinx_mcdma_start_transfer() that prevented new transfers from being queued when the channel was busy, so scatter-gather transfers can be chained onto an in-flight transfer. In scatter-gather mode, only update the CURDESC register when the active list is empty to avoid interfering with transfers already in progress. When the active list contains transfers, the hardware tail pointer extension mechanism handles chaining automatically via the descriptor next pointer chain, which is set up at channel allocation and preserved across descriptor recycling. Direct (non-SG) mode has no descriptor queue: writing the BTT register launches a transfer immediately, so a new transfer must not be programmed while one is in flight. Keep those transfers serialized by retaining the idle check on the non-SG path. MCDMA always operates in scatter-gather mode, so it is unaffected. Signed-off-by: Suraj Gupta Co-developed-by: Srinivas Neeli Signed-off-by: Srinivas Neeli Tested-by: Folker Schwesinger Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260626092656.1563871-3-suraj.gupta2@amd.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index ca396b709742..6e7b183cb499 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1580,7 +1580,14 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) return; } - if (!chan->idle) + /* + * Direct (non-SG) mode has no descriptor queue: writing the BTT + * register launches a transfer immediately, so a new transfer must + * not be programmed while one is in flight. Keep such transfers + * serialized. SG mode supports chaining onto a running transfer via + * tail-pointer extension, so it is allowed to proceed when busy. + */ + if (!chan->has_sg && !chan->idle) return; head_desc = list_first_entry(&chan->pending_list, @@ -1599,7 +1606,7 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); } - if (chan->has_sg) + if (chan->has_sg && list_empty(&chan->active_list)) xilinx_write(chan, XILINX_DMA_REG_CURDESC, head_desc->async_tx.phys); reg &= ~XILINX_DMA_CR_DELAY_MAX; @@ -1660,9 +1667,6 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) if (chan->err) return; - if (!chan->idle) - return; - if (list_empty(&chan->pending_list)) return; @@ -1685,8 +1689,9 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); /* Program current descriptor */ - xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), - head_desc->async_tx.phys); + if (chan->has_sg && list_empty(&chan->active_list)) + xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest), + head_desc->async_tx.phys); /* Program channel enable register */ reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET); From 887b3119380cde56f648130029062c223341a1b3 Mon Sep 17 00:00:00 2001 From: Suraj Gupta Date: Fri, 26 Jun 2026 14:56:56 +0530 Subject: [PATCH 17/73] dmaengine: xilinx_dma: Optimize control register write and channel start logic for AXIDMA and MCDMA in corresponding start_transfer() Optimize AXI DMA control register programming by consolidating coalesce count and delay configuration into a single register write. Previously, the coalesce count was written separately from the delay configuration, resulting in two register writes. Combine these into one write operation to reduce bus overhead. Additionally, avoid redundant channel starts in xilinx_dma_start_transfer() and xilinx_mcdma_start_transfer() by only calling xilinx_dma_start() when the channel is actually idle. Tested-by: Folker Schwesinger Signed-off-by: Suraj Gupta Co-developed-by: Srinivas Neeli Signed-off-by: Srinivas Neeli Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260626092656.1563871-4-suraj.gupta2@amd.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 6e7b183cb499..829601d8a16f 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -1603,7 +1603,6 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) reg &= ~XILINX_DMA_CR_COALESCE_MAX; reg |= chan->desc_pendingcount << XILINX_DMA_CR_COALESCE_SHIFT; - dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); } if (chan->has_sg && list_empty(&chan->active_list)) @@ -1614,7 +1613,8 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan) reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK; dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg); - xilinx_dma_start(chan); + if (chan->idle) + xilinx_dma_start(chan); if (chan->err) return; @@ -1703,7 +1703,8 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan) reg |= XILINX_MCDMA_CR_RUNSTOP_MASK; dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg); - xilinx_dma_start(chan); + if (chan->idle) + xilinx_dma_start(chan); if (chan->err) return; From 516ba2d8b7aac4238f9fcbd58579c43c71b9b695 Mon Sep 17 00:00:00 2001 From: Golla Nagendra Date: Tue, 30 Jun 2026 12:18:43 +0530 Subject: [PATCH 18/73] dmaengine: zynqmp_dma: fix race between runtime PM and device removal In zynqmp_dma_remove(), runtime PM was disabled only after checking state and doing a manual suspend. This can race with runtime PM in the remove/unbind (rmmod) path. Disable runtime PM first, then suspend only if the device is not already suspended. To prevent any further runtime PM transitions. Fixes: 72dd8b2914b5 ("dmaengine: zynqmp_dma: Add shutdown operation support") Co-developed-by: Prasanna Kumar T S M Signed-off-by: Prasanna Kumar T S M Signed-off-by: Golla Nagendra Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260630064844.705173-2-nagendra.golla@amd.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/zynqmp_dma.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index f6a812e49ddc..ca2dee0012c8 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1170,9 +1170,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev) dma_async_device_unregister(&zdev->common); zynqmp_dma_chan_remove(zdev->chan); - if (pm_runtime_active(zdev->dev)) - zynqmp_dma_runtime_suspend(zdev->dev); pm_runtime_disable(zdev->dev); + if (!pm_runtime_status_suspended(zdev->dev)) + zynqmp_dma_runtime_suspend(zdev->dev); } static const struct of_device_id zynqmp_dma_of_match[] = { From f7e89cba18a1ca6462712ae459a88dd40537b693 Mon Sep 17 00:00:00 2001 From: Golla Nagendra Date: Tue, 30 Jun 2026 12:18:44 +0530 Subject: [PATCH 19/73] dmaengine: zynqmp_dma: fix kernel doc for zynqmp_dma_remove() The zynqmp_dma_remove() function was converted from returning int to void, but the kernel doc comment was not updated to reflect this change. Remove the stale "Return: Always '0'" documentation that no longer applies to the void function. Fixes: b1c50ac25425 ("dmaengine: xilinx: zynqmp_dma: Convert to platform remove callback returning void") Signed-off-by: Golla Nagendra Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/20260630064844.705173-3-nagendra.golla@amd.com Signed-off-by: Vinod Koul --- drivers/dma/xilinx/zynqmp_dma.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/drivers/dma/xilinx/zynqmp_dma.c b/drivers/dma/xilinx/zynqmp_dma.c index ca2dee0012c8..2b03584102ac 100644 --- a/drivers/dma/xilinx/zynqmp_dma.c +++ b/drivers/dma/xilinx/zynqmp_dma.c @@ -1159,8 +1159,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev) /** * zynqmp_dma_remove - Driver remove function * @pdev: Pointer to the platform_device structure - * - * Return: Always '0' */ static void zynqmp_dma_remove(struct platform_device *pdev) { From cbabdd6ce1b313b5877c7fbb2f5e2f7936564d2f Mon Sep 17 00:00:00 2001 From: Vladimir Zapolskiy Date: Tue, 30 Jun 2026 17:42:14 +0300 Subject: [PATCH 20/73] dmaengine: hisilicon: Return -ENOMEM on dynamic memory allocation in probe Out of memory situation on driver's probe is expected to be reported to the driver's framework with a proper -ENOMEM error code. Fixes: e9f08b65250d ("dmaengine: hisilicon: Add Kunpeng DMA engine support") Signed-off-by: Vladimir Zapolskiy Reviewed-by: Frank Li Link: https://patch.msgid.link/20260630144214.4080302-1-vz@kernel.org Signed-off-by: Vinod Koul --- drivers/dma/hisi_dma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c index 28bf818f9aa6..c751a2e49e6d 100644 --- a/drivers/dma/hisi_dma.c +++ b/drivers/dma/hisi_dma.c @@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id) hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num), GFP_KERNEL); if (!hdma_dev) - return -EINVAL; + return -ENOMEM; hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2]; hdma_dev->pdev = pdev; From aa99c4d1d63bbc26a5fc4c667d89b2595743c19d Mon Sep 17 00:00:00 2001 From: Alex Bereza Date: Thu, 2 Apr 2026 09:46:22 +0200 Subject: [PATCH 21/73] dmaengine: xilinx_dma: Fix CPU stall in xilinx_dma_poll_timeout Currently when calling xilinx_dma_poll_timeout with delay_us=0 and a condition that is never fulfilled, the CPU busy-waits for prolonged time and the timeout triggers only with a massive delay causing a CPU stall. This happens due to a huge underestimation of wall clock time in poll_timeout_us_atomic. Commit 7349a69cf312 ("iopoll: Do not use timekeeping in read_poll_timeout_atomic()") changed the behavior to no longer use ktime_get at the expense of underestimation of wall clock time which appears to be very large for delay_us=0. Instead of timing out after approximately XILINX_DMA_LOOP_COUNT microseconds, the timeout takes XILINX_DMA_LOOP_COUNT * 1000 * (time that the overhead of the for loop in poll_timeout_us_atomic takes) which is in the range of several minutes for XILINX_DMA_LOOP_COUNT=1000000. Fix this by using a non-zero value for delay_us. Use delay_us=10 to keep the delay in the hot path of starting DMA transfers minimal but still avoid CPU stalls in case of unexpected hardware failures. One-off measurement with delay_us=0 causes the cpu to busy wait around 7 minutes in the timeout case. After applying this patch with delay_us=10 the measured timeout was 1053428 microseconds which is roughly equivalent to the expected 1000000 microseconds specified in XILINX_DMA_LOOP_COUNT. Add a constant XILINX_DMA_POLL_DELAY_US for delay_us value. Fixes: 9495f2648287 ("dmaengine: xilinx_vdma: Use readl_poll_timeout instead of do while loop's") Fixes: 7349a69cf312 ("iopoll: Do not use timekeeping in read_poll_timeout_atomic()") Reviewed-by: Suraj Gupta Reviewed-by: Frank Li Signed-off-by: Alex Bereza Link: https://patch.msgid.link/20260402-fix-atomic-poll-timeout-regression-v4-1-f30d6a6c13cb@bereza.email Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 829601d8a16f..ddf9635809d3 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -167,6 +167,8 @@ /* Delay loop counter to prevent hardware failure */ #define XILINX_DMA_LOOP_COUNT 1000000 +/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ +#define XILINX_DMA_POLL_DELAY_US 10 /* AXI DMA Specific Registers/Offsets */ #define XILINX_DMA_REG_SRCDSTADDR 0x18 @@ -1324,7 +1326,8 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) /* Wait for the hardware to halt */ return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_HALTED, 0, + val & XILINX_DMA_DMASR_HALTED, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1339,7 +1342,8 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) u32 val; return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - val & XILINX_DMA_DMASR_IDLE, 0, + val & XILINX_DMA_DMASR_IDLE, + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); } @@ -1356,7 +1360,8 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) /* Wait for the hardware to start */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, - !(val & XILINX_DMA_DMASR_HALTED), 0, + !(val & XILINX_DMA_DMASR_HALTED), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { @@ -1800,7 +1805,8 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan) /* Wait for the hardware to finish reset */ err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, - !(tmp & XILINX_DMA_DMACR_RESET), 0, + !(tmp & XILINX_DMA_DMACR_RESET), + XILINX_DMA_POLL_DELAY_US, XILINX_DMA_LOOP_COUNT); if (err) { From a6404c7291bcc10114e72e9d0226709ec9d05d5a Mon Sep 17 00:00:00 2001 From: Alex Bereza Date: Thu, 2 Apr 2026 09:46:23 +0200 Subject: [PATCH 22/73] dmaengine: xilinx_dma: Rename XILINX_DMA_LOOP_COUNT Rename XILINX_DMA_LOOP_COUNT to XILINX_DMA_POLL_TIMEOUT_US because it is a timeout value, not a loop count for polling register in microseconds. No functional changes. Reviewed-by: Suraj Gupta Signed-off-by: Alex Bereza Link: https://patch.msgid.link/20260402-fix-atomic-poll-timeout-regression-v4-2-f30d6a6c13cb@bereza.email Signed-off-by: Vinod Koul --- drivers/dma/xilinx/xilinx_dma.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index ddf9635809d3..98b41b8f8915 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -165,8 +165,8 @@ #define XILINX_DMA_FLUSH_MM2S 2 #define XILINX_DMA_FLUSH_BOTH 1 -/* Delay loop counter to prevent hardware failure */ -#define XILINX_DMA_LOOP_COUNT 1000000 +/* Timeout for polling various registers */ +#define XILINX_DMA_POLL_TIMEOUT_US 1000000 /* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */ #define XILINX_DMA_POLL_DELAY_US 10 @@ -1328,7 +1328,7 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan) return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, val & XILINX_DMA_DMASR_HALTED, XILINX_DMA_POLL_DELAY_US, - XILINX_DMA_LOOP_COUNT); + XILINX_DMA_POLL_TIMEOUT_US); } /** @@ -1344,7 +1344,7 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan) return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, val & XILINX_DMA_DMASR_IDLE, XILINX_DMA_POLL_DELAY_US, - XILINX_DMA_LOOP_COUNT); + XILINX_DMA_POLL_TIMEOUT_US); } /** @@ -1362,7 +1362,7 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan) err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val, !(val & XILINX_DMA_DMASR_HALTED), XILINX_DMA_POLL_DELAY_US, - XILINX_DMA_LOOP_COUNT); + XILINX_DMA_POLL_TIMEOUT_US); if (err) { dev_err(chan->dev, "Cannot start channel %p: %x\n", @@ -1807,7 +1807,7 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan) err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp, !(tmp & XILINX_DMA_DMACR_RESET), XILINX_DMA_POLL_DELAY_US, - XILINX_DMA_LOOP_COUNT); + XILINX_DMA_POLL_TIMEOUT_US); if (err) { dev_err(chan->dev, "reset timeout, cr %x, sr %x\n", From 89aba9c39bdda8a973a6ffed7c9e93321edcfc16 Mon Sep 17 00:00:00 2001 From: Ben Dooks Date: Wed, 17 Jun 2026 09:49:43 +0100 Subject: [PATCH 23/73] dmaengine: dw-axi-dmac: fix __le32 on set of CH_CTL_H_LLI_VALID When writing the lli->ctl_hi, this is an __le32 type so the value being orred should be convered to __le32 by cpu_to_le32. Fixes 1deb96c0fa58a ("dmaegine: dw-axi-dmac: Support device_prep_dma_cyclic()") -- Note, the call to axi_chan_irq_clear() is passing lli->status_lo through which is also an __le32 but it does not seem to be set anywhere. Is this also a bug? Signed-off-by: Ben Dooks Link: https://patch.msgid.link/20260617084944.705266-1-ben.dooks@codethink.co.uk Signed-off-by: Vinod Koul --- drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index bcefaff03b5c..eebed2474210 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -1126,7 +1126,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) hw_desc = &desc->hw_desc[i]; if (hw_desc->llp == llp) { axi_chan_irq_clear(chan, hw_desc->lli->status_lo); - hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID; + hw_desc->lli->ctl_hi |= cpu_to_le32(CH_CTL_H_LLI_VALID); desc->completed_blocks = i; if (((hw_desc->len * (i + 1)) % desc->period_len) == 0) From 0d3e3376b289cdaff5b3b6c1581999926ff1000f Mon Sep 17 00:00:00 2001 From: Pengpeng Hou Date: Mon, 15 Jun 2026 17:16:45 +0800 Subject: [PATCH 24/73] dmaengine: pl330: remove debugfs file on teardown init_pl330_debugfs() creates a debugfs file with struct pl330_dmac as private data. pl330_remove() then unregisters the DMA device and frees the PL330 channel/thread state without removing that file. Keep the debugfs dentry and remove it before tearing down the DMAC state used by the debugfs show callback. Signed-off-by: Pengpeng Hou Link: https://patch.msgid.link/20260615091645.28878-1-pengpeng@iscas.ac.cn Signed-off-by: Vinod Koul --- drivers/dma/pl330.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/drivers/dma/pl330.c b/drivers/dma/pl330.c index 25ba84b18704..6214d9000db8 100644 --- a/drivers/dma/pl330.c +++ b/drivers/dma/pl330.c @@ -502,6 +502,7 @@ struct pl330_dmac { struct dma_pl330_chan *peripherals; /* keep at end */ int quirks; + struct dentry *dbgfs; struct reset_control *rstc; struct reset_control *rstc_ocp; }; @@ -2952,14 +2953,24 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs); static inline void init_pl330_debugfs(struct pl330_dmac *pl330) { - debugfs_create_file(dev_name(pl330->ddma.dev), - S_IFREG | 0444, NULL, pl330, - &pl330_debugfs_fops); + pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev), + S_IFREG | 0444, NULL, pl330, + &pl330_debugfs_fops); +} + +static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330) +{ + debugfs_remove(pl330->dbgfs); + pl330->dbgfs = NULL; } #else static inline void init_pl330_debugfs(struct pl330_dmac *pl330) { } + +static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330) +{ +} #endif /* @@ -3204,6 +3215,8 @@ static void pl330_remove(struct amba_device *adev) struct dma_pl330_chan *pch, *_p; int i, irq; + deinit_pl330_debugfs(pl330); + pm_runtime_get_noresume(pl330->ddma.dev); if (adev->dev.of_node) From 49eee9b6c2b5646bd0cfd891bb7c0a218301aa7c Mon Sep 17 00:00:00 2001 From: Adrian Ng Ho Yin Date: Mon, 6 Jul 2026 10:23:11 +0800 Subject: [PATCH 25/73] MAINTAINERS: replace maintainer for Altera mSGDMA driver Olivier Dautricourt has stepped down as maintainer of the Altera msgDMA driver as he no longer has access to the hardware. Replace him with Adrian Ng Ho Yin as the new maintainer and update the status from "Odd Fixes" to "Maintained". Signed-off-by: Adrian Ng Ho Yin Acked-by: Krzysztof Kozlowski Link: https://patch.msgid.link/addaf51275355667045ec300fc8d725e2e273807.1782911845.git.adrian.ho.yin.ng@altera.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/altr,msgdma.yaml | 2 +- MAINTAINERS | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Documentation/devicetree/bindings/dma/altr,msgdma.yaml b/Documentation/devicetree/bindings/dma/altr,msgdma.yaml index 391bf5838602..bea302b89453 100644 --- a/Documentation/devicetree/bindings/dma/altr,msgdma.yaml +++ b/Documentation/devicetree/bindings/dma/altr,msgdma.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Altera mSGDMA IP core maintainers: - - Olivier Dautricourt + - Adrian Ng Ho Yin description: | Altera / Intel modular Scatter-Gather Direct Memory Access (mSGDMA) diff --git a/MAINTAINERS b/MAINTAINERS index 15011f5752a9..4d294df29a0d 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -952,10 +952,10 @@ S: Maintained F: drivers/mailbox/mailbox-altera.c ALTERA MSGDMA IP CORE DRIVER -M: Olivier Dautricourt +M: Adrian Ng Ho Yin R: Stefan Roese L: dmaengine@vger.kernel.org -S: Odd Fixes +S: Maintained F: Documentation/devicetree/bindings/dma/altr,msgdma.yaml F: drivers/dma/altera-msgdma.c From 242a57d2d0b4de346cc33c385fec4f901c476517 Mon Sep 17 00:00:00 2001 From: Kuldeep Singh Date: Tue, 14 Jul 2026 15:35:16 +0530 Subject: [PATCH 26/73] dt-bindings: dma: qcom,bam-dma: Increase iommus maxItems to 7 Qualcomm Shikra platform describes the BAM DMA node with 7 iommus entries. The current schema limit to 6, so update the binding to allow up to 7 entries. Acked-by: Krzysztof Kozlowski Signed-off-by: Kuldeep Singh Link: https://patch.msgid.link/20260714-b4-shikra_crypto_changse-v4-5-06a4ea97c209@oss.qualcomm.com Signed-off-by: Vinod Koul --- Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml index 0923fb189ada..e72adc172af1 100644 --- a/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml +++ b/Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml @@ -48,7 +48,7 @@ properties: iommus: minItems: 1 - maxItems: 6 + maxItems: 7 num-channels: $ref: /schemas/types.yaml#/definitions/uint32 From bf1af4dfdc017dfe989c0dbcf0e608dc95f1d2cb Mon Sep 17 00:00:00 2001 From: Griffin Kroah-Hartman Date: Mon, 6 Jul 2026 16:57:06 +0200 Subject: [PATCH 27/73] dmaengine: fsl-edma: Add error handling for devm_kasprintf Add error handling statement to fls_edma3_irq_init() for the devm_kasprintf call. Assisted-by: gkh_clanker_2000 Cc: stable Cc: Frank Li Cc: Vinod Koul Cc: imx@lists.linux.dev Signed-off-by: Griffin Kroah-Hartman Signed-off-by: Greg Kroah-Hartman Reviewed-by: Frank Li Link: https://patch.msgid.link/2026070605-frying-fling-b9c5@gregkh Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/fsl-edma-main.c b/drivers/dma/fsl-edma-main.c index 36155ab1602a..d9fb717b5b53 100644 --- a/drivers/dma/fsl-edma-main.c +++ b/drivers/dma/fsl-edma-main.c @@ -414,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err", dev_name(&pdev->dev)); + if (!errirq_name) + return -ENOMEM; ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared, 0, errirq_name, fsl_edma); From 99c46385edb155cfd2919b4a44d97f9ea811639e Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:19 -0400 Subject: [PATCH 28/73] dmaengine: dw-edma: Move control field update of DMA link to the last step The control field in a DMA link list entry must be updated as the final step because it includes the CB bit, which indicates whether the entry is ready. Add dma_wmb() to ensure the correct memory write ordering. Currently the driver does not update DMA link entries while the DMA is running, so no visible failure occurs. However, fixing the ordering now prepares the driver for supporting link entry updates during DMA operation. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-1-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-v0-core.c | 10 ++++++---- drivers/dma/dw-edma/dw-hdma-v0-core.c | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index cfdd6463252e..ee5c3c317557 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -284,17 +284,18 @@ static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; - lli->control = control; lli->transfer_size = size; lli->sar.reg = sar; lli->dar.reg = dar; + dma_wmb(); + lli->control = control; } else { struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; - writel(control, &lli->control); writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); writeq(dar, &lli->dar.reg); + writel(control, &lli->control); } } @@ -306,13 +307,14 @@ static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; - llp->control = control; llp->llp.reg = pointer; + dma_wmb(); + llp->control = control; } else { struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; - writel(control, &llp->control); writeq(pointer, &llp->llp.reg); + writel(control, &llp->control); } } diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 632abb8b481c..1201f1ab5f35 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -160,17 +160,18 @@ static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; - lli->control = control; lli->transfer_size = size; lli->sar.reg = sar; lli->dar.reg = dar; + dma_wmb(); + lli->control = control; } else { struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; - writel(control, &lli->control); writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); writeq(dar, &lli->dar.reg); + writel(control, &lli->control); } } @@ -182,13 +183,14 @@ static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; - llp->control = control; llp->llp.reg = pointer; + dma_wmb(); + llp->control = control; } else { struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; - writel(control, &llp->control); writeq(pointer, &llp->llp.reg); + writel(control, &llp->control); } } From 07680003068a8008b4585423d533ab9e86a8a4d3 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:20 -0400 Subject: [PATCH 29/73] dmaengine: dw-edma: Add xfer_sz field to struct dw_edma_chunk Reusing ll_region.sz as the transfer size is misleading because ll_region.sz represents the memory size of the EDMA link list, not the amount of data to be transferred. Add a new xfer_sz field to explicitly indicate the total transfer size of a chunk. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-2-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 4 ++-- drivers/dma/dw-edma/dw-edma-core.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 1fec1b52e3d4..53469c8c8b82 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -192,7 +192,7 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) return 0; dw_edma_core_start(dw, child, !desc->xfer_sz); - desc->xfer_sz += child->ll_region.sz; + desc->xfer_sz += child->xfer_sz; dw_edma_free_burst(child); list_del(&child->list); kfree(child); @@ -527,7 +527,7 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, else if (xfer->type == EDMA_XFER_INTERLEAVED) burst->sz = xfer->xfer.il->sgl[i % fsz].size; - chunk->ll_region.sz += burst->sz; + chunk->xfer_sz += burst->sz; desc->alloc_sz += burst->sz; if (dir == DMA_DEV_TO_MEM) { diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 6474cacf7195..db5f45bf048c 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -57,6 +57,7 @@ struct dw_edma_chunk { u32 bursts_alloc; u8 cb; + u32 xfer_sz; struct dw_edma_region ll_region; /* Linked list */ }; From 956028c5dde9fd1a8a462db53609c1c547cafb5c Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:21 -0400 Subject: [PATCH 30/73] dmaengine: dw-edma: Move ll_region from struct dw_edma_chunk to struct dw_edma_chan ll_region is identical for all chunks belonging to the same DMA channel, so there is no need to copy it into each chunk. Move ll_region to struct dw_edma_chan to avoid redundant copies. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-3-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 15 ++++----------- drivers/dma/dw-edma/dw-edma-core.h | 2 +- drivers/dma/dw-edma/dw-edma-v0-core.c | 18 ++++++++++-------- drivers/dma/dw-edma/dw-hdma-v0-core.c | 18 ++++++++++-------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 53469c8c8b82..2652ad8e7a8f 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -64,7 +64,6 @@ static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) { - struct dw_edma_chip *chip = desc->chan->dw->chip; struct dw_edma_chan *chan = desc->chan; struct dw_edma_chunk *chunk; @@ -81,13 +80,6 @@ static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) * - Even chunks originate CB equal to 1 */ chunk->cb = !(desc->chunks_alloc % 2); - if (chan->dir == EDMA_DIR_WRITE) { - chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr; - chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr; - } else { - chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr; - chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr; - } if (desc->chunk) { /* Create and add new element into the linked list */ @@ -925,10 +917,11 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan->status = EDMA_ST_IDLE; if (chan->dir == EDMA_DIR_WRITE) - chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ); + chan->ll_region = chip->ll_region_wr[chan->id]; else - chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ); - chan->ll_max -= 1; + chan->ll_region = chip->ll_region_rd[chan->id]; + + chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1; dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n", str_write_read(chan->dir == EDMA_DIR_WRITE), diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index db5f45bf048c..b96089baf0f9 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -58,7 +58,6 @@ struct dw_edma_chunk { u8 cb; u32 xfer_sz; - struct dw_edma_region ll_region; /* Linked list */ }; struct dw_edma_desc { @@ -79,6 +78,7 @@ struct dw_edma_chan { enum dw_edma_dir dir; u32 ll_max; + struct dw_edma_region ll_region; /* Linked list */ struct msi_msg msi; diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index ee5c3c317557..51e50f1fdcac 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -280,9 +280,10 @@ static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); + struct dw_edma_chan *chan = chunk->chan; if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; lli->transfer_size = size; lli->sar.reg = sar; @@ -290,7 +291,7 @@ static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, dma_wmb(); lli->control = control; } else { - struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); @@ -303,15 +304,16 @@ static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); + struct dw_edma_chan *chan = chunk->chan; if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + struct dw_edma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; llp->llp.reg = pointer; dma_wmb(); llp->control = control; } else { - struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + struct dw_edma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs; writeq(pointer, &llp->llp.reg); writel(control, &llp->control); @@ -345,7 +347,7 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) if (!chunk->cb) control |= DW_EDMA_V0_CB; - dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); + dw_edma_v0_write_ll_link(chunk, i, control, chan->ll_region.paddr); } static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) @@ -359,7 +361,7 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * last MWr TLP is completed */ if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->ll_region.vaddr.io); + readl(chunk->chan->ll_region.vaddr.io); } static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) @@ -430,9 +432,9 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) /* Linked list */ /* llp is not aligned on 64bit -> keep 32bit accesses */ SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chunk->ll_region.paddr)); + lower_32_bits(chan->ll_region.paddr)); SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chunk->ll_region.paddr)); + upper_32_bits(chan->ll_region.paddr)); } dw_edma_v0_sync_ll_data(chunk); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 1201f1ab5f35..20089d57f8ab 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -156,9 +156,10 @@ static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); + struct dw_edma_chan *chan = chunk->chan; if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs; + struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; lli->transfer_size = size; lli->sar.reg = sar; @@ -166,7 +167,7 @@ static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, dma_wmb(); lli->control = control; } else { - struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs; + struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs; writel(size, &lli->transfer_size); writeq(sar, &lli->sar.reg); @@ -179,15 +180,16 @@ static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); + struct dw_edma_chan *chan = chunk->chan; if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { - struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs; + struct dw_hdma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; llp->llp.reg = pointer; dma_wmb(); llp->control = control; } else { - struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs; + struct dw_hdma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs; writeq(pointer, &llp->llp.reg); writel(control, &llp->control); @@ -210,7 +212,7 @@ static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) if (!chunk->cb) control |= DW_HDMA_V0_CB; - dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr); + dw_hdma_v0_write_ll_link(chunk, i, control, chunk->chan->ll_region.paddr); } static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) @@ -224,7 +226,7 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * last MWr TLP is completed */ if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->ll_region.vaddr.io); + readl(chunk->chan->ll_region.vaddr.io); } static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) @@ -251,9 +253,9 @@ static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) /* Linked list */ /* llp is not aligned on 64bit -> keep 32bit accesses */ SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chunk->ll_region.paddr)); + lower_32_bits(chan->ll_region.paddr)); SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chunk->ll_region.paddr)); + upper_32_bits(chan->ll_region.paddr)); /* Set consumer cycle */ SET_CH_32(dw, chan->dir, chan->id, cycle_sync, HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); From be35502a14726e916ac41545840c5d682c5d54e6 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:22 -0400 Subject: [PATCH 31/73] dmaengine: dw-edma: Pass down dw_edma_chan to reduce one level of indirection Some helper functions do not use any information from dw_edma_chunk, so passing a dw_edma_chan pointer directly avoids an unnecessary level of pointer dereferencing and simplifies data access. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-4-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-v0-core.c | 22 ++++++++++------------ drivers/dma/dw-edma/dw-hdma-v0-core.c | 23 +++++++++++------------ 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 51e50f1fdcac..c341aa534341 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -276,13 +276,12 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, return ret; } -static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, +static void dw_edma_v0_write_ll_data(struct dw_edma_chan *chan, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); - struct dw_edma_chan *chan = chunk->chan; - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; lli->transfer_size = size; @@ -300,13 +299,12 @@ static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, } } -static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk, +static void dw_edma_v0_write_ll_link(struct dw_edma_chan *chan, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli); - struct dw_edma_chan *chan = chunk->chan; - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_edma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; llp->llp.reg = pointer; @@ -339,7 +337,7 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) control |= DW_EDMA_V0_RIE; } - dw_edma_v0_write_ll_data(chunk, i++, control, child->sz, + dw_edma_v0_write_ll_data(chan, i++, control, child->sz, child->sar, child->dar); } @@ -347,10 +345,10 @@ static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) if (!chunk->cb) control |= DW_EDMA_V0_CB; - dw_edma_v0_write_ll_link(chunk, i, control, chan->ll_region.paddr); + dw_edma_v0_write_ll_link(chan, i, control, chan->ll_region.paddr); } -static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* * In case of remote eDMA engine setup, the DW PCIe RP/EP internal @@ -360,8 +358,8 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * LL memory in a hope that the MRd TLP will return only after the * last MWr TLP is completed */ - if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->chan->ll_region.vaddr.io); + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chan->ll_region.vaddr.io); } static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) @@ -437,7 +435,7 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) upper_32_bits(chan->ll_region.paddr)); } - dw_edma_v0_sync_ll_data(chunk); + dw_edma_v0_sync_ll_data(chan); /* Doorbell */ SET_RW_32(dw, chan->dir, doorbell, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 20089d57f8ab..156b1cc22509 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -152,13 +152,12 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, return ret; } -static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, +static void dw_hdma_v0_write_ll_data(struct dw_edma_chan *chan, int i, u32 control, u32 size, u64 sar, u64 dar) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); - struct dw_edma_chan *chan = chunk->chan; - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs; lli->transfer_size = size; @@ -176,13 +175,12 @@ static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i, } } -static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, +static void dw_hdma_v0_write_ll_link(struct dw_edma_chan *chan, int i, u32 control, u64 pointer) { ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli); - struct dw_edma_chan *chan = chunk->chan; - if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { + if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) { struct dw_hdma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs; llp->llp.reg = pointer; @@ -198,6 +196,7 @@ static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk, static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { + struct dw_edma_chan *chan = chunk->chan; struct dw_edma_burst *child; u32 control = 0, i = 0; @@ -205,17 +204,17 @@ static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) control = DW_HDMA_V0_CB; list_for_each_entry(child, &chunk->burst->list, list) - dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz, + dw_hdma_v0_write_ll_data(chan, i++, control, child->sz, child->sar, child->dar); control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; if (!chunk->cb) control |= DW_HDMA_V0_CB; - dw_hdma_v0_write_ll_link(chunk, i, control, chunk->chan->ll_region.paddr); + dw_hdma_v0_write_ll_link(chan, i, control, chunk->chan->ll_region.paddr); } -static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) +static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* * In case of remote HDMA engine setup, the DW PCIe RP/EP internal @@ -225,8 +224,8 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk) * LL memory in a hope that the MRd TLP will return only after the * last MWr TLP is completed */ - if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - readl(chunk->chan->ll_region.vaddr.io); + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + readl(chan->ll_region.vaddr.io); } static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) @@ -261,7 +260,7 @@ static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } - dw_hdma_v0_sync_ll_data(chunk); + dw_hdma_v0_sync_ll_data(chan); /* Doorbell */ SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); From df9c5515501c8860a228ade47b93aff5b66544ee Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:23 -0400 Subject: [PATCH 32/73] dmaengine: dw-edma: Add helper dw_(edma|hdma)_v0_core_ch_enable() Move the channel-enable logic into a new helper function, dw_(edma|hdma)_v0_core_ch_enable(), in preparation for supporting dynamic link entry additions. No functional changes. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-5-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-v0-core.c | 128 +++++++++++++------------- drivers/dma/dw-edma/dw-hdma-v0-core.c | 54 ++++++----- 2 files changed, 93 insertions(+), 89 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index c341aa534341..8d38867cd998 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -318,6 +318,67 @@ static void dw_edma_v0_write_ll_link(struct dw_edma_chan *chan, } } +static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + unsigned long flags; + u32 tmp; + + /* Enable engine */ + SET_RW_32(dw, chan->dir, engine_en, BIT(0)); + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { + switch (chan->id) { + case 0: + SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, BIT(0)); + break; + case 1: + SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, BIT(0)); + break; + case 2: + SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, BIT(0)); + break; + case 3: + SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, BIT(0)); + break; + case 4: + SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, BIT(0)); + break; + case 5: + SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, BIT(0)); + break; + case 6: + SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, BIT(0)); + break; + case 7: + SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, BIT(0)); + break; + } + } + /* Interrupt unmask - done, abort */ + raw_spin_lock_irqsave(&dw->lock, flags); + + tmp = GET_RW_32(dw, chan->dir, int_mask); + tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + SET_RW_32(dw, chan->dir, int_mask, tmp); + /* Linked list error */ + tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); + tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); + SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); + + raw_spin_unlock_irqrestore(&dw->lock, flags); + + /* Channel control */ + SET_CH_32(dw, chan->dir, chan->id, ch_control1, + (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); + /* Linked list */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chan->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chan->ll_region.paddr)); +} + static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { struct dw_edma_burst *child; @@ -366,74 +427,11 @@ static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; - unsigned long flags; - u32 tmp; dw_edma_v0_core_write_chunk(chunk); - if (first) { - /* Enable engine */ - SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { - switch (chan->id) { - case 0: - SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, - BIT(0)); - break; - case 1: - SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, - BIT(0)); - break; - case 2: - SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, - BIT(0)); - break; - case 3: - SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, - BIT(0)); - break; - case 4: - SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, - BIT(0)); - break; - case 5: - SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, - BIT(0)); - break; - case 6: - SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, - BIT(0)); - break; - case 7: - SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, - BIT(0)); - break; - } - } - /* Interrupt unmask - done, abort */ - raw_spin_lock_irqsave(&dw->lock, flags); - - tmp = GET_RW_32(dw, chan->dir, int_mask); - tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); - tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); - SET_RW_32(dw, chan->dir, int_mask, tmp); - /* Linked list error */ - tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); - tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id)); - SET_RW_32(dw, chan->dir, linked_list_err_en, tmp); - - raw_spin_unlock_irqrestore(&dw->lock, flags); - - /* Channel control */ - SET_CH_32(dw, chan->dir, chan->id, ch_control1, - (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); - /* Linked list */ - /* llp is not aligned on 64bit -> keep 32bit accesses */ - SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chan->ll_region.paddr)); - SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chan->ll_region.paddr)); - } + if (first) + dw_edma_v0_core_ch_enable(chan); dw_edma_v0_sync_ll_data(chan); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 156b1cc22509..31bbdc6a4064 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -194,6 +194,34 @@ static void dw_hdma_v0_write_ll_link(struct dw_edma_chan *chan, } } +static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + u32 tmp; + + /* Enable engine */ + SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); + /* Interrupt unmask - stop, abort */ + tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); + tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + /* Interrupt enable - stop, abort */ + tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; + if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; + SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); + /* Channel control */ + SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); + /* Linked list */ + /* llp is not aligned on 64bit -> keep 32bit accesses */ + SET_CH_32(dw, chan->dir, chan->id, llp.lsb, + lower_32_bits(chan->ll_region.paddr)); + SET_CH_32(dw, chan->dir, chan->id, llp.msb, + upper_32_bits(chan->ll_region.paddr)); + /* Set consumer cycle */ + SET_CH_32(dw, chan->dir, chan->id, cycle_sync, + HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); +} + static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) { struct dw_edma_chan *chan = chunk->chan; @@ -232,33 +260,11 @@ static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; - u32 tmp; dw_hdma_v0_core_write_chunk(chunk); - if (first) { - /* Enable engine */ - SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); - /* Interrupt unmask - stop, abort */ - tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); - tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - /* Interrupt enable - stop, abort */ - tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; - if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; - SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); - /* Channel control */ - SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); - /* Linked list */ - /* llp is not aligned on 64bit -> keep 32bit accesses */ - SET_CH_32(dw, chan->dir, chan->id, llp.lsb, - lower_32_bits(chan->ll_region.paddr)); - SET_CH_32(dw, chan->dir, chan->id, llp.msb, - upper_32_bits(chan->ll_region.paddr)); - /* Set consumer cycle */ - SET_CH_32(dw, chan->dir, chan->id, cycle_sync, - HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); - } + if (first) + dw_hdma_v0_core_ch_enable(chan); dw_hdma_v0_sync_ll_data(chan); From 2cc7ba6b168f2d0bd79f47ccc14168352a3c6cd2 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:24 -0400 Subject: [PATCH 33/73] dmaengine: dw-edma: Add callbacks to fill link list entries Introduce four new callbacks to fill link list entries in preparation for replacing dw_(edma|hdma)_v0_core_start(). Filling link list entries is expected to become more complex, and without this abstraction both eDMA and HDMA paths would need to duplicate the same logic. Add fill-entry callbacks so the code can be shared cleanly between eDMA and HDMA implementations. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-6-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.h | 29 +++++++++++++++++ drivers/dma/dw-edma/dw-edma-v0-core.c | 46 +++++++++++++++++++++++++++ drivers/dma/dw-edma/dw-hdma-v0-core.c | 38 ++++++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index b96089baf0f9..bab4d49c92fe 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -126,6 +126,12 @@ struct dw_edma_core_ops { irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort); void (*start)(struct dw_edma_chunk *chunk, bool first); + void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq); + void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr); + void (*ch_doorbell)(struct dw_edma_chan *chan); + void (*ch_enable)(struct dw_edma_chan *chan); + void (*ch_config)(struct dw_edma_chan *chan); void (*debugfs_on)(struct dw_edma *dw); void (*ack_emulated_irq)(struct dw_edma *dw); @@ -204,6 +210,29 @@ void dw_edma_core_ch_config(struct dw_edma_chan *chan) chan->dw->core->ch_config(chan); } +static inline void +dw_edma_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) +{ + chan->dw->core->ll_data(chan, burst, idx, cb, irq); +} + +static inline void +dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + chan->dw->core->ll_link(chan, idx, cb, addr); +} + +static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan) +{ + chan->dw->core->ch_doorbell(chan); +} + +static inline void dw_edma_core_ch_enable(struct dw_edma_chan *chan) +{ + chan->dw->core->ch_enable(chan); +} + static inline void dw_edma_core_debugfs_on(struct dw_edma *dw) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 8d38867cd998..c0746e535141 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -509,6 +509,48 @@ static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) } } +static void +dw_edma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) +{ + u32 control = 0; + + if (cb) + control |= DW_EDMA_V0_CB; + + if (irq) { + control |= DW_EDMA_V0_LIE; + + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + control |= DW_EDMA_V0_RIE; + } + + dw_edma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar, + burst->dar); +} + +static void +dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + u32 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; + + if (!cb) + control |= DW_EDMA_V0_CB; + + dw_edma_v0_write_ll_link(chan, idx, control, addr); +} + +static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + dw_edma_v0_sync_ll_data(chan); + + /* Doorbell */ + SET_RW_32(dw, chan->dir, doorbell, + FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); +} + /* eDMA debugfs callbacks */ static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw) { @@ -540,6 +582,10 @@ static const struct dw_edma_core_ops dw_edma_v0_core = { .ch_status = dw_edma_v0_core_ch_status, .handle_int = dw_edma_v0_core_handle_int, .start = dw_edma_v0_core_start, + .ll_data = dw_edma_v0_core_ll_data, + .ll_link = dw_edma_v0_core_ll_link, + .ch_doorbell = dw_edma_v0_core_ch_doorbell, + .ch_enable = dw_edma_v0_core_ch_enable, .ch_config = dw_edma_v0_core_ch_config, .debugfs_on = dw_edma_v0_core_debugfs_on, .ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 31bbdc6a4064..16fe3ef43948 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -348,6 +348,40 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data); } +static void +dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, + u32 idx, bool cb, bool irq) +{ + u32 control = 0; + + if (cb) + control |= DW_HDMA_V0_CB; + + dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar, + burst->dar); +} + +static void +dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr) +{ + u32 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; + + if (!cb) + control |= DW_HDMA_V0_CB; + + dw_hdma_v0_write_ll_link(chan, idx, control, addr); +} + +static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + dw_hdma_v0_sync_ll_data(chan); + + /* Doorbell */ + SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); +} + /* HDMA debugfs callbacks */ static void dw_hdma_v0_core_debugfs_on(struct dw_edma *dw) { @@ -366,6 +400,10 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = { .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int, .start = dw_hdma_v0_core_start, + .ll_data = dw_hdma_v0_core_ll_data, + .ll_link = dw_hdma_v0_core_ll_link, + .ch_doorbell = dw_hdma_v0_core_ch_doorbell, + .ch_enable = dw_hdma_v0_core_ch_enable, .ch_config = dw_hdma_v0_core_ch_config, .debugfs_on = dw_hdma_v0_core_debugfs_on, .db_offset = dw_hdma_v0_core_db_offset, From de60121a08799c4db503d44a9028856976f1cfff Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:25 -0400 Subject: [PATCH 34/73] dmaengine: dw-edma: Add non_ll_start() callback Add a non_ll_start() callback and move the common non-linked-list channel handling into the EDMA core so it can be shared by both the EDMA and HDMA. Prepare for the upcoming reorganization of the burst and chunk structures. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-7-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.h | 12 +++++++++++- drivers/dma/dw-edma/dw-hdma-v0-core.c | 23 ++++------------------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index bab4d49c92fe..e18d6e827c2c 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -126,6 +126,7 @@ struct dw_edma_core_ops { irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort); void (*start)(struct dw_edma_chunk *chunk, bool first); + void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child); void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst, u32 idx, bool cb, bool irq); void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr); @@ -201,7 +202,16 @@ dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, static inline void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first) { - dw->core->start(chunk, first); + if (chunk->chan->non_ll) { + struct dw_edma_burst *child; + + child = list_first_entry_or_null(&chunk->burst->list, + struct dw_edma_burst, list); + if (child) + dw->core->non_ll_start(chunk->chan, child); + } else { + dw->core->start(chunk, first); + } } static inline diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 16fe3ef43948..641a513bc52e 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -272,18 +272,12 @@ static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); } -static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk) +static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan, + struct dw_edma_burst *child) { - struct dw_edma_chan *chan = chunk->chan; struct dw_edma *dw = chan->dw; - struct dw_edma_burst *child; u32 val; - child = list_first_entry_or_null(&chunk->burst->list, - struct dw_edma_burst, list); - if (!child) - return; - SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN); /* Source address */ @@ -324,16 +318,6 @@ static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk) HDMA_V0_DOORBELL_START); } -static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first) -{ - struct dw_edma_chan *chan = chunk->chan; - - if (chan->non_ll) - dw_hdma_v0_core_non_ll_start(chunk); - else - dw_hdma_v0_core_ll_start(chunk, first); -} - static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; @@ -399,7 +383,8 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = { .ch_count = dw_hdma_v0_core_ch_count, .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int, - .start = dw_hdma_v0_core_start, + .start = dw_hdma_v0_core_ll_start, + .non_ll_start = dw_hdma_v0_core_non_ll_start, .ll_data = dw_hdma_v0_core_ll_data, .ll_link = dw_hdma_v0_core_ll_link, .ch_doorbell = dw_hdma_v0_core_ch_doorbell, From 0d7a2719d85f9940b05eeed18e5f398b0b3219b1 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:26 -0400 Subject: [PATCH 35/73] dmaengine: dw-edma: Use common dw_edma_core_start() for both eDMA and HDMA Use common dw_edma_core_start() for both eDMA and HDMA. Remove .start() callback functions at eDMA and HDMA. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-8-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 32 ++++++++++++++++-- drivers/dma/dw-edma/dw-edma-core.h | 16 --------- drivers/dma/dw-edma/dw-edma-v0-core.c | 48 --------------------------- drivers/dma/dw-edma/dw-hdma-v0-core.c | 37 --------------------- 4 files changed, 30 insertions(+), 103 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 2652ad8e7a8f..f52d9fd18e57 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -163,9 +163,37 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc) dw_edma_free_desc(vd2dw_edma_desc(vdesc)); } +static void dw_edma_core_start(struct dw_edma_chunk *chunk, bool first) +{ + struct dw_edma_chan *chan = chunk->chan; + struct dw_edma_burst *child; + u32 i = 0; + int j; + + if (chan->non_ll) { + child = list_first_entry_or_null(&chunk->burst->list, + struct dw_edma_burst, list); + if (child) + chan->dw->core->non_ll_start(chunk->chan, child); + return; + } + + j = chunk->bursts_alloc; + list_for_each_entry(child, &chunk->burst->list, list) { + j--; + dw_edma_core_ll_data(chan, child, i++, chunk->cb, !j); + } + + dw_edma_core_ll_link(chan, i, chunk->cb, chan->ll_region.paddr); + + if (first) + dw_edma_core_ch_enable(chan); + + dw_edma_core_ch_doorbell(chan); +} + static int dw_edma_start_transfer(struct dw_edma_chan *chan) { - struct dw_edma *dw = chan->dw; struct dw_edma_chunk *child; struct dw_edma_desc *desc; struct virt_dma_desc *vd; @@ -183,7 +211,7 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) if (!child) return 0; - dw_edma_core_start(dw, child, !desc->xfer_sz); + dw_edma_core_start(child, !desc->xfer_sz); desc->xfer_sz += child->xfer_sz; dw_edma_free_burst(child); list_del(&child->list); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index e18d6e827c2c..27415f3a2d04 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -125,7 +125,6 @@ struct dw_edma_core_ops { enum dma_status (*ch_status)(struct dw_edma_chan *chan); irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort); - void (*start)(struct dw_edma_chunk *chunk, bool first); void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child); void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst, u32 idx, bool cb, bool irq); @@ -199,21 +198,6 @@ dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort); } -static inline -void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first) -{ - if (chunk->chan->non_ll) { - struct dw_edma_burst *child; - - child = list_first_entry_or_null(&chunk->burst->list, - struct dw_edma_burst, list); - if (child) - dw->core->non_ll_start(chunk->chan, child); - } else { - dw->core->start(chunk, first); - } -} - static inline void dw_edma_core_ch_config(struct dw_edma_chan *chan) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index c0746e535141..7b4933c66f9f 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -379,36 +379,6 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) upper_32_bits(chan->ll_region.paddr)); } -static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk) -{ - struct dw_edma_burst *child; - struct dw_edma_chan *chan = chunk->chan; - u32 control = 0, i = 0; - int j; - - if (chunk->cb) - control = DW_EDMA_V0_CB; - - j = chunk->bursts_alloc; - list_for_each_entry(child, &chunk->burst->list, list) { - j--; - if (!j) { - control |= DW_EDMA_V0_LIE; - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - control |= DW_EDMA_V0_RIE; - } - - dw_edma_v0_write_ll_data(chan, i++, control, child->sz, - child->sar, child->dar); - } - - control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB; - if (!chunk->cb) - control |= DW_EDMA_V0_CB; - - dw_edma_v0_write_ll_link(chan, i, control, chan->ll_region.paddr); -} - static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* @@ -423,23 +393,6 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan) readl(chan->ll_region.vaddr.io); } -static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first) -{ - struct dw_edma_chan *chan = chunk->chan; - struct dw_edma *dw = chan->dw; - - dw_edma_v0_core_write_chunk(chunk); - - if (first) - dw_edma_v0_core_ch_enable(chan); - - dw_edma_v0_sync_ll_data(chan); - - /* Doorbell */ - SET_RW_32(dw, chan->dir, doorbell, - FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id)); -} - static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan) { struct dw_edma *dw = chan->dw; @@ -581,7 +534,6 @@ static const struct dw_edma_core_ops dw_edma_v0_core = { .ch_count = dw_edma_v0_core_ch_count, .ch_status = dw_edma_v0_core_ch_status, .handle_int = dw_edma_v0_core_handle_int, - .start = dw_edma_v0_core_start, .ll_data = dw_edma_v0_core_ll_data, .ll_link = dw_edma_v0_core_ll_link, .ch_doorbell = dw_edma_v0_core_ch_doorbell, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 641a513bc52e..4bf5a441afbf 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -222,26 +222,6 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan) HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT); } -static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk) -{ - struct dw_edma_chan *chan = chunk->chan; - struct dw_edma_burst *child; - u32 control = 0, i = 0; - - if (chunk->cb) - control = DW_HDMA_V0_CB; - - list_for_each_entry(child, &chunk->burst->list, list) - dw_hdma_v0_write_ll_data(chan, i++, control, child->sz, - child->sar, child->dar); - - control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB; - if (!chunk->cb) - control |= DW_HDMA_V0_CB; - - dw_hdma_v0_write_ll_link(chan, i, control, chunk->chan->ll_region.paddr); -} - static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan) { /* @@ -256,22 +236,6 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan) readl(chan->ll_region.vaddr.io); } -static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first) -{ - struct dw_edma_chan *chan = chunk->chan; - struct dw_edma *dw = chan->dw; - - dw_hdma_v0_core_write_chunk(chunk); - - if (first) - dw_hdma_v0_core_ch_enable(chan); - - dw_hdma_v0_sync_ll_data(chan); - - /* Doorbell */ - SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START); -} - static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan, struct dw_edma_burst *child) { @@ -383,7 +347,6 @@ static const struct dw_edma_core_ops dw_hdma_v0_core = { .ch_count = dw_hdma_v0_core_ch_count, .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int, - .start = dw_hdma_v0_core_ll_start, .non_ll_start = dw_hdma_v0_core_non_ll_start, .ll_data = dw_hdma_v0_core_ll_data, .ll_link = dw_hdma_v0_core_ll_link, From 3f3fa81cd2ace2c95a0368f2c6e6d1a1d983281a Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:27 -0400 Subject: [PATCH 36/73] dmaengine: dw-edma: Use burst array instead of linked list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current descriptor layout is: struct dw_edma_desc *desc └─ chunk list └─ burst list Creating a DMA descriptor requires at least three kzalloc() calls because each burst is allocated as a linked-list node. Since the number of bursts is already known when the descriptor is created, a linked list is not necessary. Allocate a burst array when creating each chunk to simplify the code and eliminate one kzalloc() call. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-9-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 120 ++++++----------------------- drivers/dma/dw-edma/dw-edma-core.h | 9 +-- 2 files changed, 26 insertions(+), 103 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index f52d9fd18e57..c028011cc61c 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -40,38 +40,15 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) return cpu_addr; } -static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk) -{ - struct dw_edma_burst *burst; - - burst = kzalloc_obj(*burst, GFP_NOWAIT); - if (unlikely(!burst)) - return NULL; - - INIT_LIST_HEAD(&burst->list); - if (chunk->burst) { - /* Create and add new element into the linked list */ - chunk->bursts_alloc++; - list_add_tail(&burst->list, &chunk->burst->list); - } else { - /* List head */ - chunk->bursts_alloc = 0; - chunk->burst = burst; - } - - return burst; -} - -static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) +static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc, u32 nburst) { struct dw_edma_chan *chan = desc->chan; struct dw_edma_chunk *chunk; - chunk = kzalloc_obj(*chunk, GFP_NOWAIT); + chunk = kzalloc_flex(*chunk, burst, nburst, GFP_NOWAIT); if (unlikely(!chunk)) return NULL; - INIT_LIST_HEAD(&chunk->list); chunk->chan = chan; /* Toggling change bit (CB) in each chunk, this is a mechanism to * inform the eDMA HW block that this is a new linked list ready @@ -81,20 +58,10 @@ static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc) */ chunk->cb = !(desc->chunks_alloc % 2); - if (desc->chunk) { - /* Create and add new element into the linked list */ - if (!dw_edma_alloc_burst(chunk)) { - kfree(chunk); - return NULL; - } - desc->chunks_alloc++; - list_add_tail(&chunk->list, &desc->chunk->list); - } else { - /* List head */ - chunk->burst = NULL; - desc->chunks_alloc = 0; - desc->chunk = chunk; - } + chunk->nburst = nburst; + + list_add_tail(&chunk->list, &desc->chunk_list); + desc->chunks_alloc++; return chunk; } @@ -108,53 +75,23 @@ static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) return NULL; desc->chan = chan; - if (!dw_edma_alloc_chunk(desc)) { - kfree(desc); - return NULL; - } + + INIT_LIST_HEAD(&desc->chunk_list); return desc; } -static void dw_edma_free_burst(struct dw_edma_chunk *chunk) -{ - struct dw_edma_burst *child, *_next; - - /* Remove all the list elements */ - list_for_each_entry_safe(child, _next, &chunk->burst->list, list) { - list_del(&child->list); - kfree(child); - chunk->bursts_alloc--; - } - - /* Remove the list head */ - kfree(child); - chunk->burst = NULL; -} - -static void dw_edma_free_chunk(struct dw_edma_desc *desc) +static void dw_edma_free_desc(struct dw_edma_desc *desc) { struct dw_edma_chunk *child, *_next; - if (!desc->chunk) - return; - /* Remove all the list elements */ - list_for_each_entry_safe(child, _next, &desc->chunk->list, list) { - dw_edma_free_burst(child); + list_for_each_entry_safe(child, _next, &desc->chunk_list, list) { list_del(&child->list); kfree(child); desc->chunks_alloc--; } - /* Remove the list head */ - kfree(child); - desc->chunk = NULL; -} - -static void dw_edma_free_desc(struct dw_edma_desc *desc) -{ - dw_edma_free_chunk(desc); kfree(desc); } @@ -166,23 +103,17 @@ static void vchan_free_desc(struct virt_dma_desc *vdesc) static void dw_edma_core_start(struct dw_edma_chunk *chunk, bool first) { struct dw_edma_chan *chan = chunk->chan; - struct dw_edma_burst *child; u32 i = 0; - int j; if (chan->non_ll) { - child = list_first_entry_or_null(&chunk->burst->list, - struct dw_edma_burst, list); - if (child) - chan->dw->core->non_ll_start(chunk->chan, child); + if (chunk->nburst == 1) + chan->dw->core->non_ll_start(chunk->chan, &chunk->burst[0]); return; } - j = chunk->bursts_alloc; - list_for_each_entry(child, &chunk->burst->list, list) { - j--; - dw_edma_core_ll_data(chan, child, i++, chunk->cb, !j); - } + for (i = 0; i < chunk->nburst; i++) + dw_edma_core_ll_data(chan, &chunk->burst[i], i, chunk->cb, + i == chunk->nburst - 1); dw_edma_core_ll_link(chan, i, chunk->cb, chan->ll_region.paddr); @@ -206,14 +137,13 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) if (!desc) return 0; - child = list_first_entry_or_null(&desc->chunk->list, + child = list_first_entry_or_null(&desc->chunk_list, struct dw_edma_chunk, list); if (!child) return 0; dw_edma_core_start(child, !desc->xfer_sz); desc->xfer_sz += child->xfer_sz; - dw_edma_free_burst(child); list_del(&child->list); kfree(child); desc->chunks_alloc--; @@ -425,14 +355,14 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); enum dma_transfer_direction dir = xfer->direction; struct scatterlist *sg = NULL; - struct dw_edma_chunk *chunk; + struct dw_edma_chunk *chunk = NULL; struct dw_edma_burst *burst; struct dw_edma_desc *desc; u64 src_addr, dst_addr; size_t fsz = 0; u32 bursts_max; u32 cnt = 0; - int i; + u32 i; if (!chan->configured) return NULL; @@ -499,10 +429,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, if (unlikely(!desc)) goto err_alloc; - chunk = dw_edma_alloc_chunk(desc); - if (unlikely(!chunk)) - goto err_alloc; - if (xfer->type == EDMA_XFER_INTERLEAVED) { src_addr = xfer->xfer.il->src_start; dst_addr = xfer->xfer.il->dst_start; @@ -530,15 +456,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) break; - if (chunk->bursts_alloc == bursts_max) { - chunk = dw_edma_alloc_chunk(desc); + if (!(i % bursts_max)) { + u32 n = min(cnt - i, bursts_max); + + chunk = dw_edma_alloc_chunk(desc, n); if (unlikely(!chunk)) goto err_alloc; } - burst = dw_edma_alloc_burst(chunk); - if (unlikely(!burst)) - goto err_alloc; + burst = chunk->burst + (i % bursts_max); if (xfer->type == EDMA_XFER_CYCLIC) burst->sz = xfer->xfer.cyclic.len; diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 27415f3a2d04..4950c57fca34 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -43,7 +43,6 @@ struct dw_edma_chan; struct dw_edma_chunk; struct dw_edma_burst { - struct list_head list; u64 sar; u64 dar; u32 sz; @@ -52,18 +51,16 @@ struct dw_edma_burst { struct dw_edma_chunk { struct list_head list; struct dw_edma_chan *chan; - struct dw_edma_burst *burst; - - u32 bursts_alloc; - u8 cb; u32 xfer_sz; + u32 nburst; + struct dw_edma_burst burst[] __counted_by(nburst); }; struct dw_edma_desc { struct virt_dma_desc vd; struct dw_edma_chan *chan; - struct dw_edma_chunk *chunk; + struct list_head chunk_list; u32 chunks_alloc; From bed94a469f2ce783f17ac0699f26277bc3c74118 Mon Sep 17 00:00:00 2001 From: Frank Li Date: Mon, 13 Jul 2026 13:03:28 -0400 Subject: [PATCH 37/73] dmaengine: dw-edma: Remove struct dw_edma_chunk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The current descriptor layout is: struct dw_edma_desc *desc └─ chunk list └─ burst[] Creating a DMA descriptor requires at least two kzalloc() calls because each chunk is allocated as a linked-list node. Since the number of bursts is already known when the descriptor is created, this linked-list layer is unnecessary. Move the burst array directly into struct dw_edma_desc and remove the struct dw_edma_chunk layer entirely. Use start_burst and done_burst to track the current bursts, which current are in the DMA link list. Tested-by: Koichiro Den Tested-By: Devendra Verma Signed-off-by: Frank Li Link: https://patch.msgid.link/20260713-edma_ll-v7-10-6fb7498c901e@nxp.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 147 ++++++++++------------------- drivers/dma/dw-edma/dw-edma-core.h | 28 +++--- 2 files changed, 63 insertions(+), 112 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index c028011cc61c..30eeb7bffad8 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -40,82 +40,54 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) return cpu_addr; } -static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc, u32 nburst) -{ - struct dw_edma_chan *chan = desc->chan; - struct dw_edma_chunk *chunk; - - chunk = kzalloc_flex(*chunk, burst, nburst, GFP_NOWAIT); - if (unlikely(!chunk)) - return NULL; - - chunk->chan = chan; - /* Toggling change bit (CB) in each chunk, this is a mechanism to - * inform the eDMA HW block that this is a new linked list ready - * to be consumed. - * - Odd chunks originate CB equal to 0 - * - Even chunks originate CB equal to 1 - */ - chunk->cb = !(desc->chunks_alloc % 2); - - chunk->nburst = nburst; - - list_add_tail(&chunk->list, &desc->chunk_list); - desc->chunks_alloc++; - - return chunk; -} - -static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan) +static struct dw_edma_desc * +dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst) { struct dw_edma_desc *desc; - desc = kzalloc_obj(*desc, GFP_NOWAIT); + desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT); if (unlikely(!desc)) return NULL; desc->chan = chan; - - INIT_LIST_HEAD(&desc->chunk_list); + desc->nburst = nburst; + desc->cb = true; return desc; } -static void dw_edma_free_desc(struct dw_edma_desc *desc) -{ - struct dw_edma_chunk *child, *_next; - - /* Remove all the list elements */ - list_for_each_entry_safe(child, _next, &desc->chunk_list, list) { - list_del(&child->list); - kfree(child); - desc->chunks_alloc--; - } - - kfree(desc); -} - static void vchan_free_desc(struct virt_dma_desc *vdesc) { - dw_edma_free_desc(vd2dw_edma_desc(vdesc)); + kfree(vd2dw_edma_desc(vdesc)); } -static void dw_edma_core_start(struct dw_edma_chunk *chunk, bool first) +static void dw_edma_core_start(struct dw_edma_desc *desc, bool first) { - struct dw_edma_chan *chan = chunk->chan; - u32 i = 0; + struct dw_edma_chan *chan = desc->chan; + size_t i = 0; if (chan->non_ll) { - if (chunk->nburst == 1) - chan->dw->core->non_ll_start(chunk->chan, &chunk->burst[0]); + chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]); + desc->done_burst = desc->start_burst; + desc->start_burst += 1; return; } - for (i = 0; i < chunk->nburst; i++) - dw_edma_core_ll_data(chan, &chunk->burst[i], i, chunk->cb, - i == chunk->nburst - 1); + for (i = 0; i + desc->start_burst < desc->nburst; i++) { + u32 idx = i + desc->start_burst; - dw_edma_core_ll_link(chan, i, chunk->cb, chan->ll_region.paddr); + if (i == chan->ll_max) + break; + + dw_edma_core_ll_data(chan, &desc->burst[idx], + i, desc->cb, + idx == desc->nburst - 1 || i == chan->ll_max - 1); + } + + desc->done_burst = desc->start_burst; + desc->start_burst += i; + + dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr); if (first) dw_edma_core_ch_enable(chan); @@ -125,7 +97,6 @@ static void dw_edma_core_start(struct dw_edma_chunk *chunk, bool first) static int dw_edma_start_transfer(struct dw_edma_chan *chan) { - struct dw_edma_chunk *child; struct dw_edma_desc *desc; struct virt_dma_desc *vd; @@ -137,16 +108,9 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) if (!desc) return 0; - child = list_first_entry_or_null(&desc->chunk_list, - struct dw_edma_chunk, list); - if (!child) - return 0; + dw_edma_core_start(desc, !desc->start_burst); - dw_edma_core_start(child, !desc->xfer_sz); - desc->xfer_sz += child->xfer_sz; - list_del(&child->list); - kfree(child); - desc->chunks_alloc--; + desc->cb = !desc->cb; return 1; } @@ -337,8 +301,10 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, vd = vchan_find_desc(&chan->vc, cookie); if (vd) { desc = vd2dw_edma_desc(vd); - if (desc) - residue = desc->alloc_sz - desc->xfer_sz; + + residue = desc->alloc_sz; + if (desc && desc->done_burst) + residue -= desc->burst[desc->done_burst - 1].xfer_sz; } spin_unlock_irqrestore(&chan->vc.lock, flags); @@ -355,13 +321,11 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan); enum dma_transfer_direction dir = xfer->direction; struct scatterlist *sg = NULL; - struct dw_edma_chunk *chunk = NULL; struct dw_edma_burst *burst; struct dw_edma_desc *desc; u64 src_addr, dst_addr; size_t fsz = 0; - u32 bursts_max; - u32 cnt = 0; + size_t cnt = 0; u32 i; if (!chan->configured) @@ -418,17 +382,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, return NULL; } - /* - * For non-LL mode, only a single burst can be handled - * in a single chunk unlike LL mode where multiple bursts - * can be configured in a single chunk. - */ - bursts_max = chan->non_ll ? 1 : chan->ll_max; - - desc = dw_edma_alloc_desc(chan); - if (unlikely(!desc)) - goto err_alloc; - if (xfer->type == EDMA_XFER_INTERLEAVED) { src_addr = xfer->xfer.il->src_start; dst_addr = xfer->xfer.il->dst_start; @@ -452,19 +405,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, fsz = xfer->xfer.il->frame_size; } + desc = dw_edma_alloc_desc(chan, cnt); + if (unlikely(!desc)) + return NULL; + for (i = 0; i < cnt; i++) { if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg) break; - if (!(i % bursts_max)) { - u32 n = min(cnt - i, bursts_max); - - chunk = dw_edma_alloc_chunk(desc, n); - if (unlikely(!chunk)) - goto err_alloc; - } - - burst = chunk->burst + (i % bursts_max); + burst = desc->burst + i; if (xfer->type == EDMA_XFER_CYCLIC) burst->sz = xfer->xfer.cyclic.len; @@ -473,8 +422,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, else if (xfer->type == EDMA_XFER_INTERLEAVED) burst->sz = xfer->xfer.il->sgl[i % fsz].size; - chunk->xfer_sz += burst->sz; desc->alloc_sz += burst->sz; + burst->xfer_sz = desc->alloc_sz; if (dir == DMA_DEV_TO_MEM) { burst->sar = src_addr; @@ -529,12 +478,6 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer, } return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags); - -err_alloc: - if (desc) - dw_edma_free_desc(desc); - - return NULL; } static struct dma_async_tx_descriptor * @@ -605,8 +548,14 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd, return; desc = vd2dw_edma_desc(vd); - if (desc) - residue = desc->alloc_sz - desc->xfer_sz; + if (desc) { + residue = desc->alloc_sz; + + if (result == DMA_TRANS_NOERROR) + residue -= desc->burst[desc->start_burst - 1].xfer_sz; + else if (desc->done_burst) + residue -= desc->burst[desc->done_burst - 1].xfer_sz; + } res = &vd->tx_result; res->result = result; @@ -625,7 +574,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) switch (chan->request) { case EDMA_REQ_NONE: desc = vd2dw_edma_desc(vd); - if (!desc->chunks_alloc) { + if (desc->start_burst >= desc->nburst) { dw_hdma_set_callback_result(vd, DMA_TRANS_NOERROR); list_del(&vd->node); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 4950c57fca34..3c958ca05144 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -46,26 +46,21 @@ struct dw_edma_burst { u64 sar; u64 dar; u32 sz; -}; - -struct dw_edma_chunk { - struct list_head list; - struct dw_edma_chan *chan; - u8 cb; + /* precalulate summary of previous burst total size */ u32 xfer_sz; - u32 nburst; - struct dw_edma_burst burst[] __counted_by(nburst); }; struct dw_edma_desc { struct virt_dma_desc vd; struct dw_edma_chan *chan; - struct list_head chunk_list; - - u32 chunks_alloc; u32 alloc_sz; - u32 xfer_sz; + + size_t done_burst; + size_t start_burst; + u8 cb; + size_t nburst; + struct dw_edma_burst burst[] __counted_by(nburst); }; struct dw_edma_chan { @@ -128,7 +123,6 @@ struct dw_edma_core_ops { void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr); void (*ch_doorbell)(struct dw_edma_chan *chan); void (*ch_enable)(struct dw_edma_chan *chan); - void (*ch_config)(struct dw_edma_chan *chan); void (*debugfs_on)(struct dw_edma *dw); void (*ack_emulated_irq)(struct dw_edma *dw); @@ -170,6 +164,14 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan) return vc2dw_edma_chan(to_virt_chan(dchan)); } +static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan) +{ + if (chan->dir == EDMA_DIR_WRITE) + return chan->dw->chip->ll_region_wr[chan->id].paddr; + + return chan->dw->chip->ll_region_rd[chan->id].paddr; +} + static inline void dw_edma_core_off(struct dw_edma *dw) { From 643c1e1ae3eb3cd9be31e83c2240e41849a6cb56 Mon Sep 17 00:00:00 2001 From: Jun Guo Date: Thu, 21 May 2026 15:29:23 +0800 Subject: [PATCH 38/73] dmaengine: arm-dma350: enable ANYCH interrupt for shared IRQ wiring Enable DMANSECCTRL.INTREN_ANYCHINTR during probe so channel interrupts are propagated when integrators wire DMA-350 channels onto a shared IRQ line. Signed-off-by: Jun Guo Reviewed-by: Robin Murphy Link: https://patch.msgid.link/20260521072924.3000282-2-jun.guo@cixtech.com Signed-off-by: Vinod Koul --- drivers/dma/arm-dma350.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/drivers/dma/arm-dma350.c b/drivers/dma/arm-dma350.c index 84220fa83029..09403aca8bb0 100644 --- a/drivers/dma/arm-dma350.c +++ b/drivers/dma/arm-dma350.c @@ -13,6 +13,11 @@ #include "dmaengine.h" #include "virt-dma.h" +#define DMANSECCTRL 0x200 + +#define NSEC_CTRL 0x0c +#define INTREN_ANYCHINTR_EN BIT(0) + #define DMAINFO 0x0f00 #define DMA_BUILDCFG0 0xb0 @@ -582,6 +587,10 @@ static int d350_probe(struct platform_device *pdev) dmac->dma.device_issue_pending = d350_issue_pending; INIT_LIST_HEAD(&dmac->dma.channels); + reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL); + writel_relaxed(reg | INTREN_ANYCHINTR_EN, + base + DMANSECCTRL + NSEC_CTRL); + /* Would be nice to have per-channel caps for this... */ memset = true; for (int i = 0; i < nchan; i++) { From 64173b6b7609acb88fb763041c9e496caa6e834d Mon Sep 17 00:00:00 2001 From: Devendra K Verma Date: Mon, 13 Jul 2026 12:18:54 +0530 Subject: [PATCH 39/73] dmaengine: dw-edma: Enable HDMA 64R/W Channels As per 'Designware Cores PCI Express Controller Databook', Section 7.1 - Overview, HDMA supports 64 Read and 64 Write channels. Current controller driver supports up to 8 read and write channels only. In order to utilize all the channels the controller driver need to have the channel related structs and variables as per the number of channels supported by IP. Following changes are made to enable 64 Read / 64 Write channel support: o Defined HDMA specific macros to reflect the channel count. o The count of ll_regions and dt_regions in dw_edma_chip and dw_edma_pcie_data shall be in accordance to number of read and write channels. o In dw_edma_probe() configure the channels as per the channels of the IP used. o Changed mask types to u64 for higher channel counts. Signed-off-by: Devendra K Verma Reviewed-by: Frank Li Link: https://patch.msgid.link/20260713064854.4065262-1-devverma@amd.com Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 19 +++++++++++++------ drivers/dma/dw-edma/dw-edma-core.h | 5 +++-- drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++---- drivers/dma/dw-edma/dw-edma-v0-core.c | 6 +++--- drivers/dma/dw-edma/dw-hdma-v0-core.c | 27 +++++++++++++++++++-------- drivers/dma/dw-edma/dw-hdma-v0-regs.h | 2 +- include/linux/dma/edma.h | 10 ++++++---- 7 files changed, 49 insertions(+), 28 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 30eeb7bffad8..384c482058d1 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -840,9 +840,9 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) irq = &dw->irq[pos]; if (chan->dir == EDMA_DIR_WRITE) - irq->wr_mask |= BIT(chan->id); + bitmap_set(irq->wr_mask, chan->id, 1); else - irq->rd_mask |= BIT(chan->id); + bitmap_set(irq->rd_mask, chan->id, 1); irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); @@ -983,6 +983,8 @@ int dw_edma_probe(struct dw_edma_chip *chip) struct dw_edma *dw; u32 wr_alloc = 0; u32 rd_alloc = 0; + u16 max_wr_cnt; + u16 max_rd_cnt; int i, err; if (!chip) @@ -998,20 +1000,25 @@ int dw_edma_probe(struct dw_edma_chip *chip) dw->chip = chip; - if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) + if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) { dw_hdma_v0_core_register(dw); - else + max_wr_cnt = HDMA_MAX_WR_CH; + max_rd_cnt = HDMA_MAX_RD_CH; + } else { dw_edma_v0_core_register(dw); + max_wr_cnt = EDMA_MAX_WR_CH; + max_rd_cnt = EDMA_MAX_RD_CH; + } raw_spin_lock_init(&dw->lock); dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); - dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH); + dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, max_wr_cnt); dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, dw_edma_core_ch_count(dw, EDMA_DIR_READ)); - dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH); + dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, max_rd_cnt); if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) return -EINVAL; diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 3c958ca05144..e39c11bc91fb 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -84,9 +84,10 @@ struct dw_edma_chan { struct dw_edma_irq { struct msi_msg msi; - u32 wr_mask; - u32 rd_mask; struct dw_edma *dw; + + DECLARE_BITMAP(wr_mask, HDMA_MAX_WR_CH); + DECLARE_BITMAP(rd_mask, HDMA_MAX_RD_CH); }; struct dw_edma { diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 791c46e8ae4c..b34e1e45aeb3 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -62,11 +62,11 @@ struct dw_edma_pcie_data { /* eDMA registers location */ struct dw_edma_block rg; /* eDMA memory linked list location */ - struct dw_edma_block ll_wr[EDMA_MAX_WR_CH]; - struct dw_edma_block ll_rd[EDMA_MAX_RD_CH]; + struct dw_edma_block ll_wr[HDMA_MAX_WR_CH]; + struct dw_edma_block ll_rd[HDMA_MAX_RD_CH]; /* eDMA memory data location */ - struct dw_edma_block dt_wr[EDMA_MAX_WR_CH]; - struct dw_edma_block dt_rd[EDMA_MAX_RD_CH]; + struct dw_edma_block dt_wr[HDMA_MAX_WR_CH]; + struct dw_edma_block dt_rd[HDMA_MAX_RD_CH]; /* Other */ enum dw_edma_map_format mf; u8 irqs; diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 7b4933c66f9f..7bb3ec461cce 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -239,7 +239,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, irqreturn_t ret = IRQ_NONE; struct dw_edma_chan *chan; unsigned long off; - u32 mask; + unsigned long *mask; if (dir == EDMA_DIR_WRITE) { total = dw->wr_ch_cnt; @@ -252,7 +252,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, } val = dw_edma_v0_core_status_done_int(dw, dir); - val &= mask; + val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; @@ -263,7 +263,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, } val = dw_edma_v0_core_status_abort_int(dw, dir); - val &= mask; + val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 4bf5a441afbf..87ba3e8c9fa1 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -53,13 +53,24 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) static void dw_hdma_v0_core_off(struct dw_edma *dw) { int id; + enum dw_edma_dir dir; - for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) { - SET_BOTH_CH_32(dw, id, int_setup, - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - SET_BOTH_CH_32(dw, id, int_clear, - HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - SET_BOTH_CH_32(dw, id, ch_en, 0); + dir = EDMA_DIR_WRITE; + for (id = 0; id < dw->wr_ch_cnt; id++) { + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); + } + + dir = EDMA_DIR_READ; + for (id = 0; id < dw->rd_ch_cnt; id++) { + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); } } @@ -118,7 +129,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, unsigned long total, pos, val; irqreturn_t ret = IRQ_NONE; struct dw_edma_chan *chan; - unsigned long off, mask; + unsigned long off, *mask; if (dir == EDMA_DIR_WRITE) { total = dw->wr_ch_cnt; @@ -130,7 +141,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } - for_each_set_bit(pos, &mask, total) { + for_each_set_bit(pos, mask, total) { chan = &dw->chan[pos + off]; val = dw_hdma_v0_core_status_int(chan); diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index 7759ba9b4850..48e40efceb2e 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -11,7 +11,7 @@ #include -#define HDMA_V0_MAX_NR_CH 8 +#define HDMA_V0_MAX_NR_CH 64 #define HDMA_V0_CH_EN BIT(0) #define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6) #define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5) diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 1fafd5b0e315..da7a5cc93ad4 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -14,6 +14,8 @@ #define EDMA_MAX_WR_CH 8 #define EDMA_MAX_RD_CH 8 +#define HDMA_MAX_WR_CH 64 +#define HDMA_MAX_RD_CH 64 struct dw_edma; @@ -89,12 +91,12 @@ struct dw_edma_chip { u16 ll_wr_cnt; u16 ll_rd_cnt; /* link list address */ - struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH]; + struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH]; + struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH]; /* data region */ - struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH]; - struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH]; + struct dw_edma_region dt_region_wr[HDMA_MAX_WR_CH]; + struct dw_edma_region dt_region_rd[HDMA_MAX_RD_CH]; /* interrupt emulation */ int db_irq; From faa4ddedab7d75e0f28e2d14c14a428664deeb90 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Thu, 16 Jul 2026 13:29:46 -0700 Subject: [PATCH 40/73] dma: fsl_raid: convert descriptor stores to big-endian The descriptor structs (fsl_re_cmpnd_frame / fsl_re_hw_desc) are in-memory but their fields are __be32, because the structures are handed to the device as big-endian. The driver stored CPU-endian u32 values into them directly, which is both wrong (the engine would see byte-swapped lengths/addresses) and flagged by sparse as a base-type mismatch. Wrap those stores in cpu_to_be32() so the values are little->big converted. Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202008111749.yy85rFMD%25lkp@intel.com/ Assisted-by: opencode:hy3-free Signed-off-by: Rosen Penev Reviewed-by: Frank Li Link: https://patch.msgid.link/20260716202949.677290-2-rosenp@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/fsl_raid.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 99945845d8b5..2778d07a05e8 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -242,9 +242,9 @@ static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, u32 efrl = length & FSL_RE_CF_LENGTH_MASK; efrl |= final << FSL_RE_CF_FINAL_SHIFT; - cf[index].efrl32 = efrl; - cf[index].addr_high = upper_32_bits(addr); - cf[index].addr_low = lower_32_bits(addr); + cf[index].efrl32 = cpu_to_be32(efrl); + cf[index].addr_high = cpu_to_be32(upper_32_bits(addr)); + cf[index].addr_low = cpu_to_be32(lower_32_bits(addr)); } static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, @@ -256,9 +256,10 @@ static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan); INIT_LIST_HEAD(&desc->node); - desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT; - desc->hwdesc.lbea32 = upper_32_bits(paddr); - desc->hwdesc.addr_low = lower_32_bits(paddr); + desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT << + FSL_RE_HWDESC_FMT_SHIFT); + desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr)); + desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr)); desc->cf_addr = cf; desc->cf_paddr = paddr; From 416e9fd2059631405031aaa033a7a7851467be86 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Thu, 16 Jul 2026 13:29:47 -0700 Subject: [PATCH 41/73] dma: fsl_raid: set final bit via fill_cfd_frame() argument The final-frame bit is now passed as the "final" argument of fill_cfd_frame() (as fsl_re_prep_dma_memcpy already did) and set in CPU order before the single cpu_to_be32() store, replacing the previous read-modify-write of the __be32 efrl32 field. Assisted-by: opencode:hy3-free Signed-off-by: Rosen Penev Reviewed-by: Frank Li Link: https://patch.msgid.link/20260716202949.677290-3-rosenp@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/fsl_raid.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 2778d07a05e8..888f55b672a5 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -375,11 +375,11 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( for (i = 2, j = 0; j < save_src_cnt; i++, j++) fill_cfd_frame(cf, i, len, src[j], 0); + /* Fill the last frame and mark it final */ if (cont_q) - fill_cfd_frame(cf, i++, len, dest, 0); - - /* Setting the final bit in the last source buffer frame in CFD */ - cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; + fill_cfd_frame(cf, i, len, dest, 1); + else + fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); return &desc->async_tx; } @@ -505,16 +505,16 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( p[save_src_cnt + 2] = 1; fill_cfd_frame(cf, i++, len, dest[0], 0); fill_cfd_frame(cf, i++, len, dest[1], 0); - fill_cfd_frame(cf, i++, len, dest[1], 0); + fill_cfd_frame(cf, i++, len, dest[1], 1); } else { dev_err(re_chan->dev, "PQ tx continuation error!\n"); return NULL; } + } else { + /* Mark the last source buffer frame final */ + fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); } - /* Setting the final bit in the last source buffer frame in CFD */ - cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT; - return &desc->async_tx; } From 68b7fbc235290937e4e2daa097896b6b94618f15 Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Thu, 16 Jul 2026 13:29:48 -0700 Subject: [PATCH 42/73] dma: fsl_raid: keep MMIO bases as void __iomem and cast at access The fsl_re_ctrl and fsl_re_chan_cfg structures describe memory-mapped RAID Engine registers accessed only via ioread32be()/iowrite32be(), yet the pointers to them (re_regs in struct fsl_re_drv_private, and jrregs in struct fsl_re_chan) were not __iomem-qualified, so sparse emitted "different address spaces" warnings for every register access. Store both MMIO bases as a plain void __iomem * and derive jrregs with void __iomem * arithmetic from re_regs, rather than carrying typed register struct pointers through the driver. Each function that touches the registers introduces a local typed pointer (struct fsl_re_ctrl __iomem *ctrl) and uses ->field, which is the idiomatic kernel pattern and keeps the registers' __iomem qualification intact. Reported-by: kernel test robot Link: https://lore.kernel.org/oe-kbuild-all/202008111749.yy85rFMD%25lkp@intel.com/ Assisted-by: opencode:hy3-free Signed-off-by: Rosen Penev Reviewed-by: Frank Li Link: https://patch.msgid.link/20260716202949.677290-4-rosenp@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/fsl_raid.c | 19 ++++++++++--------- drivers/dma/fsl_raid.h | 4 ++-- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 888f55b672a5..524f83faf3da 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -657,8 +657,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev, goto err_free; } - chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs + - off + ptr); + chan->jrregs = re_priv->base + off + ptr; /* read irq property from dts */ chan->irq = irq_of_parse_and_map(np, 0); @@ -746,6 +745,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev, /* Probe function for RAID Engine */ static int fsl_re_probe(struct platform_device *ofdev) { + struct fsl_re_ctrl __iomem *re_regs; struct fsl_re_drv_private *re_priv; struct device_node *child; u32 off; @@ -764,20 +764,21 @@ static int fsl_re_probe(struct platform_device *ofdev) return -ENODEV; /* IOMAP the entire RAID Engine region */ - re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res)); - if (!re_priv->re_regs) + re_regs = devm_ioremap(dev, res->start, resource_size(res)); + if (!re_regs) return -EBUSY; + re_priv->base = re_regs; /* Program the RE mode */ - out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE); + out_be32(&re_regs->global_config, FSL_RE_NON_DPAA_MODE); /* Program Galois Field polynomial */ - out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY); + out_be32(&re_regs->galois_field_config, FSL_RE_GFM_POLY); dev_info(dev, "version %x, mode %x, gfp %x\n", - in_be32(&re_priv->re_regs->re_version_id), - in_be32(&re_priv->re_regs->global_config), - in_be32(&re_priv->re_regs->galois_field_config)); + in_be32(&re_regs->re_version_id), + in_be32(&re_regs->global_config), + in_be32(&re_regs->galois_field_config)); dma_dev = &re_priv->dma_dev; dma_dev->dev = dev; diff --git a/drivers/dma/fsl_raid.h b/drivers/dma/fsl_raid.h index 69d743c04973..adbfede330a7 100644 --- a/drivers/dma/fsl_raid.h +++ b/drivers/dma/fsl_raid.h @@ -256,7 +256,7 @@ struct fsl_re_hw_desc { struct fsl_re_drv_private { u8 total_chans; struct dma_device dma_dev; - struct fsl_re_ctrl *re_regs; + void __iomem *base; struct fsl_re_chan *re_jrs[FSL_RE_MAX_CHANS]; struct dma_pool *cf_desc_pool; struct dma_pool *hw_desc_pool; @@ -273,7 +273,7 @@ struct fsl_re_chan { struct device *dev; struct fsl_re_drv_private *re_dev; struct dma_chan chan; - struct fsl_re_chan_cfg *jrregs; + struct fsl_re_chan_cfg __iomem *jrregs; int irq; struct tasklet_struct irqtask; u32 alloc_count; From 30cb1b9fd4295f54d22b3a15ecf8525c9fe03ebe Mon Sep 17 00:00:00 2001 From: Rosen Penev Date: Thu, 16 Jul 2026 13:29:49 -0700 Subject: [PATCH 43/73] dma: fsl_raid: use devm_platform_ioremap_resource Replace the open-coded platform_get_resource() plus devm_ioremap() sequence with devm_platform_ioremap_resource(), which fetches the resource, requests the region and maps it in one call. Switch the error check to IS_ERR()/PTR_ERR() and drop the now-unused struct resource pointer. The raideng node has a single reg region (0x320000, 0x10000); the job-queue/ring children are separate OF devices probed independently, so the region reservation added by devm_ioremap_resource() is exclusive and does not introduce overlap failures. Assisted-by: opencode:hy3-free Signed-off-by: Rosen Penev Reviewed-by: Frank Li Link: https://patch.msgid.link/20260716202949.677290-5-rosenp@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/fsl_raid.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/drivers/dma/fsl_raid.c b/drivers/dma/fsl_raid.c index 524f83faf3da..2d86f61105e5 100644 --- a/drivers/dma/fsl_raid.c +++ b/drivers/dma/fsl_raid.c @@ -751,22 +751,18 @@ static int fsl_re_probe(struct platform_device *ofdev) u32 off; u8 ridx = 0; struct dma_device *dma_dev; - struct resource *res; int rc; struct device *dev = &ofdev->dev; + /* IOMAP the entire RAID Engine region */ + re_regs = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(re_regs)) + return PTR_ERR(re_regs); + re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL); if (!re_priv) return -ENOMEM; - res = platform_get_resource(ofdev, IORESOURCE_MEM, 0); - if (!res) - return -ENODEV; - - /* IOMAP the entire RAID Engine region */ - re_regs = devm_ioremap(dev, res->start, resource_size(res)); - if (!re_regs) - return -EBUSY; re_priv->base = re_regs; /* Program the RE mode */ From dc8a5238e1c7db74a7b4f02bcf0b05559b6627c5 Mon Sep 17 00:00:00 2001 From: bui duc phuc Date: Thu, 16 Jul 2026 12:27:58 +0700 Subject: [PATCH 44/73] dmaengine: validate dev and name in dma_request_chan() dma_request_chan() assumes both @dev and @name are valid, but neither is checked before use. dev is dereferenced immediately via dev_fwnode(), which accesses dev->of_node or dev->fwnode without checking for NULL. Likewise, if name is NULL and the OF/ACPI lookup does not succeed, the legacy filter-map path eventually passes it to strcmp(), resulting in a NULL pointer dereference. These are caller bugs rather than normal lookup failures, so add a WARN_ON() at function entry to catch invalid arguments early during development instead of crashing later. No functional change for valid callers. Signed-off-by: bui duc phuc Reviewed-by: Frank Li Link: https://patch.msgid.link/20260716052758.23465-1-phucduc.bui@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/dmaengine.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index 23e3bb18c166..6ffd8bd82154 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c @@ -814,10 +814,15 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device, */ struct dma_chan *dma_request_chan(struct device *dev, const char *name) { - struct fwnode_handle *fwnode = dev_fwnode(dev); + struct fwnode_handle *fwnode; struct dma_device *d, *_d; struct dma_chan *chan = NULL; + if (WARN_ON(!dev || !name)) + return ERR_PTR(-EINVAL); + + fwnode = dev_fwnode(dev); + if (is_of_node(fwnode)) chan = of_dma_request_slave_channel(to_of_node(fwnode), name); else if (is_acpi_device_node(fwnode)) From 338c853b7c3b422fefda195b9b8010e40611c96f Mon Sep 17 00:00:00 2001 From: Christophe JAILLET Date: Tue, 14 Jul 2026 16:06:33 +0200 Subject: [PATCH 45/73] dmaengine: Constify struct dma_descriptor_metadata_ops 'struct dma_descriptor_metadata_ops' in not modified in these drivers. Constifying these structures moves some data to a read-only section, so increases overall security, especially when the structure holds some function pointers. On a x86_64, with allmodconfig, as an example: Before: ====== text data bss dec hex filename 120635 21584 64 142283 22bcb drivers/dma/xilinx/xilinx_dma.o After: ===== text data bss dec hex filename 120699 21520 64 142283 22bcb drivers/dma/xilinx/xilinx_dma.o Signed-off-by: Christophe JAILLET Reviewed-by: Frank Li Reviewed-by: Sai Sree Kartheek Adivi Reviewed-by: Manivannan Sadhasivam Reviewed-by: Radhey Shyam Pandey Link: https://patch.msgid.link/b0a22171f3ed68e156a2fa84383e99c23ec6b2ff.1784037977.git.christophe.jaillet@wanadoo.fr Signed-off-by: Vinod Koul --- drivers/dma/ti/k3-udma.c | 2 +- drivers/dma/xilinx/xilinx_dma.c | 2 +- include/linux/dmaengine.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/dma/ti/k3-udma.c b/drivers/dma/ti/k3-udma.c index 1cf158eb7bdb..fb21e0df5ab7 100644 --- a/drivers/dma/ti/k3-udma.c +++ b/drivers/dma/ti/k3-udma.c @@ -3408,7 +3408,7 @@ static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc, return 0; } -static struct dma_descriptor_metadata_ops metadata_ops = { +static const struct dma_descriptor_metadata_ops metadata_ops = { .attach = udma_attach_metadata, .get_ptr = udma_get_metadata_ptr, .set_len = udma_set_metadata_len, diff --git a/drivers/dma/xilinx/xilinx_dma.c b/drivers/dma/xilinx/xilinx_dma.c index 98b41b8f8915..bef2b031dba1 100644 --- a/drivers/dma/xilinx/xilinx_dma.c +++ b/drivers/dma/xilinx/xilinx_dma.c @@ -655,7 +655,7 @@ static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx, return seg->hw.app; } -static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { +static const struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = { .get_ptr = xilinx_dma_get_metadata_ptr, }; diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 6fe46c0c9452..fe33a20abc61 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -631,7 +631,7 @@ struct dma_async_tx_descriptor { void *callback_param; struct dmaengine_unmap_data *unmap; enum dma_desc_metadata_mode desc_metadata_mode; - struct dma_descriptor_metadata_ops *metadata_ops; + const struct dma_descriptor_metadata_ops *metadata_ops; #ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH struct dma_async_tx_descriptor *next; struct dma_async_tx_descriptor *parent; From ef1b080e03acc83d5bde841da67036985acd50dc Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:31 +0900 Subject: [PATCH 46/73] dmaengine: dw-edma: Fix HDMA channel status register access GET_CH_32() takes the direction before the channel ID, but dw_hdma_v0_core_ch_status() passed them in the opposite order. This can make the status callback read another HDMA channel status register. Use the same argument order as the other HDMA register accesses. Fixes: e74c39573d35 ("dmaengine: dw-edma: Add support for native HDMA") Cc: stable@vger.kernel.org Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-2-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-hdma-v0-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 87ba3e8c9fa1..fe64f9931bba 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -90,7 +90,7 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan) u32 tmp; tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK, - GET_CH_32(dw, chan->id, chan->dir, ch_stat)); + GET_CH_32(dw, chan->dir, chan->id, ch_stat)); if (tmp == 1) return DMA_IN_PROGRESS; From 99109a51efd28c9a661fbfb9469b023c517b31d1 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:32 +0900 Subject: [PATCH 47/73] dmaengine: dw-edma: Terminate all descriptors without callbacks The DMA Engine client documentation says in the "Terminate APIs" section of Documentation/driver-api/dmaengine/client.rst: "No callback functions will be called for any incomplete transfers." dw-edma instead calls vchan_cookie_complete() when a deferred STOP reaches the interrupt handler. This schedules a callback for the active descriptor and leaves other issued or submitted descriptors queued. A late callback after dmaengine_terminate_sync() can dereference client state that has already been freed, while leftover descriptors may later restart into reused buffers or leak. Move all issued and submitted descriptors to the terminated list whenever termination completes. For a pending STOP, do this from both the DONE and ABORT paths. Complete their cookies in order without scheduling callbacks. A STOP can remain pending until the running transfer raises an interrupt. Make device_synchronize() wait for such a pending STOP to complete before releasing terminated descriptors. Reuse it from free_chan_resources(), then release the remaining virt-dma resources. Sleep instead of busy-polling while waiting, and warn if the existing timeout expires. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-3-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 90 +++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 14 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 384c482058d1..8b094857f821 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include #include @@ -115,6 +116,35 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan) return 1; } +static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd) +{ + list_del(&vd->node); + dma_cookie_complete(&vd->tx); + vchan_terminate_vdesc(vd); +} + +static void dw_edma_terminate_vdesc_list(struct list_head *head) +{ + struct virt_dma_desc *vd, *_vd; + + list_for_each_entry_safe(vd, _vd, head, node) + dw_edma_terminate_vdesc(vd); +} + +/* Must be called with vc.lock held. */ +static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan) +{ + /* + * This order must not be reversed. Cookies are assigned when + * descriptors are submitted, so desc_issued contains older cookies + * than desc_submitted. Completing desc_submitted first could move + * chan->vc.chan.completed_cookie backwards when desc_issued is + * terminated afterwards. + */ + dw_edma_terminate_vdesc_list(&chan->vc.desc_issued); + dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted); +} + static void dw_edma_device_caps(struct dma_chan *dchan, struct dma_slave_caps *caps) { @@ -236,20 +266,22 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { - /* Do nothing */ + dw_edma_terminate_all_descs(chan); } else if (chan->status == EDMA_ST_PAUSE) { + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->status == EDMA_ST_IDLE) { - chan->configured = false; + dw_edma_terminate_all_descs(chan); } else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) { /* * The channel is in a false BUSY state, probably didn't * receive or lost an interrupt */ + dw_edma_terminate_all_descs(chan); chan->status = EDMA_ST_IDLE; - chan->configured = false; } else if (chan->request > EDMA_REQ_PAUSE) { err = -EPERM; } else { @@ -587,8 +619,7 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) break; case EDMA_REQ_STOP: - list_del(&vd->node); - vchan_cookie_complete(vd); + dw_edma_terminate_all_descs(chan); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; break; @@ -612,7 +643,9 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_lock_irqsave(&chan->vc.lock, flags); vd = vchan_next_desc(&chan->vc); - if (vd) { + if (vd && chan->request == EDMA_REQ_STOP) { + dw_edma_terminate_all_descs(chan); + } else if (vd) { dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED); list_del(&vd->node); vchan_cookie_complete(vd); @@ -770,21 +803,49 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan) return 0; } -static void dw_edma_free_chan_resources(struct dma_chan *dchan) +static void dw_edma_wait_termination(struct dma_chan *dchan) { + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long timeout = jiffies + msecs_to_jiffies(5000); - int ret; + bool stopping; + /* + * A STOP may be deferred to a later interrupt while the channel is still + * running. Wait until that handler completes the termination. + */ while (time_before(jiffies, timeout)) { - ret = dw_edma_device_terminate_all(dchan); - if (!ret) - break; + scoped_guard(spinlock_irqsave, &chan->vc.lock) + stopping = chan->request == EDMA_REQ_STOP; - if (time_after_eq(jiffies, timeout)) + if (!stopping) return; - cpu_relax(); + fsleep(1000); } + + dev_warn(chan->dw->chip->dev, + "timeout waiting for channel termination\n"); +} + +static void dw_edma_device_synchronize(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_wait_termination(dchan); + vchan_synchronize(&chan->vc); +} + +static void dw_edma_free_chan_resources(struct dma_chan *dchan) +{ + struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); + + dw_edma_device_terminate_all(dchan); + dw_edma_device_synchronize(dchan); + + scoped_guard(spinlock_irqsave, &chan->vc.lock) + chan->configured = false; + + vchan_free_chan_resources(&chan->vc); } static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) @@ -883,6 +944,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) dma->device_pause = dw_edma_device_pause; dma->device_resume = dw_edma_device_resume; dma->device_terminate_all = dw_edma_device_terminate_all; + dma->device_synchronize = dw_edma_device_synchronize; dma->device_issue_pending = dw_edma_device_issue_pending; dma->device_tx_status = dw_edma_device_tx_status; dma->device_prep_config_sg = dw_edma_device_prep_config_sg; From dd80e259f65d932634e26d366570d71669ef6654 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:33 +0900 Subject: [PATCH 48/73] dmaengine: dw-edma: Serialize abort state updates dw_edma_abort_interrupt() drops vc.lock before changing request and status. issue_pending() can acquire the lock in that small window, observe the old busy state, and skip starting queued descriptors. Then the abort handler overwrites the channel status as idle, leaving the new descriptors stranded for good. Keep descriptor completion and the state transition in the same critical section. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-4-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 8b094857f821..c0d9f7e373f3 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -650,9 +650,9 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) list_del(&vd->node); vchan_cookie_complete(vd); } - spin_unlock_irqrestore(&chan->vc.lock, flags); chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + spin_unlock_irqrestore(&chan->vc.lock, flags); } static void dw_edma_emul_irq_ack(struct irq_data *d) From c154060016a9db2ac889bfdb0a3c1322f9be8ded Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:34 +0900 Subject: [PATCH 49/73] dmaengine: dw-edma: Complete descriptors before pausing If PAUSE is requested while the final burst of a descriptor is in flight, the DONE interrupt takes the PAUSE path without checking whether the descriptor has been depleted. The depleted descriptor remains on the issued list and the channel enters EDMA_ST_PAUSE. On resume, dw_edma_start_transfer() can select that depleted descriptor again even though no burst remains, leaving the channel in an invalid busy state. Check for descriptor completion before acknowledging PAUSE. If there is no work to start on resume, leave the channel idle. Also ignore DONE interrupts while the channel is paused so a stale or repeated interrupt cannot change its state or start queued work. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Cc: stable@vger.kernel.org Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-5-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index c0d9f7e373f3..7a567f924538 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -255,7 +255,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) err = -EPERM; } else { chan->status = EDMA_ST_BUSY; - dw_edma_start_transfer(chan); + if (!dw_edma_start_transfer(chan)) + chan->status = EDMA_ST_IDLE; } return err; @@ -601,10 +602,16 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) unsigned long flags; spin_lock_irqsave(&chan->vc.lock, flags); + if (chan->status == EDMA_ST_PAUSE) { + spin_unlock_irqrestore(&chan->vc.lock, flags); + return; + } + vd = vchan_next_desc(&chan->vc); if (vd) { switch (chan->request) { case EDMA_REQ_NONE: + case EDMA_REQ_PAUSE: desc = vd2dw_edma_desc(vd); if (desc->start_burst >= desc->nburst) { dw_hdma_set_callback_result(vd, @@ -613,6 +620,12 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) vchan_cookie_complete(vd); } + if (chan->request == EDMA_REQ_PAUSE) { + chan->request = EDMA_REQ_NONE; + chan->status = EDMA_ST_PAUSE; + break; + } + /* Continue transferring if there are remaining chunks or issued requests. */ chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE; @@ -624,11 +637,6 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan) chan->status = EDMA_ST_IDLE; break; - case EDMA_REQ_PAUSE: - chan->request = EDMA_REQ_NONE; - chan->status = EDMA_ST_PAUSE; - break; - default: break; } From f7d1619f3e10c619b62c6cd6d95371b5c526c85a Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:35 +0900 Subject: [PATCH 50/73] dmaengine: dw-edma: Serialize channel state checks pause() and resume() read and update channel state without holding vc.lock, while the interrupt handlers update the same state under it. Take the same lock around those state checks so that request, status, and configured stay consistent. For example, pause() can observe EDMA_ST_BUSY right before the interrupt handler completes the final descriptor and moves the channel to EDMA_ST_IDLE, and then record EDMA_REQ_PAUSE on an already idle channel. No further interrupt will acknowledge the request, and since issue_pending() requires EDMA_REQ_NONE, the channel is wedged for good: terminate_all() leaves the stale request behind, so even reconfiguring the channel does not recover it. issue_pending() already runs under vc.lock, but it tests configured before taking it. Move that test under the lock as well, so configured, request, and status are evaluated as one channel-state snapshot. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-6-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 7a567f924538..cd10f9c6d217 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -230,6 +230,8 @@ static int dw_edma_device_pause(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) err = -EPERM; else if (chan->status != EDMA_ST_BUSY) @@ -247,6 +249,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); int err = 0; + guard(spinlock_irqsave)(&chan->vc.lock); + if (!chan->configured) { err = -EPERM; } else if (chan->status != EDMA_ST_PAUSE) { @@ -297,11 +301,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); unsigned long flags; - if (!chan->configured) - return; - spin_lock_irqsave(&chan->vc.lock, flags); - if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE && + if (chan->configured && vchan_issue_pending(&chan->vc) && + chan->request == EDMA_REQ_NONE && chan->status == EDMA_ST_IDLE) { chan->status = EDMA_ST_BUSY; dw_edma_start_transfer(chan); From c0d9c6275adcca7c0ca5f4270bf88026f9864bd1 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:36 +0900 Subject: [PATCH 51/73] dmaengine: dw-edma: Clear stale requests on termination terminate_all() can finish immediately when the channel is unconfigured, paused, idle, or already stopped in hardware. A pending PAUSE request can survive these paths and block issue_pending() even after termination. Clear the request whenever termination leaves the channel idle. A running channel keeps its STOP request until the interrupt handler consumes it. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-7-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index cd10f9c6d217..ab63d7bddaab 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -292,6 +292,8 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan) } else { chan->request = EDMA_REQ_STOP; } + if (chan->status == EDMA_ST_IDLE) + chan->request = EDMA_REQ_NONE; return err; } From eb4022c1fae2bb029178779bc2b1fe84a9510dbf Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:37 +0900 Subject: [PATCH 52/73] dmaengine: dw-edma-pcie: Drop redundant pci_free_irq_vectors() dw_edma_pcie enables the PCI device with pcim_enable_device(), so IRQ vectors allocated by pci_alloc_irq_vectors() are released by pcim_msi_release() on device release. The driver should not call pci_free_irq_vectors() manually. Drop the redundant remove-time cleanup and rely on the managed PCI device lifetime instead, as documented by commit 03e4905402ae ("PCI/MSI: Clarify pci_free_irq_vectors() usage for managed devices"). Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-8-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index b34e1e45aeb3..db7f1ea35bf3 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -555,9 +555,6 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev) err = dw_edma_remove(chip); if (err) pci_warn(pdev, "can't remove device properly: %d\n", err); - - /* Freeing IRQs */ - pci_free_irq_vectors(pdev); } static const struct pci_device_id dw_edma_pcie_id_table[] = { From 5b0a8818fc5237676bdcd9659180e1edf8e943d7 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:38 +0900 Subject: [PATCH 53/73] dmaengine: dw-edma: Snapshot the v0 interrupt status once per handler pass The v0 interrupt handler reads the interrupt status register twice per invocation, once through the DONE accessor and once through the ABORT accessor, although both fields live in the same 32-bit register. On remote setups (dw-edma-pcie) each read is a non-posted round trip across the PCIe link costing on the order of a microsecond, and with one completion interrupt per element the duplicate adds up. As an example, profiling the R-Car S4 remote path put the handler at ~7us per invocation, dominated by such reads. Read the register once and derive the DONE and ABORT views from the snapshot. No abort is lost to this because the pass only clears status bits it observed, so an abort raised after the snapshot keeps its status and its own interrupt delivery brings it to the next pass. An abort on an observed channel cannot race the clear either. Software can restart the halted channel only after abort() runs, and abort() is called after dw_edma_v0_core_clear_abort_int(). Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-9-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-v0-core.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 7bb3ec461cce..f1abbbacca5e 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -218,18 +218,6 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan) FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id))); } -static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir) -{ - return FIELD_GET(EDMA_V0_DONE_INT_MASK, - GET_RW_32(dw, dir, int_status)); -} - -static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir) -{ - return FIELD_GET(EDMA_V0_ABORT_INT_MASK, - GET_RW_32(dw, dir, int_status)); -} - static irqreturn_t dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, dw_edma_handler_t done, dw_edma_handler_t abort) @@ -240,6 +228,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, struct dw_edma_chan *chan; unsigned long off; unsigned long *mask; + u32 sts; if (dir == EDMA_DIR_WRITE) { total = dw->wr_ch_cnt; @@ -251,7 +240,17 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, mask = dw_irq->rd_mask; } - val = dw_edma_v0_core_status_done_int(dw, dir); + /* + * DONE and ABORT status share one register, and on remote setups + * every read is a non-posted round trip across the PCIe link. Take + * one snapshot and derive both views from it. An abort raised + * after the snapshot is deferred, not lost: only bits observed in + * the snapshot are ever cleared below, so its status remains set and + * triggers another handler pass. + */ + sts = GET_RW_32(dw, dir, int_status); + + val = FIELD_GET(EDMA_V0_DONE_INT_MASK, sts); val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; @@ -262,7 +261,7 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, ret = IRQ_HANDLED; } - val = dw_edma_v0_core_status_abort_int(dw, dir); + val = FIELD_GET(EDMA_V0_ABORT_INT_MASK, sts); val &= *mask; for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; From 35de39e4511f00a87f814b5f0afeb082a8fbc5d7 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Sat, 18 Jul 2026 03:06:39 +0900 Subject: [PATCH 54/73] dmaengine: dw-edma: Defer channel IRQ handling to workqueue On some SoCs (e.g. R-Car S4) the endpoint-side eDMA raises a single fixed SPI that is hardwired to CPU0 and covers every read and write channel. Handling channel events directly in that hard IRQ context serializes the completion processing of all channels on one CPU: descriptor recycling and refill, client callbacks (the vchan tasklet runs on the scheduling CPU) and the doorbell writes all funnel through CPU0, while the handler additionally spins on each channel's vc.lock. Especially under heavy multichannel load, this contention becomes a performance bottleneck. Keep the hard IRQ handler minimal: clear the status, dispatch channel events, and defer per-channel processing to work items. A work item per channel preserves ordering while allowing different channels to run in parallel on any CPU. Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260717180639.2643243-10-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 69 +++++++++++++++++++++++++++--- drivers/dma/dw-edma/dw-edma-core.h | 11 +++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index ab63d7bddaab..e4dde3518b83 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -30,6 +30,11 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd) return container_of(vd, struct dw_edma_desc, vd); } +enum dw_edma_irq_event { + DW_EDMA_IRQ_DONE = BIT(0), + DW_EDMA_IRQ_ABORT = BIT(1), +}; + static inline u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr) { @@ -667,6 +672,39 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan) spin_unlock_irqrestore(&chan->vc.lock, flags); } +static void dw_edma_irq_work(struct work_struct *work) +{ + struct dw_edma_chan *chan = container_of(work, struct dw_edma_chan, + irq_work); + unsigned int events; + + do { + events = atomic_xchg(&chan->irq_pending, 0); + + if (events & DW_EDMA_IRQ_DONE) + dw_edma_done_interrupt(chan); + if (events & DW_EDMA_IRQ_ABORT) + dw_edma_abort_interrupt(chan); + } while (atomic_read(&chan->irq_pending)); +} + +static void dw_edma_queue_irq_work(struct dw_edma_chan *chan, + enum dw_edma_irq_event event) +{ + atomic_or(event, &chan->irq_pending); + queue_work(chan->dw->wq, &chan->irq_work); +} + +static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan) +{ + dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE); +} + +static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan) +{ + dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_ABORT); +} + static void dw_edma_emul_irq_ack(struct irq_data *d) { struct dw_edma *dw = irq_data_get_irq_chip_data(d); @@ -761,8 +799,8 @@ static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data) struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE, - dw_edma_done_interrupt, - dw_edma_abort_interrupt); + dw_edma_done_interrupt_deferred, + dw_edma_abort_interrupt_deferred); } static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) @@ -770,8 +808,8 @@ static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data) struct dw_edma_irq *dw_irq = data; return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ, - dw_edma_done_interrupt, - dw_edma_abort_interrupt); + dw_edma_done_interrupt_deferred, + dw_edma_abort_interrupt_deferred); } static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data) @@ -844,6 +882,8 @@ static void dw_edma_device_synchronize(struct dma_chan *dchan) struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan); dw_edma_wait_termination(dchan); + cancel_work_sync(&chan->irq_work); + atomic_set(&chan->irq_pending, 0); vchan_synchronize(&chan->vc); } @@ -891,6 +931,8 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan->configured = false; chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + INIT_WORK(&chan->irq_work, dw_edma_irq_work); + atomic_set(&chan->irq_pending, 0); if (chan->dir == EDMA_DIR_WRITE) chan->ll_region = chip->ll_region_wr[chan->id]; @@ -1112,10 +1154,21 @@ int dw_edma_probe(struct dw_edma_chip *chip) /* Disable eDMA, only to establish the ideal initial conditions */ dw_edma_core_off(dw); + /* + * Deferred IRQ works are queued from the hard IRQ handlers, so the + * workqueue must exist before any IRQ is requested. + */ + dw->wq = alloc_workqueue("dw-edma:%s", WQ_UNBOUND | WQ_HIGHPRI, 0, + dev_name(chip->dev)); + if (!dw->wq) + return -ENOMEM; + /* Request IRQs */ err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc); - if (err) + if (err) { + destroy_workqueue(dw->wq); return err; + } /* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */ err = dw_edma_emul_irq_alloc(dw); @@ -1138,6 +1191,7 @@ int dw_edma_probe(struct dw_edma_chip *chip) for (i = (dw->nr_irqs - 1); i >= 0; i--) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); dw_edma_emul_irq_free(dw); + destroy_workqueue(dw->wq); return err; } @@ -1162,6 +1216,11 @@ int dw_edma_remove(struct dw_edma_chip *chip) free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]); dw_edma_emul_irq_free(dw); + for (i = 0; i < dw->wr_ch_cnt + dw->rd_ch_cnt; i++) + cancel_work_sync(&dw->chan[i].irq_work); + + destroy_workqueue(dw->wq); + /* Deregister eDMA device */ dma_async_device_unregister(&dw->dma); list_for_each_entry_safe(chan, _chan, &dw->dma.channels, diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index e39c11bc91fb..90ba37737c3a 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -9,8 +9,10 @@ #ifndef _DW_EDMA_CORE_H #define _DW_EDMA_CORE_H +#include #include #include +#include #include "../virt-dma.h" @@ -80,6 +82,9 @@ struct dw_edma_chan { struct dma_slave_config config; bool non_ll; + + struct work_struct irq_work; + atomic_t irq_pending; }; struct dw_edma_irq { @@ -103,6 +108,12 @@ struct dw_edma { struct dw_edma_chan *chan; + /* + * WQ_HIGHPRI keeps completion processing responsive under heavy load; + * WQ_UNBOUND lets different channels run on different CPUs. + */ + struct workqueue_struct *wq; + raw_spinlock_t lock; /* Protect v0 shared registers */ struct dw_edma_chip *chip; From 3e5933f2003e89d733b8d3a3a026ab9af411fbb5 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:02 +0900 Subject: [PATCH 55/73] dmaengine: dw-edma: Factor out HDMA interrupt setup helper The HDMA linked-list and non-linked-list start paths both program the stop/abort interrupt setup register using the same local/remote enable policy. Only the interrupt-mask handling differs by transfer mode. Factor the common setup into dw_hdma_v0_core_int_setup() before adding per-channel interrupt routing support. No functional change intended. Suggested-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-2-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-hdma-v0-core.c | 34 ++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index fe64f9931bba..4df541bf34e4 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -49,6 +49,21 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \ } while (0) +static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val) +{ + if (chan->non_ll) + val |= HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK; + else + val &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + + val |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + val |= HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_REMOTE_ABORT_INT_EN; + + return val; +} + /* HDMA management callbacks */ static void dw_hdma_v0_core_off(struct dw_edma *dw) { @@ -214,11 +229,7 @@ static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan) SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0)); /* Interrupt unmask - stop, abort */ tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup); - tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); - /* Interrupt enable - stop, abort */ - tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; - if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN; + tmp = dw_hdma_v0_core_int_setup(chan, tmp); SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp); /* Channel control */ SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN); @@ -271,17 +282,8 @@ static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan, SET_CH_32(dw, chan->dir, chan->id, transfer_size, child->sz); /* Interrupt setup */ - val = GET_CH_32(dw, chan->dir, chan->id, int_setup) | - HDMA_V0_STOP_INT_MASK | - HDMA_V0_ABORT_INT_MASK | - HDMA_V0_LOCAL_STOP_INT_EN | - HDMA_V0_LOCAL_ABORT_INT_EN; - - if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) { - val |= HDMA_V0_REMOTE_STOP_INT_EN | - HDMA_V0_REMOTE_ABORT_INT_EN; - } - + val = GET_CH_32(dw, chan->dir, chan->id, int_setup); + val = dw_hdma_v0_core_int_setup(chan, val); SET_CH_32(dw, chan->dir, chan->id, int_setup, val); /* Channel control setup */ From 9c317842eceece7efbaf07ac1a08c5d5e8be7ad0 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:03 +0900 Subject: [PATCH 56/73] dmaengine: dw-edma: Add per-channel interrupt routing control DesignWare eDMA can signal completion locally through edma_int[] and remotely through IMWr/MSI. When channels are delegated to a remote frontend, the local endpoint side and the remote host side must not both service the same DONE/ABORT status. Add channel interrupt routing state and initialize it from the controller instance configuration. Update the eDMA and HDMA native paths so linked-list interrupt generation, HDMA non-linked-list interrupt enables, and DONE/ABORT masking follow the selected mode. For HDMA native non-linked-list channels, keep the local stop/abort enables set so status is latched. In remote mode, also enable remote signaling and mask the local interrupt pins. Keep the existing dw-edma-pcie host-side instances in remote interrupt routing mode so their IMWr/MSI completion model remains unchanged after local routing becomes the zero value. Note: - The routing mode describes where a channel should report completion. It does not by itself say whether this dw-edma instance owns the interrupt status. A local instance must ignore remote-only channels, and a remote instance must ignore local-only channels, even if such interrupts are unexpectedly delivered. Otherwise the non-owner side could steal the interrupt from the owner by clearing shared DONE/ABORT status. Cc: Devendra K Verma Suggested-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-3-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 10 +++++++++ drivers/dma/dw-edma/dw-edma-core.h | 13 +++++++++++ drivers/dma/dw-edma/dw-edma-v0-core.c | 30 +++++++++++++++++++++---- drivers/dma/dw-edma/dw-hdma-v0-core.c | 25 ++++++++++++++------- include/linux/dma/edma.h | 32 +++++++++++++++++++++++++++ 5 files changed, 98 insertions(+), 12 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index e4dde3518b83..3a8610862cff 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -168,6 +168,15 @@ static void dw_edma_device_caps(struct dma_chan *dchan, } } +static enum dw_edma_ch_irq_mode +dw_edma_get_default_irq_mode(struct dw_edma_chan *chan) +{ + struct dw_edma_chip *chip = chan->dw->chip; + + return chip->flags & DW_EDMA_CHIP_LOCAL ? DW_EDMA_CH_IRQ_LOCAL : + DW_EDMA_CH_IRQ_REMOTE; +} + static int dw_edma_device_config(struct dma_chan *dchan, struct dma_slave_config *config) { @@ -931,6 +940,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan->configured = false; chan->request = EDMA_REQ_NONE; chan->status = EDMA_ST_IDLE; + chan->irq_mode = dw_edma_get_default_irq_mode(chan); INIT_WORK(&chan->irq_work, dw_edma_irq_work); atomic_set(&chan->irq_pending, 0); diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index 90ba37737c3a..ff20965db4b2 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -76,6 +76,8 @@ struct dw_edma_chan { struct msi_msg msi; + enum dw_edma_ch_irq_mode irq_mode; + enum dw_edma_request request; enum dw_edma_status status; u8 configured; @@ -259,4 +261,15 @@ dw_edma_core_db_offset(struct dw_edma *dw) return dw->core->db_offset(dw); } +static inline bool +dw_edma_core_ch_ignore_irq(struct dw_edma_chan *chan) +{ + struct dw_edma *dw = chan->dw; + + if (dw->chip->flags & DW_EDMA_CHIP_LOCAL) + return chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE; + else + return chan->irq_mode == DW_EDMA_CH_IRQ_LOCAL; +} + #endif /* _DW_EDMA_CORE_H */ diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index f1abbbacca5e..66a605f28a12 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -255,6 +255,9 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; + dw_edma_v0_core_clear_done_int(chan); done(chan); @@ -266,6 +269,9 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, for_each_set_bit(pos, &val, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; + dw_edma_v0_core_clear_abort_int(chan); abort(chan); @@ -353,12 +359,17 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) break; } } - /* Interrupt unmask - done, abort */ + /* Interrupt mask/unmask - done, abort */ raw_spin_lock_irqsave(&dw->lock, flags); tmp = GET_RW_32(dw, chan->dir, int_mask); - tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); - tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) { + tmp |= FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp |= FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + } else { + tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id)); + tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)); + } SET_RW_32(dw, chan->dir, int_mask, tmp); /* Linked list error */ tmp = GET_RW_32(dw, chan->dir, linked_list_err_en); @@ -473,7 +484,18 @@ dw_edma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst, if (irq) { control |= DW_EDMA_V0_LIE; - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) + /* + * A local instance never issues transfers on a remote-routed + * channel: on CHIP_LOCAL instances, REMOTE routing denotes a + * channel handed over to the remote side, which programs the + * linked list through its own instance. The remote-only + * recipe (LIE|RIE with the local interrupt masked) is thus + * applied by the instance that owns the transfer, and the + * LIE-only write below never executes for a remote-routed + * channel. + */ + if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) && + chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) control |= DW_EDMA_V0_RIE; } diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 4df541bf34e4..e9b4bc66886a 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -51,15 +51,22 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch) static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val) { - if (chan->non_ll) - val |= HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK; - else - val &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN | + HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK); - val |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN; - if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL)) - val |= HDMA_V0_REMOTE_STOP_INT_EN | - HDMA_V0_REMOTE_ABORT_INT_EN; + /* + * HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and + * LAIE are enabled. A remote handler needs those status bits to + * identify the source of the IMWr, so keep local generation enabled + * and mask the local interrupt pins instead. + */ + val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN; + + if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) + val |= HDMA_V0_REMOTE_ABORT_INT_EN | + HDMA_V0_REMOTE_STOP_INT_EN | + HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK; return val; } @@ -158,6 +165,8 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, for_each_set_bit(pos, mask, total) { chan = &dw->chan[pos + off]; + if (unlikely(dw_edma_core_ch_ignore_irq(chan))) + continue; val = dw_hdma_v0_core_status_int(chan); if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) { diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index da7a5cc93ad4..17cbe7ce55aa 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -62,6 +62,38 @@ enum dw_edma_chip_flags { DW_EDMA_CHIP_LOCAL = BIT(0), }; +/** + * enum dw_edma_ch_irq_mode - per-channel interrupt routing control + * @DW_EDMA_CH_IRQ_LOCAL: local interrupt only (edma_int[]) + * @DW_EDMA_CH_IRQ_REMOTE: remote interrupt only (IMWr/MSI), without + * delivering local edma_int[]. + * + * DesignWare EP eDMA can signal interrupts locally through the edma_int[] + * bus, and remotely using posted memory writes (IMWr) that may be + * interpreted as MSI/MSI-X by the RC. + * + * For the v0 eDMA linked-list programming path, DMA_*_INT_MASK gates the local + * edma_int[] assertion, while there is no dedicated per-channel mask for IMWr + * generation. To request a remote-only interrupt, Synopsys recommends setting + * both LIE and RIE, and masking the local interrupt in DMA_*_INT_MASK. See the + * DesignWare endpoint databook 6.30a, Linked List Mode interrupt handling + * ("Software Programming of an Endpoint's LIE and RIE Bits for Linked List + * Transfers", Attention). + * + * A local (DW_EDMA_CHIP_LOCAL) instance never issues transfers on a + * remote-routed channel: REMOTE routing on such an instance denotes a channel + * handed over to and driven by the remote side, and the recipe above is + * applied by the driving instance. + * + * HDMA linked-list watermark interrupts have the same LWIE/RWIE guidance. HDMA + * non-linked-list mode has dedicated local and remote stop/abort interrupt + * enables. + */ +enum dw_edma_ch_irq_mode { + DW_EDMA_CH_IRQ_LOCAL = 0, + DW_EDMA_CH_IRQ_REMOTE, +}; + /** * struct dw_edma_chip - representation of DesignWare eDMA controller hardware * @dev: struct device of the eDMA controller From c9c25b2f32601409a53845b41183a29c8260f33e Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:04 +0900 Subject: [PATCH 57/73] dmaengine: dw-edma: Add core quiesce operations Add core operations that quiesce only the resources represented by a dw-edma instance, separate from the existing full controller off path. For unrolled eDMA, quiesce masks the direction, disables ENGINE_EN, waits for it to read back clear, then clears pending interrupt status. If ENGINE_EN does not clear, return the timeout to the caller. HDMA compatibility mode does not implement ENGINE_EN, so quiesce clears CHi_PWR_EN for each represented channel instead. Both register maps share interrupt control per direction, so quiescing one channel quiesces the whole direction. Callers must own that direction and stop the peer from programming it first. HDMA native has per-channel registers and can quiesce the represented channel directly. No caller is added yet, so this is a no-functional-change preparation for delegated channel reclaim and partial-owned remove paths. Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-4-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.h | 14 +++ drivers/dma/dw-edma/dw-edma-v0-core.c | 141 +++++++++++++++++++++----- drivers/dma/dw-edma/dw-hdma-v0-core.c | 31 ++++++ 3 files changed, 158 insertions(+), 28 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index ff20965db4b2..bd3d532ebc89 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -127,6 +127,8 @@ typedef void (*dw_edma_handler_t)(struct dw_edma_chan *); struct dw_edma_core_ops { void (*off)(struct dw_edma *dw); + int (*quiesce)(struct dw_edma *dw); + int (*ch_quiesce)(struct dw_edma_chan *chan); u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir); enum dma_status (*ch_status)(struct dw_edma_chan *chan); irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, @@ -192,6 +194,18 @@ void dw_edma_core_off(struct dw_edma *dw) dw->core->off(dw); } +static inline +int dw_edma_core_quiesce(struct dw_edma *dw) +{ + return dw->core->quiesce(dw); +} + +static inline +int dw_edma_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return chan->dw->core->ch_quiesce(chan); +} + static inline u16 dw_edma_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 66a605f28a12..13e69ce6c41d 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include @@ -160,6 +161,87 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) /* eDMA management callbacks */ +static void dw_edma_v0_core_ch_power(struct dw_edma *dw, + enum dw_edma_dir dir, u16 id, bool enable) +{ + u32 value = enable ? BIT(0) : 0; + + if (WARN_ON_ONCE(id >= EDMA_V0_MAX_NR_CH)) + return; + + switch (id) { + case 0: + SET_RW_COMPAT(dw, dir, ch0_pwr_en, value); + break; + case 1: + SET_RW_COMPAT(dw, dir, ch1_pwr_en, value); + break; + case 2: + SET_RW_COMPAT(dw, dir, ch2_pwr_en, value); + break; + case 3: + SET_RW_COMPAT(dw, dir, ch3_pwr_en, value); + break; + case 4: + SET_RW_COMPAT(dw, dir, ch4_pwr_en, value); + break; + case 5: + SET_RW_COMPAT(dw, dir, ch5_pwr_en, value); + break; + case 6: + SET_RW_COMPAT(dw, dir, ch6_pwr_en, value); + break; + case 7: + SET_RW_COMPAT(dw, dir, ch7_pwr_en, value); + break; + } +} + +static int dw_edma_v0_core_engine_disable(struct dw_edma *dw, + enum dw_edma_dir dir) +{ + u32 value; + int ret; + + SET_RW_32(dw, dir, engine_en, 0); + ret = read_poll_timeout(GET_RW_32, value, !(value & BIT(0)), 100, + 200000, false, dw, dir, engine_en); + if (ret) + dev_warn(dw->chip->dev, "%s engine did not stop within 200ms\n", + dir == EDMA_DIR_WRITE ? "write" : "read"); + + return ret; +} + +static int dw_edma_v0_core_dir_off(struct dw_edma *dw, enum dw_edma_dir dir) +{ + u16 count, id; + int ret = 0; + + scoped_guard(raw_spinlock_irqsave, &dw->lock) + SET_RW_32(dw, dir, int_mask, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { + /* + * DWC PCIe Controller Databook 6.10a-lca06, "Legacy DMA + * and HDMA Software Compatibility": HDMA compatibility mode + * does not implement ENGINE_EN, but retains CHi_PWR_EN for + * per-channel enable and disable. + */ + count = dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt; + for (id = 0; id < count; id++) + dw_edma_v0_core_ch_power(dw, dir, id, false); + } else { + ret = dw_edma_v0_core_engine_disable(dw, dir); + } + + SET_RW_32(dw, dir, int_clear, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + return ret; +} + static void dw_edma_v0_core_off(struct dw_edma *dw) { SET_BOTH_32(dw, int_mask, @@ -169,6 +251,33 @@ static void dw_edma_v0_core_off(struct dw_edma *dw) SET_BOTH_32(dw, engine_en, 0); } +static int dw_edma_v0_core_quiesce(struct dw_edma *dw) +{ + int ret = 0; + int err; + + if (dw->wr_ch_cnt) + ret = dw_edma_v0_core_dir_off(dw, EDMA_DIR_WRITE); + if (dw->rd_ch_cnt) { + err = dw_edma_v0_core_dir_off(dw, EDMA_DIR_READ); + if (!ret) + ret = err; + } + + return ret; +} + +/* + * The unrolled eDMA and HDMA compatibility register maps share interrupt + * control per direction, so the whole direction is quiesced. Callers must + * own the direction entirely and prevent the peer from programming it after + * this point. Partial ownership mode validates direction granularity. + */ +static int dw_edma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return dw_edma_v0_core_dir_off(chan->dw, chan->dir); +} + static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { u32 num_ch; @@ -331,34 +440,8 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) /* Enable engine */ SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { - switch (chan->id) { - case 0: - SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, BIT(0)); - break; - case 1: - SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, BIT(0)); - break; - case 2: - SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, BIT(0)); - break; - case 3: - SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, BIT(0)); - break; - case 4: - SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, BIT(0)); - break; - case 5: - SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, BIT(0)); - break; - case 6: - SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, BIT(0)); - break; - case 7: - SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, BIT(0)); - break; - } - } + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) + dw_edma_v0_core_ch_power(dw, chan->dir, chan->id, true); /* Interrupt mask/unmask - done, abort */ raw_spin_lock_irqsave(&dw->lock, flags); @@ -552,6 +635,8 @@ static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw) static const struct dw_edma_core_ops dw_edma_v0_core = { .off = dw_edma_v0_core_off, + .quiesce = dw_edma_v0_core_quiesce, + .ch_quiesce = dw_edma_v0_core_ch_quiesce, .ch_count = dw_edma_v0_core_ch_count, .ch_status = dw_edma_v0_core_ch_status, .handle_int = dw_edma_v0_core_handle_int, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index e9b4bc66886a..8d20bb028b52 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -72,6 +72,16 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val) } /* HDMA management callbacks */ +static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir, + u16 id) +{ + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); +} + static void dw_hdma_v0_core_off(struct dw_edma *dw) { int id; @@ -96,6 +106,25 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw) } } +static int dw_hdma_v0_core_quiesce(struct dw_edma *dw) +{ + int id; + + for (id = 0; id < dw->wr_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_WRITE, id); + for (id = 0; id < dw->rd_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_READ, id); + + return 0; +} + +static int dw_hdma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + dw_hdma_v0_core_ch_off(chan->dw, chan->dir, chan->id); + + return 0; +} + static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { /* @@ -366,6 +395,8 @@ static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw) static const struct dw_edma_core_ops dw_hdma_v0_core = { .off = dw_hdma_v0_core_off, + .quiesce = dw_hdma_v0_core_quiesce, + .ch_quiesce = dw_hdma_v0_core_ch_quiesce, .ch_count = dw_hdma_v0_core_ch_count, .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int, From 647217abea849d3d45f8cb0b8ee5b78d50f26985 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:05 +0900 Subject: [PATCH 58/73] dmaengine: dw-edma: Initialize IRQ data before requesting IRQs dw_edma_irq_request() passes struct dw_edma_irq to request_irq() before dw_edma_channel_setup() fills the back pointer. A shared interrupt can therefore enter the handler with dw_irq->dw still NULL, leading to a NULL pointer dereference. Set the back pointer before installing each handler. Fixes: e63d79d1ffcd ("dmaengine: Add Synopsys eDMA IP core driver") Cc: stable@vger.kernel.org Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-5-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 3a8610862cff..ee9a8c32adf4 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -969,7 +969,6 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) else bitmap_set(irq->rd_mask, chan->id, 1); - irq->dw = dw; memcpy(&chan->msi, &irq->msi, sizeof(chan->msi)); dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n", @@ -1051,6 +1050,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, if (chip->nr_irqs == 1) { /* Common IRQ shared among all channels */ irq = chip->ops->irq_vector(dev, 0); + dw->irq[0].dw = dw; err = request_irq(irq, dw_edma_interrupt_common, IRQF_SHARED, dw->name, &dw->irq[0]); if (err) { @@ -1073,6 +1073,7 @@ static int dw_edma_irq_request(struct dw_edma *dw, for (i = 0; i < (*wr_alloc + *rd_alloc); i++) { irq = chip->ops->irq_vector(dev, i); + dw->irq[i].dw = dw; err = request_irq(irq, i < *wr_alloc ? dw_edma_interrupt_write : From b24a5e9db6863567df1a10fcd29e01467672c56e Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:06 +0900 Subject: [PATCH 59/73] dmaengine: dw-edma: Add partial channel ownership mode A DesignWare eDMA instance may represent only a subset of channels that is also initialized by another OS instance, such as an endpoint-side OS. Add a partial ownership flag for instances that must preserve controller-wide state owned by that peer. In partial ownership mode, dw-edma skips the initial core reset and uses the limited quiesce path in probe() and remove() instead of the full core-off path. The flag also makes the driver validate the ownership granularity required by each register layout before registering channels. Probe fails if the limited quiesce cannot stop its resources; remove reports the error after completing the remaining driver teardown. Partial instances also skip interrupt-emulation doorbell allocation: the emulated doorbell is a controller-level resource, and a partial owner must not claim it on behalf of the whole block. For EDMA_MF_EDMA_UNROLL and EDMA_MF_HDMA_COMPAT, the driver programs per-direction registers, such as DMA_{WRITE,READ}_INT_MASK_OFF and DMA_{WRITE,READ}_INT_CLEAR_OFF. These register layouts have at most EDMA_MAX_{WR,RD}_CH channels per direction, so the capped hardware channel count still represents the whole direction. A partial instance can therefore expose write or read channels only if it owns every channel in that direction; otherwise two OS instances could update the same direction-wide registers without a shared locking protocol. In contrast, HDMA native uses per-channel registers, so it can be owned at channel granularity. Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-6-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 81 +++++++++++++++++++++++++----- include/linux/dma/edma.h | 7 +++ 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index ee9a8c32adf4..c9df5bbe2392 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -745,6 +745,9 @@ static int dw_edma_emul_irq_alloc(struct dw_edma *dw) chip->db_irq = 0; chip->db_offset = ~0; + if (chip->flags & DW_EDMA_CHIP_PARTIAL) + return 0; + /* * Only meaningful when the core provides the deassert sequence * for interrupt emulation. @@ -1104,10 +1107,33 @@ static int dw_edma_irq_request(struct dw_edma *dw, return err; } +static int dw_edma_check_partial(struct dw_edma_chip *chip, + u16 hw_wr_ch_cnt, u16 hw_rd_ch_cnt) +{ + if (!(chip->flags & DW_EDMA_CHIP_PARTIAL)) + return 0; + + if (chip->mf != EDMA_MF_EDMA_UNROLL && + chip->mf != EDMA_MF_HDMA_COMPAT) + return 0; + + /* + * Direction-wide registers are shared by all channels in that + * direction, so a direction must have a single owner. + */ + if ((chip->ll_wr_cnt && chip->ll_wr_cnt != hw_wr_ch_cnt) || + (chip->ll_rd_cnt && chip->ll_rd_cnt != hw_rd_ch_cnt)) + return -EOPNOTSUPP; + + return 0; +} + int dw_edma_probe(struct dw_edma_chip *chip) { struct device *dev; struct dw_edma *dw; + u16 hw_wr_ch_cnt; + u16 hw_rd_ch_cnt; u32 wr_alloc = 0; u32 rd_alloc = 0; u16 max_wr_cnt; @@ -1121,6 +1147,17 @@ int dw_edma_probe(struct dw_edma_chip *chip) if (!dev || !chip->ops) return -EINVAL; + if (chip->flags & DW_EDMA_CHIP_PARTIAL) { + switch (chip->mf) { + case EDMA_MF_EDMA_UNROLL: + case EDMA_MF_HDMA_COMPAT: + case EDMA_MF_HDMA_NATIVE: + break; + default: + return -EOPNOTSUPP; + } + } + dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL); if (!dw) return -ENOMEM; @@ -1139,13 +1176,21 @@ int dw_edma_probe(struct dw_edma_chip *chip) raw_spin_lock_init(&dw->lock); - dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt, - dw_edma_core_ch_count(dw, EDMA_DIR_WRITE)); - dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, max_wr_cnt); + /* + * chip->ll_*_cnt describes the channels exposed by this instance. Keep + * the usable hardware counts separate for partial ownership checks. + */ + hw_wr_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_WRITE), + max_wr_cnt); + hw_rd_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_READ), + max_rd_cnt); - dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt, - dw_edma_core_ch_count(dw, EDMA_DIR_READ)); - dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, max_rd_cnt); + err = dw_edma_check_partial(chip, hw_wr_ch_cnt, hw_rd_ch_cnt); + if (err) + return err; + + dw->wr_ch_cnt = min(chip->ll_wr_cnt, hw_wr_ch_cnt); + dw->rd_ch_cnt = min(chip->ll_rd_cnt, hw_rd_ch_cnt); if (!dw->wr_ch_cnt && !dw->rd_ch_cnt) return -EINVAL; @@ -1162,8 +1207,18 @@ int dw_edma_probe(struct dw_edma_chip *chip) snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s", dev_name(chip->dev)); - /* Disable eDMA, only to establish the ideal initial conditions */ - dw_edma_core_off(dw); + if (chip->flags & DW_EDMA_CHIP_PARTIAL) { + /* + * Do not reset the shared controller, but drain stale state + * from resources represented by this instance. + */ + err = dw_edma_core_quiesce(dw); + if (err) + return err; + } else { + /* Disable eDMA only when this instance owns the controller. */ + dw_edma_core_off(dw); + } /* * Deferred IRQ works are queued from the hard IRQ handlers, so the @@ -1213,14 +1268,16 @@ int dw_edma_remove(struct dw_edma_chip *chip) struct dw_edma_chan *chan, *_chan; struct device *dev = chip->dev; struct dw_edma *dw = chip->dw; - int i; + int i, err = 0; /* Skip removal if no private data found */ if (!dw) return -ENODEV; - /* Disable eDMA */ - dw_edma_core_off(dw); + if (chip->flags & DW_EDMA_CHIP_PARTIAL) + err = dw_edma_core_quiesce(dw); + else + dw_edma_core_off(dw); /* Free irqs */ for (i = (dw->nr_irqs - 1); i >= 0; i--) @@ -1240,7 +1297,7 @@ int dw_edma_remove(struct dw_edma_chip *chip) list_del(&chan->vc.chan.device_node); } - return 0; + return err; } EXPORT_SYMBOL_GPL(dw_edma_remove); diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 17cbe7ce55aa..2ce9d2d49aef 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -57,9 +57,16 @@ enum dw_edma_map_format { /** * enum dw_edma_chip_flags - Flags specific to an eDMA chip * @DW_EDMA_CHIP_LOCAL: eDMA is used locally by an endpoint + * @DW_EDMA_CHIP_PARTIAL: Only channels described by this instance are + * owned by this driver. Controller-wide state + * must be preserved, and layouts with shared + * direction-wide registers must only be shared at + * direction granularity. Layouts with per-channel + * registers may be shared at channel granularity. */ enum dw_edma_chip_flags { DW_EDMA_CHIP_LOCAL = BIT(0), + DW_EDMA_CHIP_PARTIAL = BIT(1), }; /** From 1415d6d1f101398441da43e82a076afe1c5aebd3 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:07 +0900 Subject: [PATCH 60/73] dmaengine: dw-edma-pcie: Track non-LL mode in DMA data The dw-edma-pcie driver copies static template data into a mutable dw_edma_pcie_data instance before applying capability-derived updates. Keep the derived non-LL mode in that copy as well, instead of only tracking it in a local variable in dw_edma_pcie_probe(). This prepares for keeping capability parsing behind match data without a separate non-LL output parameter. No functional change intended. Suggested-by: Frank Li Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-7-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index db7f1ea35bf3..9b3d3df2ffcd 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -73,6 +73,7 @@ struct dw_edma_pcie_data { u16 wr_ch_cnt; u16 rd_ch_cnt; u64 devmem_phys_off; + bool cfg_non_ll; }; static const struct dw_edma_pcie_data snps_edda_data = { @@ -326,7 +327,6 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; - bool non_ll = false; if (!pdata) return -ENODEV; @@ -361,14 +361,14 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, * the HDMA IP. */ if (vsec_data->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) - non_ll = true; + vsec_data->cfg_non_ll = true; /* * Configure the channel LL and data blocks if number of * channels enabled in VSEC capability are more than the * channels configured in xilinx_mdb_data. */ - if (!non_ll) + if (!vsec_data->cfg_non_ll) dw_edma_set_chan_region_offset(vsec_data, BAR_2, 0, DW_PCIE_XILINX_MDB_LL_OFF_GAP, DW_PCIE_XILINX_MDB_LL_SIZE, @@ -421,7 +421,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->mf = vsec_data->mf; chip->nr_irqs = nr_irqs; chip->ops = &dw_edma_pcie_plat_ops; - chip->cfg_non_ll = non_ll; + chip->cfg_non_ll = vsec_data->cfg_non_ll; chip->ll_wr_cnt = vsec_data->wr_ch_cnt; chip->ll_rd_cnt = vsec_data->rd_ch_cnt; @@ -430,7 +430,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, if (!chip->reg_base) return -ENOMEM; - for (i = 0; i < chip->ll_wr_cnt && !non_ll; i++) { + for (i = 0; i < chip->ll_wr_cnt && !vsec_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; @@ -457,7 +457,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dt_region->sz = dt_block->sz; } - for (i = 0; i < chip->ll_rd_cnt && !non_ll; i++) { + for (i = 0; i < chip->ll_rd_cnt && !vsec_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; From f3468b998a00c1491b774af188d5f741c64a1c71 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:08 +0900 Subject: [PATCH 61/73] dmaengine: dw-edma-pcie: Add capability match data Move device-specific capability parsing behind per-device match data. The existing probe path mixes two decisions: which static template a PCI ID uses, and which device-specific capability parser adjusts that template. Split those decisions so device-specific discovery can be added through match data instead of adding more vendor checks to dw_edma_pcie_probe(). No functional change is intended for the existing Synopsys EDDA and AMD (Xilinx) MDB/CPM6 matches. They still copy the same static template data and run the same capability parsing logic before BAR mapping. The AMD (Xilinx) MDB/CPM6 entries also keep using endpoint memory physical addresses for descriptor windows through a new match-data flag. Suggested-by: Frank Li Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-8-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 137 ++++++++++++++++++++--------- 1 file changed, 95 insertions(+), 42 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 9b3d3df2ffcd..50d2a53f798d 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -76,6 +76,19 @@ struct dw_edma_pcie_data { bool cfg_non_ll; }; +struct dw_edma_pcie_match_data { + const struct dw_edma_pcie_data *data; + /* + * Mandatory callback. It may leave @pdata unchanged when the static + * template already describes the device. + */ + int (*parse_caps)(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata); + unsigned long flags; +}; + +#define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0) + static const struct dw_edma_pcie_data snps_edda_data = { /* eDMA registers location */ .rg.bar = BAR_0, @@ -310,24 +323,70 @@ static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev, pdata->devmem_phys_off = off; } +static int +dw_edma_pcie_parse_synopsys_caps(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + dw_edma_pcie_get_synopsys_dma_data(pdev, pdata); + + return 0; +} + +static int +dw_edma_pcie_parse_xilinx_caps(struct pci_dev *pdev, + struct dw_edma_pcie_data *pdata) +{ + dw_edma_pcie_get_xilinx_dma_data(pdev, pdata); + + /* + * There is no valid address found for the LL memory space on the + * device side. In the absence of LL base address use the non-LL mode or + * simple mode supported by the HDMA IP. + */ + if (pdata->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) { + pdata->cfg_non_ll = true; + return 0; + } + + /* + * Configure the channel LL and data blocks if number of channels + * enabled in VSEC capability are more than the channels configured in + * xilinx_mdb_data. + */ + dw_edma_set_chan_region_offset(pdata, BAR_2, 0, + DW_PCIE_XILINX_MDB_LL_OFF_GAP, + DW_PCIE_XILINX_MDB_LL_SIZE, + DW_PCIE_XILINX_MDB_DT_OFF_GAP, + DW_PCIE_XILINX_MDB_DT_SIZE); + + return 0; +} + static u64 dw_edma_get_phys_addr(struct pci_dev *pdev, + const struct dw_edma_pcie_match_data *match, struct dw_edma_pcie_data *pdata, enum pci_barno bar) { - if (pdev->vendor == PCI_VENDOR_ID_XILINX) + if (match->flags & DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF) return pdata->devmem_phys_off; + return pci_bus_address(pdev, bar); } static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { - struct dw_edma_pcie_data *pdata = (void *)pid->driver_data; + const struct dw_edma_pcie_match_data *match = (void *)pid->driver_data; + const struct dw_edma_pcie_data *pdata; struct device *dev = &pdev->dev; struct dw_edma_chip *chip; int err, nr_irqs; int i, mask; + if (!match) + return -ENODEV; + pdata = match->data; + if (!pdata) return -ENODEV; @@ -345,36 +404,13 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); - /* - * Tries to find if exists a PCIe Vendor-Specific Extended Capability - * for the DMA, if one exists, then reconfigures it. - */ - dw_edma_pcie_get_synopsys_dma_data(pdev, vsec_data); + /* Let device-specific discovery override the static template data. */ + if (!match->parse_caps) + return -EINVAL; - if (pdev->vendor == PCI_VENDOR_ID_XILINX) { - dw_edma_pcie_get_xilinx_dma_data(pdev, vsec_data); - - /* - * There is no valid address found for the LL memory - * space on the device side. In the absence of LL base - * address use the non-LL mode or simple mode supported by - * the HDMA IP. - */ - if (vsec_data->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) - vsec_data->cfg_non_ll = true; - - /* - * Configure the channel LL and data blocks if number of - * channels enabled in VSEC capability are more than the - * channels configured in xilinx_mdb_data. - */ - if (!vsec_data->cfg_non_ll) - dw_edma_set_chan_region_offset(vsec_data, BAR_2, 0, - DW_PCIE_XILINX_MDB_LL_OFF_GAP, - DW_PCIE_XILINX_MDB_LL_SIZE, - DW_PCIE_XILINX_MDB_DT_OFF_GAP, - DW_PCIE_XILINX_MDB_DT_SIZE); - } + err = match->parse_caps(pdev, vsec_data); + if (err) + return err; /* Mapping PCI BAR regions */ mask = BIT(vsec_data->rg.bar); @@ -441,8 +477,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, - ll_block->bar); + ll_region->paddr = dw_edma_get_phys_addr(pdev, match, + vsec_data, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -451,8 +487,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, - dt_block->bar); + dt_region->paddr = dw_edma_get_phys_addr(pdev, match, + vsec_data, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } @@ -468,8 +504,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, - ll_block->bar); + ll_region->paddr = dw_edma_get_phys_addr(pdev, match, + vsec_data, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -478,8 +514,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data, - dt_block->bar); + dt_region->paddr = dw_edma_get_phys_addr(pdev, match, + vsec_data, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } @@ -557,12 +593,29 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev) pci_warn(pdev, "can't remove device properly: %d\n", err); } +static const struct dw_edma_pcie_match_data snps_edda_match_data = { + .data = &snps_edda_data, + .parse_caps = dw_edma_pcie_parse_synopsys_caps, +}; + +static const struct dw_edma_pcie_match_data xilinx_mdb_match_data = { + .data = &xilinx_mdb_data, + .parse_caps = dw_edma_pcie_parse_xilinx_caps, + .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, +}; + +static const struct dw_edma_pcie_match_data xilinx_cpm6_dma_match_data = { + .data = &xilinx_cpm6_dma_data, + .parse_caps = dw_edma_pcie_parse_xilinx_caps, + .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, +}; + static const struct pci_device_id dw_edma_pcie_id_table[] = { - { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) }, + { PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_match_data) }, { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054), - (kernel_ulong_t)&xilinx_mdb_data }, + .driver_data = (kernel_ulong_t)&xilinx_mdb_match_data }, { PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B00F), - .driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_data }, + .driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_match_data }, { } }; MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table); From 6cb89b38dcd7e2144c7af8c6c6bc9d30e84b73c9 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:09 +0900 Subject: [PATCH 62/73] dmaengine: dw-edma-pcie: Rename vsec_data to dma_data dw_edma_pcie_probe() now obtains DMA layout data through device-specific capability callbacks, not only from PCIe Vendor-Specific Extended Capabilities. Rename the local data copy from vsec_data to dma_data before adding endpoint DMA BAR metadata discovery, which does not rely on VSEC. No functional change intended. Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-9-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 74 +++++++++++++++--------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 50d2a53f798d..f79597a7ccf9 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -390,9 +390,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, if (!pdata) return -ENODEV; - struct dw_edma_pcie_data *vsec_data __free(kfree) = - kmalloc_obj(*vsec_data); - if (!vsec_data) + struct dw_edma_pcie_data *dma_data __free(kfree) = + kmemdup(pdata, sizeof(*dma_data), GFP_KERNEL); + if (!dma_data) return -ENOMEM; /* Enable PCI device */ @@ -402,25 +402,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return err; } - memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data)); - /* Let device-specific discovery override the static template data. */ if (!match->parse_caps) return -EINVAL; - err = match->parse_caps(pdev, vsec_data); + err = match->parse_caps(pdev, dma_data); if (err) return err; /* Mapping PCI BAR regions */ - mask = BIT(vsec_data->rg.bar); - for (i = 0; i < vsec_data->wr_ch_cnt; i++) { - mask |= BIT(vsec_data->ll_wr[i].bar); - mask |= BIT(vsec_data->dt_wr[i].bar); + mask = BIT(dma_data->rg.bar); + for (i = 0; i < dma_data->wr_ch_cnt; i++) { + mask |= BIT(dma_data->ll_wr[i].bar); + mask |= BIT(dma_data->dt_wr[i].bar); } - for (i = 0; i < vsec_data->rd_ch_cnt; i++) { - mask |= BIT(vsec_data->ll_rd[i].bar); - mask |= BIT(vsec_data->dt_rd[i].bar); + for (i = 0; i < dma_data->rd_ch_cnt; i++) { + mask |= BIT(dma_data->ll_rd[i].bar); + mask |= BIT(dma_data->dt_rd[i].bar); } err = pcim_iomap_regions(pdev, mask, pci_name(pdev)); if (err) { @@ -443,7 +441,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; /* IRQs allocation */ - nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs, + nr_irqs = pci_alloc_irq_vectors(pdev, 1, dma_data->irqs, PCI_IRQ_MSI | PCI_IRQ_MSIX); if (nr_irqs < 1) { pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n", @@ -454,23 +452,23 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, /* Data structure initialization */ chip->dev = dev; - chip->mf = vsec_data->mf; + chip->mf = dma_data->mf; chip->nr_irqs = nr_irqs; chip->ops = &dw_edma_pcie_plat_ops; - chip->cfg_non_ll = vsec_data->cfg_non_ll; + chip->cfg_non_ll = dma_data->cfg_non_ll; - chip->ll_wr_cnt = vsec_data->wr_ch_cnt; - chip->ll_rd_cnt = vsec_data->rd_ch_cnt; + chip->ll_wr_cnt = dma_data->wr_ch_cnt; + chip->ll_rd_cnt = dma_data->rd_ch_cnt; - chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar]; + chip->reg_base = pcim_iomap_table(pdev)[dma_data->rg.bar]; if (!chip->reg_base) return -ENOMEM; - for (i = 0; i < chip->ll_wr_cnt && !vsec_data->cfg_non_ll; i++) { + for (i = 0; i < chip->ll_wr_cnt && !dma_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; struct dw_edma_region *dt_region = &chip->dt_region_wr[i]; - struct dw_edma_block *ll_block = &vsec_data->ll_wr[i]; - struct dw_edma_block *dt_block = &vsec_data->dt_wr[i]; + struct dw_edma_block *ll_block = &dma_data->ll_wr[i]; + struct dw_edma_block *dt_block = &dma_data->dt_wr[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) @@ -478,7 +476,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, ll_region->vaddr.io += ll_block->off; ll_region->paddr = dw_edma_get_phys_addr(pdev, match, - vsec_data, ll_block->bar); + dma_data, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -488,16 +486,16 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dt_region->vaddr.io += dt_block->off; dt_region->paddr = dw_edma_get_phys_addr(pdev, match, - vsec_data, dt_block->bar); + dma_data, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } - for (i = 0; i < chip->ll_rd_cnt && !vsec_data->cfg_non_ll; i++) { + for (i = 0; i < chip->ll_rd_cnt && !dma_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_rd[i]; struct dw_edma_region *dt_region = &chip->dt_region_rd[i]; - struct dw_edma_block *ll_block = &vsec_data->ll_rd[i]; - struct dw_edma_block *dt_block = &vsec_data->dt_rd[i]; + struct dw_edma_block *ll_block = &dma_data->ll_rd[i]; + struct dw_edma_block *dt_block = &dma_data->dt_rd[i]; ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar]; if (!ll_region->vaddr.io) @@ -505,7 +503,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, ll_region->vaddr.io += ll_block->off; ll_region->paddr = dw_edma_get_phys_addr(pdev, match, - vsec_data, ll_block->bar); + dma_data, ll_block->bar); ll_region->paddr += ll_block->off; ll_region->sz = ll_block->sz; @@ -515,7 +513,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dt_region->vaddr.io += dt_block->off; dt_region->paddr = dw_edma_get_phys_addr(pdev, match, - vsec_data, dt_block->bar); + dma_data, dt_block->bar); dt_region->paddr += dt_block->off; dt_region->sz = dt_block->sz; } @@ -533,31 +531,31 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf); pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n", - vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz, + dma_data->rg.bar, dma_data->rg.off, dma_data->rg.sz, chip->reg_base); for (i = 0; i < chip->ll_wr_cnt; i++) { pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->ll_wr[i].bar, - vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz, + i, dma_data->ll_wr[i].bar, + dma_data->ll_wr[i].off, chip->ll_region_wr[i].sz, chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr); pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->dt_wr[i].bar, - vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz, + i, dma_data->dt_wr[i].bar, + dma_data->dt_wr[i].off, chip->dt_region_wr[i].sz, chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr); } for (i = 0; i < chip->ll_rd_cnt; i++) { pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->ll_rd[i].bar, - vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz, + i, dma_data->ll_rd[i].bar, + dma_data->ll_rd[i].off, chip->ll_region_rd[i].sz, chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr); pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", - i, vsec_data->dt_rd[i].bar, - vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz, + i, dma_data->dt_rd[i].bar, + dma_data->dt_rd[i].off, chip->dt_region_rd[i].sz, chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr); } From f8850e66ae8b3f2809b665375ba9a7106a4779f8 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:10 +0900 Subject: [PATCH 63/73] dmaengine: dw-edma-pcie: Add platform ops to match data Move the platform ops pointer into match data. Existing EDDA/MDB/CPM6 matches keep using dw_edma_pcie_plat_ops. No functional changes intended. Suggested-by: Frank Li Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-10-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index f79597a7ccf9..99ddb6e6da34 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -78,6 +78,7 @@ struct dw_edma_pcie_data { struct dw_edma_pcie_match_data { const struct dw_edma_pcie_data *data; + const struct dw_edma_plat_ops *plat_ops; /* * Mandatory callback. It may leave @pdata unchanged when the static * template already describes the device. @@ -403,7 +404,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, } /* Let device-specific discovery override the static template data. */ - if (!match->parse_caps) + if (!match->parse_caps || !match->plat_ops) return -EINVAL; err = match->parse_caps(pdev, dma_data); @@ -454,7 +455,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->mf = dma_data->mf; chip->nr_irqs = nr_irqs; - chip->ops = &dw_edma_pcie_plat_ops; + chip->ops = match->plat_ops; chip->cfg_non_ll = dma_data->cfg_non_ll; chip->ll_wr_cnt = dma_data->wr_ch_cnt; @@ -593,17 +594,20 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev) static const struct dw_edma_pcie_match_data snps_edda_match_data = { .data = &snps_edda_data, + .plat_ops = &dw_edma_pcie_plat_ops, .parse_caps = dw_edma_pcie_parse_synopsys_caps, }; static const struct dw_edma_pcie_match_data xilinx_mdb_match_data = { .data = &xilinx_mdb_data, + .plat_ops = &dw_edma_pcie_plat_ops, .parse_caps = dw_edma_pcie_parse_xilinx_caps, .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, }; static const struct dw_edma_pcie_match_data xilinx_cpm6_dma_match_data = { .data = &xilinx_cpm6_dma_data, + .plat_ops = &dw_edma_pcie_plat_ops, .parse_caps = dw_edma_pcie_parse_xilinx_caps, .flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF, }; From 9504c24325ede003d111cd004e56803969764355 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:11 +0900 Subject: [PATCH 64/73] dmaengine: dw-edma-pcie: Add register offset match flag Add a match-data flag for devices whose DMA register block starts at an offset inside the mapped BAR. Existing Synopsys EDDA and AMD (Xilinx) MDB/CPM6 matches keep using the BAR mapping base directly. The offsets handled here come from static, trusted match data. Later metadata-derived offsets are validated when the endpoint DMA metadata path parses them at that trust boundary. No functional change intended. Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-11-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 99ddb6e6da34..4b2757c1eda2 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -89,6 +89,7 @@ struct dw_edma_pcie_match_data { }; #define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0) +#define DW_EDMA_PCIE_F_REG_OFFSET BIT(1) static const struct dw_edma_pcie_data snps_edda_data = { /* eDMA registers location */ @@ -464,6 +465,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->reg_base = pcim_iomap_table(pdev)[dma_data->rg.bar]; if (!chip->reg_base) return -ENOMEM; + if (match->flags & DW_EDMA_PCIE_F_REG_OFFSET) + chip->reg_base += dma_data->rg.off; for (i = 0; i < chip->ll_wr_cnt && !dma_data->cfg_non_ll; i++) { struct dw_edma_region *ll_region = &chip->ll_region_wr[i]; From 6f3328acabc4b78de2545bc4b4d2c8c7a4988624 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:12 +0900 Subject: [PATCH 65/73] dmaengine: dw-edma-pcie: Factor out descriptor block address lookup Add an optional physical address override to struct dw_edma_block and use a helper to compute descriptor block addresses. No functional change intended. Existing Synopsys EDDA and AMD (Xilinx) MDB/CPM6 block descriptors leave the override unset, so the helper still returns the same values as before. Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-12-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 34 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 4b2757c1eda2..6da890a4ad28 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -55,6 +55,8 @@ struct dw_edma_block { enum pci_barno bar; off_t off; + u64 paddr; + bool paddr_valid; size_t sz; }; @@ -375,6 +377,18 @@ static u64 dw_edma_get_phys_addr(struct pci_dev *pdev, return pci_bus_address(pdev, bar); } +static u64 dw_edma_get_block_addr(struct pci_dev *pdev, + const struct dw_edma_pcie_match_data *match, + struct dw_edma_pcie_data *pdata, + const struct dw_edma_block *block) +{ + if (block->paddr_valid) + return block->paddr; + + return dw_edma_get_phys_addr(pdev, match, pdata, block->bar) + + block->off; +} + static int dw_edma_pcie_probe(struct pci_dev *pdev, const struct pci_device_id *pid) { @@ -479,9 +493,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = dw_edma_get_phys_addr(pdev, match, - dma_data, ll_block->bar); - ll_region->paddr += ll_block->off; + ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + ll_block); ll_region->sz = ll_block->sz; dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; @@ -489,9 +502,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = dw_edma_get_phys_addr(pdev, match, - dma_data, dt_block->bar); - dt_region->paddr += dt_block->off; + dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + dt_block); dt_region->sz = dt_block->sz; } @@ -506,9 +518,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; ll_region->vaddr.io += ll_block->off; - ll_region->paddr = dw_edma_get_phys_addr(pdev, match, - dma_data, ll_block->bar); - ll_region->paddr += ll_block->off; + ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + ll_block); ll_region->sz = ll_block->sz; dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; @@ -516,9 +527,8 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, return -ENOMEM; dt_region->vaddr.io += dt_block->off; - dt_region->paddr = dw_edma_get_phys_addr(pdev, match, - dma_data, dt_block->bar); - dt_region->paddr += dt_block->off; + dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data, + dt_block); dt_region->sz = dt_block->sz; } From 81964f539601f3dcbf1ed64979aa9b5ae0e4adf4 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:13 +0900 Subject: [PATCH 66/73] dmaengine: dw-edma-pcie: Handle optional data blocks Skip data block BAR mapping and debug output when a channel has no data block size. This lets future providers describe channels that only need descriptor memory exposed. No functional change intended for existing Synopsys EDDA and AMD (Xilinx) MDB/CPM6 devices. Their static channel descriptions still provide data block sizes where data block windows are used. A zero-sized data block now means "not present" for future metadata providers. Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-13-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 6da890a4ad28..37be455b7b6f 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -430,11 +430,13 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, mask = BIT(dma_data->rg.bar); for (i = 0; i < dma_data->wr_ch_cnt; i++) { mask |= BIT(dma_data->ll_wr[i].bar); - mask |= BIT(dma_data->dt_wr[i].bar); + if (dma_data->dt_wr[i].sz) + mask |= BIT(dma_data->dt_wr[i].bar); } for (i = 0; i < dma_data->rd_ch_cnt; i++) { mask |= BIT(dma_data->ll_rd[i].bar); - mask |= BIT(dma_data->dt_rd[i].bar); + if (dma_data->dt_rd[i].sz) + mask |= BIT(dma_data->dt_rd[i].bar); } err = pcim_iomap_regions(pdev, mask, pci_name(pdev)); if (err) { @@ -497,6 +499,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, ll_block); ll_region->sz = ll_block->sz; + if (!dt_block->sz) + continue; + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; if (!dt_region->vaddr.io) return -ENOMEM; @@ -522,6 +527,9 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, ll_block); ll_region->sz = ll_block->sz; + if (!dt_block->sz) + continue; + dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar]; if (!dt_region->vaddr.io) return -ENOMEM; @@ -555,10 +563,14 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dma_data->ll_wr[i].off, chip->ll_region_wr[i].sz, chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr); + if (!dma_data->dt_wr[i].sz) + continue; + pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, dma_data->dt_wr[i].bar, dma_data->dt_wr[i].off, chip->dt_region_wr[i].sz, - chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr); + chip->dt_region_wr[i].vaddr.io, + &chip->dt_region_wr[i].paddr); } for (i = 0; i < chip->ll_rd_cnt; i++) { @@ -567,10 +579,14 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, dma_data->ll_rd[i].off, chip->ll_region_rd[i].sz, chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr); + if (!dma_data->dt_rd[i].sz) + continue; + pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n", i, dma_data->dt_rd[i].bar, dma_data->dt_rd[i].off, chip->dt_region_rd[i].sz, - chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr); + chip->dt_region_rd[i].vaddr.io, + &chip->dt_region_rd[i].paddr); } pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs); From 1d68178b3404737af2cee6c7b4395de3de6dccaa Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:14 +0900 Subject: [PATCH 67/73] dmaengine: dw-edma-pcie: Add chip flags to match data Allow PCI ID match data to pass dw_edma_chip flags into dw_edma_probe(). This keeps per-device policy in the match data instead of open-coding it in probe(). Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-14-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-pcie.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 37be455b7b6f..64f2a80072bc 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -88,6 +88,7 @@ struct dw_edma_pcie_match_data { int (*parse_caps)(struct pci_dev *pdev, struct dw_edma_pcie_data *pdata); unsigned long flags; + u32 chip_flags; }; #define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0) @@ -471,6 +472,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->dev = dev; chip->mf = dma_data->mf; + chip->flags = match->chip_flags; chip->nr_irqs = nr_irqs; chip->ops = match->plat_ops; chip->cfg_non_ll = dma_data->cfg_non_ll; From 0613e7934ee233d726af8d8c89f251a1c4df738d Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Tue, 21 Jul 2026 15:28:15 +0900 Subject: [PATCH 68/73] dmaengine: dw-edma: Program endpoint function numbers The eDMA/HDMA transfers the driver issues carry a requester function number in their TLPs, but nothing ever programs it: eDMA v0 leaves the FUNC_NUM field of the channel control word zero and HDMA leaves the per-channel func_num register at its reset value, so every transfer is attributed to function 0. That is invisible in single-function setups, but once the DMA block serves a non-zero endpoint function, its requests must carry that function's number for the host to attribute and translate them correctly. Record the function number in the chip data (PCI_FUNC() of the probing device for dw-edma-pcie) and program it per channel. Endpoint-local chip instances keep func_no at 0, so transfers issued by the endpoint-side driver remain PF0-attributed. Delegated channels are programmed by the host-side dw-edma-pcie instance when it takes over the channel, using that instance's PCI_FUNC(). Reviewed-by: Frank Li Signed-off-by: Koichiro Den Link: https://patch.msgid.link/20260721062815.4117887-15-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 1 + drivers/dma/dw-edma/dw-edma-core.h | 1 + drivers/dma/dw-edma/dw-edma-pcie.c | 1 + drivers/dma/dw-edma/dw-edma-v0-core.c | 10 +++++++++- drivers/dma/dw-edma/dw-hdma-v0-core.c | 3 +++ drivers/dma/dw-edma/dw-hdma-v0-regs.h | 1 + include/linux/dma/edma.h | 2 ++ 7 files changed, 18 insertions(+), 1 deletion(-) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index c9df5bbe2392..1f893dc54c79 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -931,6 +931,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc) chan = &dw->chan[i]; chan->dw = dw; + chan->func_no = chip->func_no; if (i < dw->wr_ch_cnt) { chan->id = i; diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index bd3d532ebc89..f6a5ad317567 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -70,6 +70,7 @@ struct dw_edma_chan { struct dw_edma *dw; int id; enum dw_edma_dir dir; + u8 func_no; u32 ll_max; struct dw_edma_region ll_region; /* Linked list */ diff --git a/drivers/dma/dw-edma/dw-edma-pcie.c b/drivers/dma/dw-edma/dw-edma-pcie.c index 64f2a80072bc..9f237ba916de 100644 --- a/drivers/dma/dw-edma/dw-edma-pcie.c +++ b/drivers/dma/dw-edma/dw-edma-pcie.c @@ -473,6 +473,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev, chip->mf = dma_data->mf; chip->flags = match->chip_flags; + chip->func_no = PCI_FUNC(pdev->devfn); chip->nr_irqs = nr_irqs; chip->ops = match->plat_ops; chip->cfg_non_ll = dma_data->cfg_non_ll; diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 13e69ce6c41d..abc1bd4c0be2 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -26,6 +26,8 @@ enum dw_edma_control { DW_EDMA_V0_LLE = BIT(9), }; +#define EDMA_V0_FUNC_NUM_MASK GENMASK(16, 12) + static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw) { return dw->chip->reg_base; @@ -160,6 +162,11 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, #define GET_CH_32(dw, dir, ch, name) \ readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) +static u32 dw_edma_v0_func_num(struct dw_edma_chan *chan) +{ + return FIELD_PREP(EDMA_V0_FUNC_NUM_MASK, chan->func_no); +} + /* eDMA management callbacks */ static void dw_edma_v0_core_ch_power(struct dw_edma *dw, enum dw_edma_dir dir, u16 id, bool enable) @@ -463,7 +470,8 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) /* Channel control */ SET_CH_32(dw, chan->dir, chan->id, ch_control1, - (DW_EDMA_V0_CCS | DW_EDMA_V0_LLE)); + DW_EDMA_V0_CCS | DW_EDMA_V0_LLE | + dw_edma_v0_func_num(chan)); /* Linked list */ /* llp is not aligned on 64bit -> keep 32bit accesses */ SET_CH_32(dw, chan->dir, chan->id, llp.lsb, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index 8d20bb028b52..36ee72efcd31 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -345,6 +345,9 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan) SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi); /* config MSI data */ SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data); + /* Configure the requester function number used by outbound TLPs. */ + SET_CH_32(dw, chan->dir, chan->id, func_num, + FIELD_PREP(HDMA_V0_FUNC_NUM_PF_MASK, chan->func_no)); } static void diff --git a/drivers/dma/dw-edma/dw-hdma-v0-regs.h b/drivers/dma/dw-edma/dw-hdma-v0-regs.h index 48e40efceb2e..2124c162a62f 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-regs.h +++ b/drivers/dma/dw-edma/dw-hdma-v0-regs.h @@ -24,6 +24,7 @@ #define HDMA_V0_CONSUMER_CYCLE_BIT BIT(0) #define HDMA_V0_DOORBELL_START BIT(0) #define HDMA_V0_CH_STATUS_MASK GENMASK(1, 0) +#define HDMA_V0_FUNC_NUM_PF_MASK GENMASK(7, 0) struct dw_hdma_v0_ch_regs { u32 ch_en; /* 0x0000 */ diff --git a/include/linux/dma/edma.h b/include/linux/dma/edma.h index 2ce9d2d49aef..3c8e2ef9dee0 100644 --- a/include/linux/dma/edma.h +++ b/include/linux/dma/edma.h @@ -117,6 +117,7 @@ enum dw_edma_ch_irq_mode { * @db_irq: Virtual IRQ dedicated to interrupt emulation * @db_offset: Offset from DMA register base * @mf: DMA register map format + * @func_no: PCI endpoint function number used by DMA TLPs * @dw: struct dw_edma that is filled by dw_edma_probe() */ struct dw_edma_chip { @@ -142,6 +143,7 @@ struct dw_edma_chip { resource_size_t db_offset; enum dw_edma_map_format mf; + u8 func_no; struct dw_edma *dw; bool cfg_non_ll; From 2ea04dca8e627f722caa7a2037cfbae0257f3501 Mon Sep 17 00:00:00 2001 From: Martin Kaiser Date: Sat, 18 Jul 2026 14:58:48 +0200 Subject: [PATCH 69/73] dmaengine: fsl-edma: tracing: no ptr dereference during log output The fsl edma events store a pointer to a struct fsl_edma_engine in the ringbuffer and dereference it when a log entry is printed. At this time, the pointer may no longer be valid. Event injection can be used to trigger a crash: $ cd /sys/kernel/tracing $ echo 'value = 0' > events/fsl_edma/edma_writeb/inject $ cat trace The log output needs only edma->membase. Add a membase field at the end of the event and use the new field for log output. Keep the existing fields for backward compatibility. Fixes: 11102d0c343b ("dmaengine: fsl-edma: add trace event support") Cc: stable@vger.kernel.org Reviewed-by: Steven Rostedt Signed-off-by: Martin Kaiser Reviewed-by: Frank Li Link: https://patch.msgid.link/20260718130024.341243-1-martin@kaiser.cx Signed-off-by: Vinod Koul --- drivers/dma/fsl-edma-trace.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/dma/fsl-edma-trace.h b/drivers/dma/fsl-edma-trace.h index d3541301a247..45d964a3726d 100644 --- a/drivers/dma/fsl-edma-trace.h +++ b/drivers/dma/fsl-edma-trace.h @@ -19,14 +19,16 @@ DECLARE_EVENT_CLASS(edma_log_io, __field(struct fsl_edma_engine *, edma) __field(void __iomem *, addr) __field(u32, value) + __field(void __iomem *, membase) ), TP_fast_assign( __entry->edma = edma; __entry->addr = addr; __entry->value = value; + __entry->membase = edma->membase; ), TP_printk("offset %08x: value %08x", - (u32)(__entry->addr - __entry->edma->membase), __entry->value) + (u32)(__entry->addr - __entry->membase), __entry->value) ); DEFINE_EVENT(edma_log_io, edma_readl, From a50184171235045d96104811414b25cf5338cf86 Mon Sep 17 00:00:00 2001 From: Guangshuo Li Date: Sat, 8 Aug 2026 19:06:42 +0800 Subject: [PATCH 70/73] dmaengine: qcom-bam-dma: fix autosuspend cleanup during removal bam_dma_probe() calls pm_runtime_use_autosuspend(), but bam_dma_remove() does not call the matching pm_runtime_dont_use_autosuspend() when removing the device. If the autosuspend delay is set to a negative value while autosuspend is enabled, the runtime PM core increments usage_count to prevent runtime suspend. Without calling pm_runtime_dont_use_autosuspend() during teardown, this reference is not dropped and usage_count remains unbalanced. Add the missing pm_runtime_dont_use_autosuspend() call before forcing the device into runtime suspend during removal. This issue was found by manual code inspection. Fixes: 7d2545599f5b ("dmaengine: qcom-bam-dma: Add pm_runtime support") Signed-off-by: Guangshuo Li Reviewed-by: Bartosz Golaszewski Link: https://patch.msgid.link/20260808110642.2770355-1-lgs201920130244@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/qcom/bam_dma.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/qcom/bam_dma.c b/drivers/dma/qcom/bam_dma.c index 1bb26af0405f..05a3b1f9e0c2 100644 --- a/drivers/dma/qcom/bam_dma.c +++ b/drivers/dma/qcom/bam_dma.c @@ -1401,6 +1401,7 @@ static void bam_dma_remove(struct platform_device *pdev) struct bam_device *bdev = platform_get_drvdata(pdev); u32 i; + pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_force_suspend(&pdev->dev); of_dma_controller_free(pdev->dev.of_node); From 04b8c3de13871653a8ecbd9f1adfea4d0694ecfe Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Fri, 24 Jul 2026 03:45:21 +0900 Subject: [PATCH 71/73] dmaengine: qcom_hidma: remove conditional return with no effect Both branches of the check return the same value, so the check has no effect. Remove it and return the value directly. This is the result of running the Coccinelle script from scripts/coccinelle/misc/cond_return_no_effect.cocci. Signed-off-by: Sang-Heon Jeon Link: https://patch.msgid.link/20260723184538.3888637-20-ekffu200098@gmail.com Signed-off-by: Vinod Koul --- drivers/dma/qcom/hidma_ll.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/drivers/dma/qcom/hidma_ll.c b/drivers/dma/qcom/hidma_ll.c index 53244e0e34a3..f081aa787f0c 100644 --- a/drivers/dma/qcom/hidma_ll.c +++ b/drivers/dma/qcom/hidma_ll.c @@ -677,11 +677,7 @@ int hidma_ll_setup(struct hidma_lldev *lldev) /* configure interrupts */ hidma_ll_setup_irq(lldev, lldev->msi_support); - rc = hidma_ll_enable(lldev); - if (rc) - return rc; - - return rc; + return hidma_ll_enable(lldev); } void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi) From 4a8b7929550eb94d60b32c2c06479444672da381 Mon Sep 17 00:00:00 2001 From: Giovanni Cabiddu Date: Wed, 5 Aug 2026 14:19:22 -0700 Subject: [PATCH 72/73] dmaengine: idxd: assign all engines to group 0 in IAA defaults The IAA device defaults only assigned engine 0 to group 0, leaving engines 1 through max_engines-1 unassigned (group_id = -1). This means that by default only a single engine processed descriptors, limiting throughput to one engine's capacity. Assign all available engines to group 0 so that the full hardware parallelism is used out of the box without requiring manual accel-config setup. Signed-off-by: Giovanni Cabiddu Signed-off-by: Vinicius Costa Gomes Reviewed-by: Dave Jiang Link: https://patch.msgid.link/20260805-iaa-crypto-fixes-zswap-v2-1-55c02694f499@intel.com Signed-off-by: Vinod Koul --- drivers/dma/idxd/defaults.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/drivers/dma/idxd/defaults.c b/drivers/dma/idxd/defaults.c index 2bbbcd02a0da..26ebfa2ca144 100644 --- a/drivers/dma/idxd/defaults.c +++ b/drivers/dma/idxd/defaults.c @@ -8,6 +8,7 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd) struct idxd_engine *engine; struct idxd_group *group; struct idxd_wq *wq; + int i; if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) return 0; @@ -41,11 +42,12 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd) /* set driver_name to "crypto" */ strscpy_pad(wq->driver_name, "crypto"); - engine = idxd->engines[0]; - - /* set engine group to 0 */ - engine->group = idxd->groups[0]; - engine->group->num_engines++; + /* assign all engines to group 0 */ + for (i = 0; i < idxd->max_engines; i++) { + engine = idxd->engines[i]; + engine->group = group; + group->num_engines++; + } return 0; } From 0d995da5fb97e8c312834575604d4423eb6225b7 Mon Sep 17 00:00:00 2001 From: Koichiro Den Date: Fri, 31 Jul 2026 01:07:01 +0900 Subject: [PATCH 73/73] dmaengine: dw-edma: Mark emulated IRQ as level-triggered The interrupt-emulation virtual IRQ uses handle_level_irq(), but the IRQ descriptor has not been marked with IRQ_LEVEL. The interrupt emulation is somewhat unusual: the eDMA interrupt handlers dispatch the virtual IRQ for every edma_int[] interrupt because software cannot reliably tell an interrupt-emulation event from one caused solely by DONE/ABORT status. If an interrupt arrives before the doorbell handler is registered for the virtual IRQ, the IRQ core marks it pending. When the IRQ is later started, check_irq_resend() treats it as non-level and replays the pending interrupt, causing the newly registered handler to run for a stale event. Mark the virtual IRQ with IRQ_LEVEL so the stale pending state is cleared without being replayed. This was observed in pci_endpoint_test as two doorbell handler calls when the DMA-variant test ran before DOORBELL_TEST. Fixes: d9d5e1bdd180 ("dmaengine: dw-edma: Add virtual IRQ for interrupt-emulation doorbells") Cc: stable@vger.kernel.org Reported-by: Niklas Cassel Closes: https://lore.kernel.org/r/ampndLtU32ODmncX@ryzen Tested-by: Niklas Cassel Signed-off-by: Koichiro Den Reviewed-by: Frank Li Reviewed-by: Manivannan Sadhasivam Link: https://patch.msgid.link/20260730160701.3550710-1-den@valinux.co.jp Signed-off-by: Vinod Koul --- drivers/dma/dw-edma/dw-edma-core.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/dma/dw-edma/dw-edma-core.c b/drivers/dma/dw-edma/dw-edma-core.c index 1f893dc54c79..03b2c2188351 100644 --- a/drivers/dma/dw-edma/dw-edma-core.c +++ b/drivers/dma/dw-edma/dw-edma-core.c @@ -764,6 +764,7 @@ static int dw_edma_emul_irq_alloc(struct dw_edma *dw) return virq; irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq); + irq_set_status_flags(virq, IRQ_LEVEL); irq_set_chip_data(virq, dw); irq_set_noprobe(virq);