mirrors/linux
2
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2026-08-31 12:52:29 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge tag 'dmaengine-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine

Browse Source 
Pull dmaengine updates from Vinod Koul:
 "Core:

   - New API to combine configuration and preparation and users

  New hardware support:

   - Mediatek MT8189 SoC uart dma support

  Updates:

   - Designware dma driver flatten desc structures and simplify code,
     interrupt-path groundwork changes, first part of PCI EP DMA support

   - Updates to zynqmp_dma with runtime PM and device removal
     improvments

   - Xilinx dma optimizations for AXIDMA and MCDMA channel management"

* tag 'dmaengine-7.3-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/vkoul/dmaengine: (73 commits)
  dmaengine: dw-edma: Mark emulated IRQ as level-triggered
  dmaengine: idxd: assign all engines to group 0 in IAA defaults
  dmaengine: qcom_hidma: remove conditional return with no effect
  dmaengine: qcom-bam-dma: fix autosuspend cleanup during removal
  dmaengine: fsl-edma: tracing: no ptr dereference during log output
  dmaengine: dw-edma: Program endpoint function numbers
  dmaengine: dw-edma-pcie: Add chip flags to match data
  dmaengine: dw-edma-pcie: Handle optional data blocks
  dmaengine: dw-edma-pcie: Factor out descriptor block address lookup
  dmaengine: dw-edma-pcie: Add register offset match flag
  dmaengine: dw-edma-pcie: Add platform ops to match data
  dmaengine: dw-edma-pcie: Rename vsec_data to dma_data
  dmaengine: dw-edma-pcie: Add capability match data
  dmaengine: dw-edma-pcie: Track non-LL mode in DMA data
  dmaengine: dw-edma: Add partial channel ownership mode
  dmaengine: dw-edma: Initialize IRQ data before requesting IRQs
  dmaengine: dw-edma: Add core quiesce operations
  dmaengine: dw-edma: Add per-channel interrupt routing control
  dmaengine: dw-edma: Factor out HDMA interrupt setup helper
  dmaengine: dw-edma: Defer channel IRQ handling to workqueue
  ...
This commit is contained in:
Linus Torvalds
2026-08-24 12:21:15 -07:00
parent 16e6a1a3cb 0d995da5fb
commit 66498c75b4
35 changed files with 1388 additions and 774 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
Documentation/devicetree/bindings/dma/altr,msgdma.yaml
View File

@@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
title: Altera mSGDMA IP core
maintainers:
- Olivier Dautricourt <olivierdautricourt@gmail.com>
- Adrian Ng Ho Yin <adrian.ho.yin.ng@altera.com>
description: |
Altera / Intel modular Scatter-Gather Direct Memory Access (mSGDMA)

1
Documentation/devicetree/bindings/dma/mediatek,uart-dma.yaml
View File

@@ -40,6 +40,7 @@ properties:
- items:
- enum:
- mediatek,mt6991-uart-dma
- mediatek,mt8189-uart-dma
- mediatek,mt8196-uart-dma
- const: mediatek,mt6985-uart-dma
- enum:

2
Documentation/devicetree/bindings/dma/qcom,bam-dma.yaml
View File

@@ -48,7 +48,7 @@ properties:
iommus:
minItems: 1
maxItems: 6
maxItems: 7
num-channels:
$ref: /schemas/types.yaml#/definitions/uint32

13
Documentation/devicetree/bindings/dma/xilinx/xlnx,axi-dma.yaml
View File

@@ -81,8 +81,13 @@ properties:
description: Should be the number of framebuffers as configured in h/w.
xlnx,flush-fsync:
type: boolean
description: Tells which channel to Flush on Frame sync.
$ref: /schemas/types.yaml#/definitions/uint32
enum: [1, 2, 3]
description:
Tells which channel to flush on frame sync.
1 - flush both channels
2 - flush mm2s channel
3 - flush s2mm channel
xlnx,sg-length-width:
$ref: /schemas/types.yaml#/definitions/uint32
@@ -93,7 +98,7 @@ properties:
Width in bits of the length register as configured in hardware.
xlnx,irq-delay:
$ref: /schemas/types.yaml#/definitions/uint32
$ref: /schemas/types.yaml#/definitions/uint8
minimum: 0
maximum: 255
description:
@@ -251,7 +256,7 @@ examples:
"m_axi_s2mm_aclk", "m_axis_mm2s_aclk",
"s_axis_s2mm_aclk";
xlnx,num-fstores = <8>;
xlnx,flush-fsync;
xlnx,flush-fsync = <1>;
xlnx,addrwidth = <32>;
dma-channel-mm2s {

9
Documentation/driver-api/dmaengine/client.rst
View File

@@ -80,6 +80,10 @@ The details of these operations are:
- slave_sg: DMA a list of scatter gather buffers from/to a peripheral
- config_sg: Similar with slave_sg, just pass down dma_slave_config
struct to avoid calling dmaengine_slave_config() every time adjusting the
burst length or the FIFO address is needed.
- peripheral_dma_vec: DMA an array of scatter gather buffers from/to a
peripheral. Similar to slave_sg, but uses an array of dma_vec
structures instead of a scatterlist.
@@ -106,6 +110,11 @@ The details of these operations are:
unsigned int sg_len, enum dma_data_direction direction,
unsigned long flags);
struct dma_async_tx_descriptor *dmaengine_prep_config_sg(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction dir,
unsigned long flags, struct dma_slave_config *config);
struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
struct dma_chan *chan, const struct dma_vec *vecs,
size_t nents, enum dma_data_direction direction,

4
MAINTAINERS
View File

@@ -951,10 +951,10 @@ S: Maintained
F: drivers/mailbox/mailbox-altera.c
ALTERA MSGDMA IP CORE DRIVER
M: Olivier Dautricourt <olivierdautricourt@gmail.com>
M: Adrian Ng Ho Yin <adrian.ho.yin.ng@altera.com>
R: Stefan Roese <sr@denx.de>
L: dmaengine@vger.kernel.org
S: Odd Fixes
S: Maintained
F: Documentation/devicetree/bindings/dma/altr,msgdma.yaml
F: drivers/dma/altera-msgdma.c

10
drivers/crypto/atmel-aes.c
View File

@@ -794,7 +794,6 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
struct dma_slave_config config;
dma_async_tx_callback callback;
struct atmel_aes_dma *dma;
int err;
memset(&config, 0, sizeof(config));
config.src_addr_width = addr_width;
@@ -819,12 +818,9 @@ static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd,
return -EINVAL;
}
err = dmaengine_slave_config(dma->chan, &config);
if (err)
return err;
desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
desc = dmaengine_prep_config_sg(dma->chan, dma->sg, dma->sg_len, dir,
DMA_PREP_INTERRUPT | DMA_CTRL_ACK,
&config);
if (!desc)
return -ENOMEM;

9
drivers/dma/arm-dma350.c
View File

@@ -13,6 +13,11 @@
#include "dmaengine.h"
#include "virt-dma.h"
#define DMANSECCTRL 0x200
#define NSEC_CTRL 0x0c
#define INTREN_ANYCHINTR_EN BIT(0)
#define DMAINFO 0x0f00
#define DMA_BUILDCFG0 0xb0
@@ -582,6 +587,10 @@ static int d350_probe(struct platform_device *pdev)
dmac->dma.device_issue_pending = d350_issue_pending;
INIT_LIST_HEAD(&dmac->dma.channels);
reg = readl_relaxed(base + DMANSECCTRL + NSEC_CTRL);
writel_relaxed(reg | INTREN_ANYCHINTR_EN,
base + DMANSECCTRL + NSEC_CTRL);
/* Would be nice to have per-channel caps for this... */
memset = true;
for (int i = 0; i < nchan; i++) {

9
drivers/dma/dmaengine.c
View File

@@ -814,10 +814,15 @@ static const struct dma_slave_map *dma_filter_match(struct dma_device *device,
*/
struct dma_chan *dma_request_chan(struct device *dev, const char *name)
{
struct fwnode_handle *fwnode = dev_fwnode(dev);
struct fwnode_handle *fwnode;
struct dma_device *d, *_d;
struct dma_chan *chan = NULL;
if (WARN_ON(!dev || !name))
return ERR_PTR(-EINVAL);
fwnode = dev_fwnode(dev);
if (is_of_node(fwnode))
chan = of_dma_request_slave_channel(to_of_node(fwnode), name);
else if (is_acpi_device_node(fwnode))
@@ -1100,6 +1105,8 @@ static int __dma_async_device_channel_register(struct dma_device *device,
chan->dev->device.parent = device->dev;
chan->dev->chan = chan;
chan->dev->dev_id = device->dev_id;
spin_lock_init(&chan->lock);
if (!name)
dev_set_name(&chan->dev->device, "dma%dchan%d", device->dev_id, chan->chan_id);
else

2
drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c
View File

@@ -1126,7 +1126,7 @@ static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan)
hw_desc = &desc->hw_desc[i];
if (hw_desc->llp == llp) {
axi_chan_irq_clear(chan, hw_desc->lli->status_lo);
hw_desc->lli->ctl_hi |= CH_CTL_H_LLI_VALID;
hw_desc->lli->ctl_hi |= cpu_to_le32(CH_CTL_H_LLI_VALID);
desc->completed_blocks = i;
if (((hw_desc->len * (i + 1)) % desc->period_len) == 0)

583
drivers/dma/dw-edma/dw-edma-core.c
View File

@@ -7,6 +7,7 @@
*/
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/device.h>
#include <linux/kernel.h>
#include <linux/dmaengine.h>
@@ -29,6 +30,11 @@ struct dw_edma_desc *vd2dw_edma_desc(struct virt_dma_desc *vd)
return container_of(vd, struct dw_edma_desc, vd);
}
enum dw_edma_irq_event {
DW_EDMA_IRQ_DONE = BIT(0),
DW_EDMA_IRQ_ABORT = BIT(1),
};
static inline
u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
{
@@ -40,141 +46,63 @@ u64 dw_edma_get_pci_address(struct dw_edma_chan *chan, phys_addr_t cpu_addr)
return cpu_addr;
}
static struct dw_edma_burst *dw_edma_alloc_burst(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *burst;
burst = kzalloc_obj(*burst, GFP_NOWAIT);
if (unlikely(!burst))
return NULL;
INIT_LIST_HEAD(&burst->list);
if (chunk->burst) {
/* Create and add new element into the linked list */
chunk->bursts_alloc++;
list_add_tail(&burst->list, &chunk->burst->list);
} else {
/* List head */
chunk->bursts_alloc = 0;
chunk->burst = burst;
}
return burst;
}
static struct dw_edma_chunk *dw_edma_alloc_chunk(struct dw_edma_desc *desc)
{
struct dw_edma_chip *chip = desc->chan->dw->chip;
struct dw_edma_chan *chan = desc->chan;
struct dw_edma_chunk *chunk;
chunk = kzalloc_obj(*chunk, GFP_NOWAIT);
if (unlikely(!chunk))
return NULL;
INIT_LIST_HEAD(&chunk->list);
chunk->chan = chan;
/* Toggling change bit (CB) in each chunk, this is a mechanism to
* inform the eDMA HW block that this is a new linked list ready
* to be consumed.
* - Odd chunks originate CB equal to 0
* - Even chunks originate CB equal to 1
*/
chunk->cb = !(desc->chunks_alloc % 2);
if (chan->dir == EDMA_DIR_WRITE) {
chunk->ll_region.paddr = chip->ll_region_wr[chan->id].paddr;
chunk->ll_region.vaddr = chip->ll_region_wr[chan->id].vaddr;
} else {
chunk->ll_region.paddr = chip->ll_region_rd[chan->id].paddr;
chunk->ll_region.vaddr = chip->ll_region_rd[chan->id].vaddr;
}
if (desc->chunk) {
/* Create and add new element into the linked list */
if (!dw_edma_alloc_burst(chunk)) {
kfree(chunk);
return NULL;
}
desc->chunks_alloc++;
list_add_tail(&chunk->list, &desc->chunk->list);
} else {
/* List head */
chunk->burst = NULL;
desc->chunks_alloc = 0;
desc->chunk = chunk;
}
return chunk;
}
static struct dw_edma_desc *dw_edma_alloc_desc(struct dw_edma_chan *chan)
static struct dw_edma_desc *
dw_edma_alloc_desc(struct dw_edma_chan *chan, size_t nburst)
{
struct dw_edma_desc *desc;
desc = kzalloc_obj(*desc, GFP_NOWAIT);
desc = kzalloc_flex(*desc, burst, nburst, GFP_NOWAIT);
if (unlikely(!desc))
return NULL;
desc->chan = chan;
if (!dw_edma_alloc_chunk(desc)) {
kfree(desc);
return NULL;
}
desc->nburst = nburst;
desc->cb = true;
return desc;
}
static void dw_edma_free_burst(struct dw_edma_chunk *chunk)
{
struct dw_edma_burst *child, *_next;
/* Remove all the list elements */
list_for_each_entry_safe(child, _next, &chunk->burst->list, list) {
list_del(&child->list);
kfree(child);
chunk->bursts_alloc--;
}
/* Remove the list head */
kfree(child);
chunk->burst = NULL;
}
static void dw_edma_free_chunk(struct dw_edma_desc *desc)
{
struct dw_edma_chunk *child, *_next;
if (!desc->chunk)
return;
/* Remove all the list elements */
list_for_each_entry_safe(child, _next, &desc->chunk->list, list) {
dw_edma_free_burst(child);
list_del(&child->list);
kfree(child);
desc->chunks_alloc--;
}
/* Remove the list head */
kfree(child);
desc->chunk = NULL;
}
static void dw_edma_free_desc(struct dw_edma_desc *desc)
{
dw_edma_free_chunk(desc);
kfree(desc);
}
static void vchan_free_desc(struct virt_dma_desc *vdesc)
{
dw_edma_free_desc(vd2dw_edma_desc(vdesc));
kfree(vd2dw_edma_desc(vdesc));
}
static void dw_edma_core_start(struct dw_edma_desc *desc, bool first)
{
struct dw_edma_chan *chan = desc->chan;
size_t i = 0;
if (chan->non_ll) {
chan->dw->core->non_ll_start(chan, &desc->burst[desc->start_burst]);
desc->done_burst = desc->start_burst;
desc->start_burst += 1;
return;
}
for (i = 0; i + desc->start_burst < desc->nburst; i++) {
u32 idx = i + desc->start_burst;
if (i == chan->ll_max)
break;
dw_edma_core_ll_data(chan, &desc->burst[idx],
i, desc->cb,
idx == desc->nburst - 1 || i == chan->ll_max - 1);
}
desc->done_burst = desc->start_burst;
desc->start_burst += i;
dw_edma_core_ll_link(chan, i, desc->cb, chan->ll_region.paddr);
if (first)
dw_edma_core_ch_enable(chan);
dw_edma_core_ch_doorbell(chan);
}
static int dw_edma_start_transfer(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
struct dw_edma_chunk *child;
struct dw_edma_desc *desc;
struct virt_dma_desc *vd;
@@ -186,21 +114,42 @@ static int dw_edma_start_transfer(struct dw_edma_chan *chan)
if (!desc)
return 0;
child = list_first_entry_or_null(&desc->chunk->list,
struct dw_edma_chunk, list);
if (!child)
return 0;
dw_edma_core_start(desc, !desc->start_burst);
dw_edma_core_start(dw, child, !desc->xfer_sz);
desc->xfer_sz += child->ll_region.sz;
dw_edma_free_burst(child);
list_del(&child->list);
kfree(child);
desc->chunks_alloc--;
desc->cb = !desc->cb;
return 1;
}
static void dw_edma_terminate_vdesc(struct virt_dma_desc *vd)
{
list_del(&vd->node);
dma_cookie_complete(&vd->tx);
vchan_terminate_vdesc(vd);
}
static void dw_edma_terminate_vdesc_list(struct list_head *head)
{
struct virt_dma_desc *vd, *_vd;
list_for_each_entry_safe(vd, _vd, head, node)
dw_edma_terminate_vdesc(vd);
}
/* Must be called with vc.lock held. */
static void dw_edma_terminate_all_descs(struct dw_edma_chan *chan)
{
/*
* This order must not be reversed. Cookies are assigned when
* descriptors are submitted, so desc_issued contains older cookies
* than desc_submitted. Completing desc_submitted first could move
* chan->vc.chan.completed_cookie backwards when desc_issued is
* terminated afterwards.
*/
dw_edma_terminate_vdesc_list(&chan->vc.desc_issued);
dw_edma_terminate_vdesc_list(&chan->vc.desc_submitted);
}
static void dw_edma_device_caps(struct dma_chan *dchan,
struct dma_slave_caps *caps)
{
@@ -219,6 +168,15 @@ static void dw_edma_device_caps(struct dma_chan *dchan,
}
}
static enum dw_edma_ch_irq_mode
dw_edma_get_default_irq_mode(struct dw_edma_chan *chan)
{
struct dw_edma_chip *chip = chan->dw->chip;
return chip->flags & DW_EDMA_CHIP_LOCAL ? DW_EDMA_CH_IRQ_LOCAL :
DW_EDMA_CH_IRQ_REMOTE;
}
static int dw_edma_device_config(struct dma_chan *dchan,
struct dma_slave_config *config)
{
@@ -267,11 +225,27 @@ static int dw_edma_device_config(struct dma_chan *dchan,
return 0;
}
static struct dma_slave_config *
dw_edma_device_get_config(struct dma_chan *dchan,
struct dma_slave_config *config)
{
struct dw_edma_chan *chan;
if (config)
return config;
chan = dchan2dw_edma_chan(dchan);
return &chan->config;
}
static int dw_edma_device_pause(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
guard(spinlock_irqsave)(&chan->vc.lock);
if (!chan->configured)
err = -EPERM;
else if (chan->status != EDMA_ST_BUSY)
@@ -289,6 +263,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
guard(spinlock_irqsave)(&chan->vc.lock);
if (!chan->configured) {
err = -EPERM;
} else if (chan->status != EDMA_ST_PAUSE) {
@@ -297,7 +273,8 @@ static int dw_edma_device_resume(struct dma_chan *dchan)
err = -EPERM;
} else {
chan->status = EDMA_ST_BUSY;
dw_edma_start_transfer(chan);
if (!dw_edma_start_transfer(chan))
chan->status = EDMA_ST_IDLE;
}
return err;
@@ -308,25 +285,29 @@ static int dw_edma_device_terminate_all(struct dma_chan *dchan)
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
int err = 0;
guard(spinlock_irqsave)(&chan->vc.lock);
if (!chan->configured) {
/* Do nothing */
dw_edma_terminate_all_descs(chan);
} else if (chan->status == EDMA_ST_PAUSE) {
dw_edma_terminate_all_descs(chan);
chan->status = EDMA_ST_IDLE;
chan->configured = false;
} else if (chan->status == EDMA_ST_IDLE) {
chan->configured = false;
dw_edma_terminate_all_descs(chan);
} else if (dw_edma_core_ch_status(chan) == DMA_COMPLETE) {
/*
* The channel is in a false BUSY state, probably didn't
* receive or lost an interrupt
*/
dw_edma_terminate_all_descs(chan);
chan->status = EDMA_ST_IDLE;
chan->configured = false;
} else if (chan->request > EDMA_REQ_PAUSE) {
err = -EPERM;
} else {
chan->request = EDMA_REQ_STOP;
}
if (chan->status == EDMA_ST_IDLE)
chan->request = EDMA_REQ_NONE;
return err;
}
@@ -336,11 +317,9 @@ static void dw_edma_device_issue_pending(struct dma_chan *dchan)
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
unsigned long flags;
if (!chan->configured)
return;
spin_lock_irqsave(&chan->vc.lock, flags);
if (vchan_issue_pending(&chan->vc) && chan->request == EDMA_REQ_NONE &&
if (chan->configured && vchan_issue_pending(&chan->vc) &&
chan->request == EDMA_REQ_NONE &&
chan->status == EDMA_ST_IDLE) {
chan->status = EDMA_ST_BUSY;
dw_edma_start_transfer(chan);
@@ -373,8 +352,10 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
vd = vchan_find_desc(&chan->vc, cookie);
if (vd) {
desc = vd2dw_edma_desc(vd);
if (desc)
residue = desc->alloc_sz - desc->xfer_sz;
residue = desc->alloc_sz;
if (desc && desc->done_burst)
residue -= desc->burst[desc->done_burst - 1].xfer_sz;
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
@@ -385,19 +366,18 @@ dw_edma_device_tx_status(struct dma_chan *dchan, dma_cookie_t cookie,
}
static struct dma_async_tx_descriptor *
dw_edma_device_transfer(struct dw_edma_transfer *xfer)
dw_edma_device_transfer(struct dw_edma_transfer *xfer,
struct dma_slave_config *config)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(xfer->dchan);
enum dma_transfer_direction dir = xfer->direction;
struct scatterlist *sg = NULL;
struct dw_edma_chunk *chunk;
struct dw_edma_burst *burst;
struct dw_edma_desc *desc;
u64 src_addr, dst_addr;
size_t fsz = 0;
u32 bursts_max;
u32 cnt = 0;
int i;
size_t cnt = 0;
u32 i;
if (!chan->configured)
return NULL;
@@ -453,27 +433,12 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
return NULL;
}
/*
* For non-LL mode, only a single burst can be handled
* in a single chunk unlike LL mode where multiple bursts
* can be configured in a single chunk.
*/
bursts_max = chan->non_ll ? 1 : chan->ll_max;
desc = dw_edma_alloc_desc(chan);
if (unlikely(!desc))
goto err_alloc;
chunk = dw_edma_alloc_chunk(desc);
if (unlikely(!chunk))
goto err_alloc;
if (xfer->type == EDMA_XFER_INTERLEAVED) {
src_addr = xfer->xfer.il->src_start;
dst_addr = xfer->xfer.il->dst_start;
} else {
src_addr = chan->config.src_addr;
dst_addr = chan->config.dst_addr;
src_addr = config->src_addr;
dst_addr = config->dst_addr;
}
if (dir == DMA_DEV_TO_MEM)
@@ -491,19 +456,15 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
fsz = xfer->xfer.il->frame_size;
}
desc = dw_edma_alloc_desc(chan, cnt);
if (unlikely(!desc))
return NULL;
for (i = 0; i < cnt; i++) {
if (xfer->type == EDMA_XFER_SCATTER_GATHER && !sg)
break;
if (chunk->bursts_alloc == bursts_max) {
chunk = dw_edma_alloc_chunk(desc);
if (unlikely(!chunk))
goto err_alloc;
}
burst = dw_edma_alloc_burst(chunk);
if (unlikely(!burst))
goto err_alloc;
burst = desc->burst + i;
if (xfer->type == EDMA_XFER_CYCLIC)
burst->sz = xfer->xfer.cyclic.len;
@@ -512,8 +473,8 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
else if (xfer->type == EDMA_XFER_INTERLEAVED)
burst->sz = xfer->xfer.il->sgl[i % fsz].size;
chunk->ll_region.sz += burst->sz;
desc->alloc_sz += burst->sz;
burst->xfer_sz = desc->alloc_sz;
if (dir == DMA_DEV_TO_MEM) {
burst->sar = src_addr;
@@ -568,19 +529,14 @@ dw_edma_device_transfer(struct dw_edma_transfer *xfer)
}
return vchan_tx_prep(&chan->vc, &desc->vd, xfer->flags);
err_alloc:
if (desc)
dw_edma_free_desc(desc);
return NULL;
}
static struct dma_async_tx_descriptor *
dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
unsigned int len,
enum dma_transfer_direction direction,
unsigned long flags, void *context)
dw_edma_device_prep_config_sg(struct dma_chan *dchan, struct scatterlist *sgl,
unsigned int len,
enum dma_transfer_direction direction,
unsigned long flags,
struct dma_slave_config *config)
{
struct dw_edma_transfer xfer;
@@ -591,7 +547,10 @@ dw_edma_device_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
xfer.flags = flags;
xfer.type = EDMA_XFER_SCATTER_GATHER;
return dw_edma_device_transfer(&xfer);
if (config && dw_edma_device_config(dchan, config))
return NULL;
return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, config));
}
static struct dma_async_tx_descriptor *
@@ -610,7 +569,7 @@ dw_edma_device_prep_dma_cyclic(struct dma_chan *dchan, dma_addr_t paddr,
xfer.flags = flags;
xfer.type = EDMA_XFER_CYCLIC;
return dw_edma_device_transfer(&xfer);
return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
}
static struct dma_async_tx_descriptor *
@@ -626,7 +585,7 @@ dw_edma_device_prep_interleaved_dma(struct dma_chan *dchan,
xfer.flags = flags;
xfer.type = EDMA_XFER_INTERLEAVED;
return dw_edma_device_transfer(&xfer);
return dw_edma_device_transfer(&xfer, dw_edma_device_get_config(dchan, NULL));
}
static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
@@ -640,8 +599,14 @@ static void dw_hdma_set_callback_result(struct virt_dma_desc *vd,
return;
desc = vd2dw_edma_desc(vd);
if (desc)
residue = desc->alloc_sz - desc->xfer_sz;
if (desc) {
residue = desc->alloc_sz;
if (result == DMA_TRANS_NOERROR)
residue -= desc->burst[desc->start_burst - 1].xfer_sz;
else if (desc->done_burst)
residue -= desc->burst[desc->done_burst - 1].xfer_sz;
}
res = &vd->tx_result;
res->result = result;
@@ -655,35 +620,41 @@ static void dw_edma_done_interrupt(struct dw_edma_chan *chan)
unsigned long flags;
spin_lock_irqsave(&chan->vc.lock, flags);
if (chan->status == EDMA_ST_PAUSE) {
spin_unlock_irqrestore(&chan->vc.lock, flags);
return;
}
vd = vchan_next_desc(&chan->vc);
if (vd) {
switch (chan->request) {
case EDMA_REQ_NONE:
case EDMA_REQ_PAUSE:
desc = vd2dw_edma_desc(vd);
if (!desc->chunks_alloc) {
if (desc->start_burst >= desc->nburst) {
dw_hdma_set_callback_result(vd,
DMA_TRANS_NOERROR);
list_del(&vd->node);
vchan_cookie_complete(vd);
}
if (chan->request == EDMA_REQ_PAUSE) {
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_PAUSE;
break;
}
/* Continue transferring if there are remaining chunks or issued requests.
*/
chan->status = dw_edma_start_transfer(chan) ? EDMA_ST_BUSY : EDMA_ST_IDLE;
break;
case EDMA_REQ_STOP:
list_del(&vd->node);
vchan_cookie_complete(vd);
dw_edma_terminate_all_descs(chan);
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
break;
case EDMA_REQ_PAUSE:
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_PAUSE;
break;
default:
break;
}
@@ -698,14 +669,49 @@ static void dw_edma_abort_interrupt(struct dw_edma_chan *chan)
spin_lock_irqsave(&chan->vc.lock, flags);
vd = vchan_next_desc(&chan->vc);
if (vd) {
if (vd && chan->request == EDMA_REQ_STOP) {
dw_edma_terminate_all_descs(chan);
} else if (vd) {
dw_hdma_set_callback_result(vd, DMA_TRANS_ABORTED);
list_del(&vd->node);
vchan_cookie_complete(vd);
}
spin_unlock_irqrestore(&chan->vc.lock, flags);
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
spin_unlock_irqrestore(&chan->vc.lock, flags);
}
static void dw_edma_irq_work(struct work_struct *work)
{
struct dw_edma_chan *chan = container_of(work, struct dw_edma_chan,
irq_work);
unsigned int events;
do {
events = atomic_xchg(&chan->irq_pending, 0);
if (events & DW_EDMA_IRQ_DONE)
dw_edma_done_interrupt(chan);
if (events & DW_EDMA_IRQ_ABORT)
dw_edma_abort_interrupt(chan);
} while (atomic_read(&chan->irq_pending));
}
static void dw_edma_queue_irq_work(struct dw_edma_chan *chan,
enum dw_edma_irq_event event)
{
atomic_or(event, &chan->irq_pending);
queue_work(chan->dw->wq, &chan->irq_work);
}
static void dw_edma_done_interrupt_deferred(struct dw_edma_chan *chan)
{
dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_DONE);
}
static void dw_edma_abort_interrupt_deferred(struct dw_edma_chan *chan)
{
dw_edma_queue_irq_work(chan, DW_EDMA_IRQ_ABORT);
}
static void dw_edma_emul_irq_ack(struct irq_data *d)
@@ -739,6 +745,9 @@ static int dw_edma_emul_irq_alloc(struct dw_edma *dw)
chip->db_irq = 0;
chip->db_offset = ~0;
if (chip->flags & DW_EDMA_CHIP_PARTIAL)
return 0;
/*
* Only meaningful when the core provides the deassert sequence
* for interrupt emulation.
@@ -755,6 +764,7 @@ static int dw_edma_emul_irq_alloc(struct dw_edma *dw)
return virq;
irq_set_chip_and_handler(virq, &dw_edma_emul_irqchip, handle_level_irq);
irq_set_status_flags(virq, IRQ_LEVEL);
irq_set_chip_data(virq, dw);
irq_set_noprobe(virq);
@@ -802,8 +812,8 @@ static inline irqreturn_t dw_edma_interrupt_write_inner(int irq, void *data)
struct dw_edma_irq *dw_irq = data;
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_WRITE,
dw_edma_done_interrupt,
dw_edma_abort_interrupt);
dw_edma_done_interrupt_deferred,
dw_edma_abort_interrupt_deferred);
}
static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
@@ -811,8 +821,8 @@ static inline irqreturn_t dw_edma_interrupt_read_inner(int irq, void *data)
struct dw_edma_irq *dw_irq = data;
return dw_edma_core_handle_int(dw_irq, EDMA_DIR_READ,
dw_edma_done_interrupt,
dw_edma_abort_interrupt);
dw_edma_done_interrupt_deferred,
dw_edma_abort_interrupt_deferred);
}
static inline irqreturn_t dw_edma_interrupt_write(int irq, void *data)
@@ -856,21 +866,51 @@ static int dw_edma_alloc_chan_resources(struct dma_chan *dchan)
return 0;
}
static void dw_edma_free_chan_resources(struct dma_chan *dchan)
static void dw_edma_wait_termination(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
unsigned long timeout = jiffies + msecs_to_jiffies(5000);
int ret;
bool stopping;
/*
* A STOP may be deferred to a later interrupt while the channel is still
* running. Wait until that handler completes the termination.
*/
while (time_before(jiffies, timeout)) {
ret = dw_edma_device_terminate_all(dchan);
if (!ret)
break;
scoped_guard(spinlock_irqsave, &chan->vc.lock)
stopping = chan->request == EDMA_REQ_STOP;
if (time_after_eq(jiffies, timeout))
if (!stopping)
return;
cpu_relax();
fsleep(1000);
}
dev_warn(chan->dw->chip->dev,
"timeout waiting for channel termination\n");
}
static void dw_edma_device_synchronize(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
dw_edma_wait_termination(dchan);
cancel_work_sync(&chan->irq_work);
atomic_set(&chan->irq_pending, 0);
vchan_synchronize(&chan->vc);
}
static void dw_edma_free_chan_resources(struct dma_chan *dchan)
{
struct dw_edma_chan *chan = dchan2dw_edma_chan(dchan);
dw_edma_device_terminate_all(dchan);
dw_edma_device_synchronize(dchan);
scoped_guard(spinlock_irqsave, &chan->vc.lock)
chan->configured = false;
vchan_free_chan_resources(&chan->vc);
}
static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
@@ -892,6 +932,7 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
chan = &dw->chan[i];
chan->dw = dw;
chan->func_no = chip->func_no;
if (i < dw->wr_ch_cnt) {
chan->id = i;
@@ -904,12 +945,16 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
chan->configured = false;
chan->request = EDMA_REQ_NONE;
chan->status = EDMA_ST_IDLE;
chan->irq_mode = dw_edma_get_default_irq_mode(chan);
INIT_WORK(&chan->irq_work, dw_edma_irq_work);
atomic_set(&chan->irq_pending, 0);
if (chan->dir == EDMA_DIR_WRITE)
chan->ll_max = (chip->ll_region_wr[chan->id].sz / EDMA_LL_SZ);
chan->ll_region = chip->ll_region_wr[chan->id];
else
chan->ll_max = (chip->ll_region_rd[chan->id].sz / EDMA_LL_SZ);
chan->ll_max -= 1;
chan->ll_region = chip->ll_region_rd[chan->id];
chan->ll_max = chan->ll_region.sz / EDMA_LL_SZ - 1;
dev_vdbg(dev, "L. List:\tChannel %s[%u] max_cnt=%u\n",
str_write_read(chan->dir == EDMA_DIR_WRITE),
@@ -925,11 +970,10 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
irq = &dw->irq[pos];
if (chan->dir == EDMA_DIR_WRITE)
irq->wr_mask |= BIT(chan->id);
bitmap_set(irq->wr_mask, chan->id, 1);
else
irq->rd_mask |= BIT(chan->id);
bitmap_set(irq->rd_mask, chan->id, 1);
irq->dw = dw;
memcpy(&chan->msi, &irq->msi, sizeof(chan->msi));
dev_vdbg(dev, "MSI:\t\tChannel %s[%u] addr=0x%.8x%.8x, data=0x%.8x\n",
@@ -968,9 +1012,10 @@ static int dw_edma_channel_setup(struct dw_edma *dw, u32 wr_alloc, u32 rd_alloc)
dma->device_pause = dw_edma_device_pause;
dma->device_resume = dw_edma_device_resume;
dma->device_terminate_all = dw_edma_device_terminate_all;
dma->device_synchronize = dw_edma_device_synchronize;
dma->device_issue_pending = dw_edma_device_issue_pending;
dma->device_tx_status = dw_edma_device_tx_status;
dma->device_prep_slave_sg = dw_edma_device_prep_slave_sg;
dma->device_prep_config_sg = dw_edma_device_prep_config_sg;
dma->device_prep_dma_cyclic = dw_edma_device_prep_dma_cyclic;
dma->device_prep_interleaved_dma = dw_edma_device_prep_interleaved_dma;
@@ -1010,6 +1055,7 @@ static int dw_edma_irq_request(struct dw_edma *dw,
if (chip->nr_irqs == 1) {
/* Common IRQ shared among all channels */
irq = chip->ops->irq_vector(dev, 0);
dw->irq[0].dw = dw;
err = request_irq(irq, dw_edma_interrupt_common,
IRQF_SHARED, dw->name, &dw->irq[0]);
if (err) {
@@ -1032,6 +1078,7 @@ static int dw_edma_irq_request(struct dw_edma *dw,
for (i = 0; i < (*wr_alloc + *rd_alloc); i++) {
irq = chip->ops->irq_vector(dev, i);
dw->irq[i].dw = dw;
err = request_irq(irq,
i < *wr_alloc ?
dw_edma_interrupt_write :
@@ -1062,12 +1109,37 @@ static int dw_edma_irq_request(struct dw_edma *dw,
return err;
}
static int dw_edma_check_partial(struct dw_edma_chip *chip,
u16 hw_wr_ch_cnt, u16 hw_rd_ch_cnt)
{
if (!(chip->flags & DW_EDMA_CHIP_PARTIAL))
return 0;
if (chip->mf != EDMA_MF_EDMA_UNROLL &&
chip->mf != EDMA_MF_HDMA_COMPAT)
return 0;
/*
* Direction-wide registers are shared by all channels in that
* direction, so a direction must have a single owner.
*/
if ((chip->ll_wr_cnt && chip->ll_wr_cnt != hw_wr_ch_cnt) ||
(chip->ll_rd_cnt && chip->ll_rd_cnt != hw_rd_ch_cnt))
return -EOPNOTSUPP;
return 0;
}
int dw_edma_probe(struct dw_edma_chip *chip)
{
struct device *dev;
struct dw_edma *dw;
u16 hw_wr_ch_cnt;
u16 hw_rd_ch_cnt;
u32 wr_alloc = 0;
u32 rd_alloc = 0;
u16 max_wr_cnt;
u16 max_rd_cnt;
int i, err;
if (!chip)
@@ -1077,26 +1149,50 @@ int dw_edma_probe(struct dw_edma_chip *chip)
if (!dev || !chip->ops)
return -EINVAL;
if (chip->flags & DW_EDMA_CHIP_PARTIAL) {
switch (chip->mf) {
case EDMA_MF_EDMA_UNROLL:
case EDMA_MF_HDMA_COMPAT:
case EDMA_MF_HDMA_NATIVE:
break;
default:
return -EOPNOTSUPP;
}
}
dw = devm_kzalloc(dev, sizeof(*dw), GFP_KERNEL);
if (!dw)
return -ENOMEM;
dw->chip = chip;
if (dw->chip->mf == EDMA_MF_HDMA_NATIVE)
if (dw->chip->mf == EDMA_MF_HDMA_NATIVE) {
dw_hdma_v0_core_register(dw);
else
max_wr_cnt = HDMA_MAX_WR_CH;
max_rd_cnt = HDMA_MAX_RD_CH;
} else {
dw_edma_v0_core_register(dw);
max_wr_cnt = EDMA_MAX_WR_CH;
max_rd_cnt = EDMA_MAX_RD_CH;
}
raw_spin_lock_init(&dw->lock);
dw->wr_ch_cnt = min_t(u16, chip->ll_wr_cnt,
dw_edma_core_ch_count(dw, EDMA_DIR_WRITE));
dw->wr_ch_cnt = min_t(u16, dw->wr_ch_cnt, EDMA_MAX_WR_CH);
/*
* chip->ll_*_cnt describes the channels exposed by this instance. Keep
* the usable hardware counts separate for partial ownership checks.
*/
hw_wr_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_WRITE),
max_wr_cnt);
hw_rd_ch_cnt = min(dw_edma_core_ch_count(dw, EDMA_DIR_READ),
max_rd_cnt);
dw->rd_ch_cnt = min_t(u16, chip->ll_rd_cnt,
dw_edma_core_ch_count(dw, EDMA_DIR_READ));
dw->rd_ch_cnt = min_t(u16, dw->rd_ch_cnt, EDMA_MAX_RD_CH);
err = dw_edma_check_partial(chip, hw_wr_ch_cnt, hw_rd_ch_cnt);
if (err)
return err;
dw->wr_ch_cnt = min(chip->ll_wr_cnt, hw_wr_ch_cnt);
dw->rd_ch_cnt = min(chip->ll_rd_cnt, hw_rd_ch_cnt);
if (!dw->wr_ch_cnt && !dw->rd_ch_cnt)
return -EINVAL;
@@ -1113,13 +1209,34 @@ int dw_edma_probe(struct dw_edma_chip *chip)
snprintf(dw->name, sizeof(dw->name), "dw-edma-core:%s",
dev_name(chip->dev));
/* Disable eDMA, only to establish the ideal initial conditions */
dw_edma_core_off(dw);
if (chip->flags & DW_EDMA_CHIP_PARTIAL) {
/*
* Do not reset the shared controller, but drain stale state
* from resources represented by this instance.
*/
err = dw_edma_core_quiesce(dw);
if (err)
return err;
} else {
/* Disable eDMA only when this instance owns the controller. */
dw_edma_core_off(dw);
}
/*
* Deferred IRQ works are queued from the hard IRQ handlers, so the
* workqueue must exist before any IRQ is requested.
*/
dw->wq = alloc_workqueue("dw-edma:%s", WQ_UNBOUND | WQ_HIGHPRI, 0,
dev_name(chip->dev));
if (!dw->wq)
return -ENOMEM;
/* Request IRQs */
err = dw_edma_irq_request(dw, &wr_alloc, &rd_alloc);
if (err)
if (err) {
destroy_workqueue(dw->wq);
return err;
}
/* Allocate a dedicated virtual IRQ for interrupt-emulation doorbells */
err = dw_edma_emul_irq_alloc(dw);
@@ -1142,6 +1259,7 @@ int dw_edma_probe(struct dw_edma_chip *chip)
for (i = (dw->nr_irqs - 1); i >= 0; i--)
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
dw_edma_emul_irq_free(dw);
destroy_workqueue(dw->wq);
return err;
}
@@ -1152,20 +1270,27 @@ int dw_edma_remove(struct dw_edma_chip *chip)
struct dw_edma_chan *chan, *_chan;
struct device *dev = chip->dev;
struct dw_edma *dw = chip->dw;
int i;
int i, err = 0;
/* Skip removal if no private data found */
if (!dw)
return -ENODEV;
/* Disable eDMA */
dw_edma_core_off(dw);
if (chip->flags & DW_EDMA_CHIP_PARTIAL)
err = dw_edma_core_quiesce(dw);
else
dw_edma_core_off(dw);
/* Free irqs */
for (i = (dw->nr_irqs - 1); i >= 0; i--)
free_irq(chip->ops->irq_vector(dev, i), &dw->irq[i]);
dw_edma_emul_irq_free(dw);
for (i = 0; i < dw->wr_ch_cnt + dw->rd_ch_cnt; i++)
cancel_work_sync(&dw->chan[i].irq_work);
destroy_workqueue(dw->wq);
/* Deregister eDMA device */
dma_async_device_unregister(&dw->dma);
list_for_each_entry_safe(chan, _chan, &dw->dma.channels,
@@ -1174,7 +1299,7 @@ int dw_edma_remove(struct dw_edma_chip *chip)
list_del(&chan->vc.chan.device_node);
}
return 0;
return err;
}
EXPORT_SYMBOL_GPL(dw_edma_remove);

113
drivers/dma/dw-edma/dw-edma-core.h
View File

@@ -9,8 +9,10 @@
#ifndef _DW_EDMA_CORE_H
#define _DW_EDMA_CORE_H
#include <linux/atomic.h>
#include <linux/msi.h>
#include <linux/dma/edma.h>
#include <linux/workqueue.h>
#include "../virt-dma.h"
@@ -43,32 +45,24 @@ struct dw_edma_chan;
struct dw_edma_chunk;
struct dw_edma_burst {
struct list_head list;
u64 sar;
u64 dar;
u32 sz;
};
struct dw_edma_chunk {
struct list_head list;
struct dw_edma_chan *chan;
struct dw_edma_burst *burst;
u32 bursts_alloc;
u8 cb;
struct dw_edma_region ll_region; /* Linked list */
/* precalulate summary of previous burst total size */
u32 xfer_sz;
};
struct dw_edma_desc {
struct virt_dma_desc vd;
struct dw_edma_chan *chan;
struct dw_edma_chunk *chunk;
u32 chunks_alloc;
u32 alloc_sz;
u32 xfer_sz;
size_t done_burst;
size_t start_burst;
u8 cb;
size_t nburst;
struct dw_edma_burst burst[] __counted_by(nburst);
};
struct dw_edma_chan {
@@ -76,24 +70,32 @@ struct dw_edma_chan {
struct dw_edma *dw;
int id;
enum dw_edma_dir dir;
u8 func_no;
u32 ll_max;
struct dw_edma_region ll_region; /* Linked list */
struct msi_msg msi;
enum dw_edma_ch_irq_mode irq_mode;
enum dw_edma_request request;
enum dw_edma_status status;
u8 configured;
struct dma_slave_config config;
bool non_ll;
struct work_struct irq_work;
atomic_t irq_pending;
};
struct dw_edma_irq {
struct msi_msg msi;
u32 wr_mask;
u32 rd_mask;
struct dw_edma *dw;
DECLARE_BITMAP(wr_mask, HDMA_MAX_WR_CH);
DECLARE_BITMAP(rd_mask, HDMA_MAX_RD_CH);
};
struct dw_edma {
@@ -109,6 +111,12 @@ struct dw_edma {
struct dw_edma_chan *chan;
/*
* WQ_HIGHPRI keeps completion processing responsive under heavy load;
* WQ_UNBOUND lets different channels run on different CPUs.
*/
struct workqueue_struct *wq;
raw_spinlock_t lock; /* Protect v0 shared registers */
struct dw_edma_chip *chip;
@@ -120,11 +128,18 @@ typedef void (*dw_edma_handler_t)(struct dw_edma_chan *);
struct dw_edma_core_ops {
void (*off)(struct dw_edma *dw);
int (*quiesce)(struct dw_edma *dw);
int (*ch_quiesce)(struct dw_edma_chan *chan);
u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir);
enum dma_status (*ch_status)(struct dw_edma_chan *chan);
irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
dw_edma_handler_t done, dw_edma_handler_t abort);
void (*start)(struct dw_edma_chunk *chunk, bool first);
void (*non_ll_start)(struct dw_edma_chan *chan, struct dw_edma_burst *child);
void (*ll_data)(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq);
void (*ll_link)(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr);
void (*ch_doorbell)(struct dw_edma_chan *chan);
void (*ch_enable)(struct dw_edma_chan *chan);
void (*ch_config)(struct dw_edma_chan *chan);
void (*debugfs_on)(struct dw_edma *dw);
void (*ack_emulated_irq)(struct dw_edma *dw);
@@ -166,12 +181,32 @@ struct dw_edma_chan *dchan2dw_edma_chan(struct dma_chan *dchan)
return vc2dw_edma_chan(to_virt_chan(dchan));
}
static inline u64 dw_edma_core_get_ll_paddr(struct dw_edma_chan *chan)
{
if (chan->dir == EDMA_DIR_WRITE)
return chan->dw->chip->ll_region_wr[chan->id].paddr;
return chan->dw->chip->ll_region_rd[chan->id].paddr;
}
static inline
void dw_edma_core_off(struct dw_edma *dw)
{
dw->core->off(dw);
}
static inline
int dw_edma_core_quiesce(struct dw_edma *dw)
{
return dw->core->quiesce(dw);
}
static inline
int dw_edma_core_ch_quiesce(struct dw_edma_chan *chan)
{
return chan->dw->core->ch_quiesce(chan);
}
static inline
u16 dw_edma_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
{
@@ -191,18 +226,35 @@ dw_edma_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
return dw_irq->dw->core->handle_int(dw_irq, dir, done, abort);
}
static inline
void dw_edma_core_start(struct dw_edma *dw, struct dw_edma_chunk *chunk, bool first)
{
dw->core->start(chunk, first);
}
static inline
void dw_edma_core_ch_config(struct dw_edma_chan *chan)
{
chan->dw->core->ch_config(chan);
}
static inline void
dw_edma_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq)
{
chan->dw->core->ll_data(chan, burst, idx, cb, irq);
}
static inline void
dw_edma_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
{
chan->dw->core->ll_link(chan, idx, cb, addr);
}
static inline void dw_edma_core_ch_doorbell(struct dw_edma_chan *chan)
{
chan->dw->core->ch_doorbell(chan);
}
static inline void dw_edma_core_ch_enable(struct dw_edma_chan *chan)
{
chan->dw->core->ch_enable(chan);
}
static inline
void dw_edma_core_debugfs_on(struct dw_edma *dw)
{
@@ -224,4 +276,15 @@ dw_edma_core_db_offset(struct dw_edma *dw)
return dw->core->db_offset(dw);
}
static inline bool
dw_edma_core_ch_ignore_irq(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
if (dw->chip->flags & DW_EDMA_CHIP_LOCAL)
return chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE;
else
return chan->irq_mode == DW_EDMA_CH_IRQ_LOCAL;
}
#endif /* _DW_EDMA_CORE_H */

264
drivers/dma/dw-edma/dw-edma-pcie.c
View File

@@ -55,6 +55,8 @@
struct dw_edma_block {
enum pci_barno bar;
off_t off;
u64 paddr;
bool paddr_valid;
size_t sz;
};
@@ -62,19 +64,36 @@ struct dw_edma_pcie_data {
/* eDMA registers location */
struct dw_edma_block rg;
/* eDMA memory linked list location */
struct dw_edma_block ll_wr[EDMA_MAX_WR_CH];
struct dw_edma_block ll_rd[EDMA_MAX_RD_CH];
struct dw_edma_block ll_wr[HDMA_MAX_WR_CH];
struct dw_edma_block ll_rd[HDMA_MAX_RD_CH];
/* eDMA memory data location */
struct dw_edma_block dt_wr[EDMA_MAX_WR_CH];
struct dw_edma_block dt_rd[EDMA_MAX_RD_CH];
struct dw_edma_block dt_wr[HDMA_MAX_WR_CH];
struct dw_edma_block dt_rd[HDMA_MAX_RD_CH];
/* Other */
enum dw_edma_map_format mf;
u8 irqs;
u16 wr_ch_cnt;
u16 rd_ch_cnt;
u64 devmem_phys_off;
bool cfg_non_ll;
};
struct dw_edma_pcie_match_data {
const struct dw_edma_pcie_data *data;
const struct dw_edma_plat_ops *plat_ops;
/*
* Mandatory callback. It may leave @pdata unchanged when the static
* template already describes the device.
*/
int (*parse_caps)(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata);
unsigned long flags;
u32 chip_flags;
};
#define DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF BIT(0)
#define DW_EDMA_PCIE_F_REG_OFFSET BIT(1)
static const struct dw_edma_pcie_data snps_edda_data = {
/* eDMA registers location */
.rg.bar = BAR_0,
@@ -309,31 +328,88 @@ static void dw_edma_pcie_get_xilinx_dma_data(struct pci_dev *pdev,
pdata->devmem_phys_off = off;
}
static int
dw_edma_pcie_parse_synopsys_caps(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata)
{
dw_edma_pcie_get_synopsys_dma_data(pdev, pdata);
return 0;
}
static int
dw_edma_pcie_parse_xilinx_caps(struct pci_dev *pdev,
struct dw_edma_pcie_data *pdata)
{
dw_edma_pcie_get_xilinx_dma_data(pdev, pdata);
/*
* There is no valid address found for the LL memory space on the
* device side. In the absence of LL base address use the non-LL mode or
* simple mode supported by the HDMA IP.
*/
if (pdata->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR) {
pdata->cfg_non_ll = true;
return 0;
}
/*
* Configure the channel LL and data blocks if number of channels
* enabled in VSEC capability are more than the channels configured in
* xilinx_mdb_data.
*/
dw_edma_set_chan_region_offset(pdata, BAR_2, 0,
DW_PCIE_XILINX_MDB_LL_OFF_GAP,
DW_PCIE_XILINX_MDB_LL_SIZE,
DW_PCIE_XILINX_MDB_DT_OFF_GAP,
DW_PCIE_XILINX_MDB_DT_SIZE);
return 0;
}
static u64 dw_edma_get_phys_addr(struct pci_dev *pdev,
const struct dw_edma_pcie_match_data *match,
struct dw_edma_pcie_data *pdata,
enum pci_barno bar)
{
if (pdev->vendor == PCI_VENDOR_ID_XILINX)
if (match->flags & DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF)
return pdata->devmem_phys_off;
return pci_bus_address(pdev, bar);
}
static u64 dw_edma_get_block_addr(struct pci_dev *pdev,
const struct dw_edma_pcie_match_data *match,
struct dw_edma_pcie_data *pdata,
const struct dw_edma_block *block)
{
if (block->paddr_valid)
return block->paddr;
return dw_edma_get_phys_addr(pdev, match, pdata, block->bar) +
block->off;
}
static int dw_edma_pcie_probe(struct pci_dev *pdev,
const struct pci_device_id *pid)
{
struct dw_edma_pcie_data *pdata = (void *)pid->driver_data;
const struct dw_edma_pcie_match_data *match = (void *)pid->driver_data;
const struct dw_edma_pcie_data *pdata;
struct device *dev = &pdev->dev;
struct dw_edma_chip *chip;
int err, nr_irqs;
int i, mask;
bool non_ll = false;
if (!match)
return -ENODEV;
pdata = match->data;
if (!pdata)
return -ENODEV;
struct dw_edma_pcie_data *vsec_data __free(kfree) =
kmalloc_obj(*vsec_data);
if (!vsec_data)
struct dw_edma_pcie_data *dma_data __free(kfree) =
kmemdup(pdata, sizeof(*dma_data), GFP_KERNEL);
if (!dma_data)
return -ENOMEM;
/* Enable PCI device */
@@ -343,48 +419,25 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
return err;
}
memcpy(vsec_data, pdata, sizeof(struct dw_edma_pcie_data));
/* Let device-specific discovery override the static template data. */
if (!match->parse_caps || !match->plat_ops)
return -EINVAL;
/*
* Tries to find if exists a PCIe Vendor-Specific Extended Capability
* for the DMA, if one exists, then reconfigures it.
*/
dw_edma_pcie_get_synopsys_dma_data(pdev, vsec_data);
if (pdev->vendor == PCI_VENDOR_ID_XILINX) {
dw_edma_pcie_get_xilinx_dma_data(pdev, vsec_data);
/*
* There is no valid address found for the LL memory
* space on the device side. In the absence of LL base
* address use the non-LL mode or simple mode supported by
* the HDMA IP.
*/
if (vsec_data->devmem_phys_off == DW_PCIE_XILINX_MDB_INVALID_ADDR)
non_ll = true;
/*
* Configure the channel LL and data blocks if number of
* channels enabled in VSEC capability are more than the
* channels configured in xilinx_mdb_data.
*/
if (!non_ll)
dw_edma_set_chan_region_offset(vsec_data, BAR_2, 0,
DW_PCIE_XILINX_MDB_LL_OFF_GAP,
DW_PCIE_XILINX_MDB_LL_SIZE,
DW_PCIE_XILINX_MDB_DT_OFF_GAP,
DW_PCIE_XILINX_MDB_DT_SIZE);
}
err = match->parse_caps(pdev, dma_data);
if (err)
return err;
/* Mapping PCI BAR regions */
mask = BIT(vsec_data->rg.bar);
for (i = 0; i < vsec_data->wr_ch_cnt; i++) {
mask |= BIT(vsec_data->ll_wr[i].bar);
mask |= BIT(vsec_data->dt_wr[i].bar);
mask = BIT(dma_data->rg.bar);
for (i = 0; i < dma_data->wr_ch_cnt; i++) {
mask |= BIT(dma_data->ll_wr[i].bar);
if (dma_data->dt_wr[i].sz)
mask |= BIT(dma_data->dt_wr[i].bar);
}
for (i = 0; i < vsec_data->rd_ch_cnt; i++) {
mask |= BIT(vsec_data->ll_rd[i].bar);
mask |= BIT(vsec_data->dt_rd[i].bar);
for (i = 0; i < dma_data->rd_ch_cnt; i++) {
mask |= BIT(dma_data->ll_rd[i].bar);
if (dma_data->dt_rd[i].sz)
mask |= BIT(dma_data->dt_rd[i].bar);
}
err = pcim_iomap_regions(pdev, mask, pci_name(pdev));
if (err) {
@@ -407,7 +460,7 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
return -ENOMEM;
/* IRQs allocation */
nr_irqs = pci_alloc_irq_vectors(pdev, 1, vsec_data->irqs,
nr_irqs = pci_alloc_irq_vectors(pdev, 1, dma_data->irqs,
PCI_IRQ_MSI | PCI_IRQ_MSIX);
if (nr_irqs < 1) {
pci_err(pdev, "fail to alloc IRQ vector (number of IRQs=%u)\n",
@@ -418,69 +471,75 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
/* Data structure initialization */
chip->dev = dev;
chip->mf = vsec_data->mf;
chip->mf = dma_data->mf;
chip->flags = match->chip_flags;
chip->func_no = PCI_FUNC(pdev->devfn);
chip->nr_irqs = nr_irqs;
chip->ops = &dw_edma_pcie_plat_ops;
chip->cfg_non_ll = non_ll;
chip->ops = match->plat_ops;
chip->cfg_non_ll = dma_data->cfg_non_ll;
chip->ll_wr_cnt = vsec_data->wr_ch_cnt;
chip->ll_rd_cnt = vsec_data->rd_ch_cnt;
chip->ll_wr_cnt = dma_data->wr_ch_cnt;
chip->ll_rd_cnt = dma_data->rd_ch_cnt;
chip->reg_base = pcim_iomap_table(pdev)[vsec_data->rg.bar];
chip->reg_base = pcim_iomap_table(pdev)[dma_data->rg.bar];
if (!chip->reg_base)
return -ENOMEM;
if (match->flags & DW_EDMA_PCIE_F_REG_OFFSET)
chip->reg_base += dma_data->rg.off;
for (i = 0; i < chip->ll_wr_cnt && !non_ll; i++) {
for (i = 0; i < chip->ll_wr_cnt && !dma_data->cfg_non_ll; i++) {
struct dw_edma_region *ll_region = &chip->ll_region_wr[i];
struct dw_edma_region *dt_region = &chip->dt_region_wr[i];
struct dw_edma_block *ll_block = &vsec_data->ll_wr[i];
struct dw_edma_block *dt_block = &vsec_data->dt_wr[i];
struct dw_edma_block *ll_block = &dma_data->ll_wr[i];
struct dw_edma_block *dt_block = &dma_data->dt_wr[i];
ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr.io)
return -ENOMEM;
ll_region->vaddr.io += ll_block->off;
ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
ll_block->bar);
ll_region->paddr += ll_block->off;
ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data,
ll_block);
ll_region->sz = ll_block->sz;
if (!dt_block->sz)
continue;
dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr.io)
return -ENOMEM;
dt_region->vaddr.io += dt_block->off;
dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
dt_block->bar);
dt_region->paddr += dt_block->off;
dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data,
dt_block);
dt_region->sz = dt_block->sz;
}
for (i = 0; i < chip->ll_rd_cnt && !non_ll; i++) {
for (i = 0; i < chip->ll_rd_cnt && !dma_data->cfg_non_ll; i++) {
struct dw_edma_region *ll_region = &chip->ll_region_rd[i];
struct dw_edma_region *dt_region = &chip->dt_region_rd[i];
struct dw_edma_block *ll_block = &vsec_data->ll_rd[i];
struct dw_edma_block *dt_block = &vsec_data->dt_rd[i];
struct dw_edma_block *ll_block = &dma_data->ll_rd[i];
struct dw_edma_block *dt_block = &dma_data->dt_rd[i];
ll_region->vaddr.io = pcim_iomap_table(pdev)[ll_block->bar];
if (!ll_region->vaddr.io)
return -ENOMEM;
ll_region->vaddr.io += ll_block->off;
ll_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
ll_block->bar);
ll_region->paddr += ll_block->off;
ll_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data,
ll_block);
ll_region->sz = ll_block->sz;
if (!dt_block->sz)
continue;
dt_region->vaddr.io = pcim_iomap_table(pdev)[dt_block->bar];
if (!dt_region->vaddr.io)
return -ENOMEM;
dt_region->vaddr.io += dt_block->off;
dt_region->paddr = dw_edma_get_phys_addr(pdev, vsec_data,
dt_block->bar);
dt_region->paddr += dt_block->off;
dt_region->paddr = dw_edma_get_block_addr(pdev, match, dma_data,
dt_block);
dt_region->sz = dt_block->sz;
}
@@ -497,32 +556,40 @@ static int dw_edma_pcie_probe(struct pci_dev *pdev,
pci_dbg(pdev, "Version:\tUnknown (0x%x)\n", chip->mf);
pci_dbg(pdev, "Registers:\tBAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p)\n",
vsec_data->rg.bar, vsec_data->rg.off, vsec_data->rg.sz,
dma_data->rg.bar, dma_data->rg.off, dma_data->rg.sz,
chip->reg_base);
for (i = 0; i < chip->ll_wr_cnt; i++) {
pci_dbg(pdev, "L. List:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data->ll_wr[i].bar,
vsec_data->ll_wr[i].off, chip->ll_region_wr[i].sz,
i, dma_data->ll_wr[i].bar,
dma_data->ll_wr[i].off, chip->ll_region_wr[i].sz,
chip->ll_region_wr[i].vaddr.io, &chip->ll_region_wr[i].paddr);
if (!dma_data->dt_wr[i].sz)
continue;
pci_dbg(pdev, "Data:\tWRITE CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data->dt_wr[i].bar,
vsec_data->dt_wr[i].off, chip->dt_region_wr[i].sz,
chip->dt_region_wr[i].vaddr.io, &chip->dt_region_wr[i].paddr);
i, dma_data->dt_wr[i].bar,
dma_data->dt_wr[i].off, chip->dt_region_wr[i].sz,
chip->dt_region_wr[i].vaddr.io,
&chip->dt_region_wr[i].paddr);
}
for (i = 0; i < chip->ll_rd_cnt; i++) {
pci_dbg(pdev, "L. List:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data->ll_rd[i].bar,
vsec_data->ll_rd[i].off, chip->ll_region_rd[i].sz,
i, dma_data->ll_rd[i].bar,
dma_data->ll_rd[i].off, chip->ll_region_rd[i].sz,
chip->ll_region_rd[i].vaddr.io, &chip->ll_region_rd[i].paddr);
if (!dma_data->dt_rd[i].sz)
continue;
pci_dbg(pdev, "Data:\tREAD CH%.2u, BAR=%u, off=0x%.8lx, sz=0x%zx bytes, addr(v=%p, p=%pa)\n",
i, vsec_data->dt_rd[i].bar,
vsec_data->dt_rd[i].off, chip->dt_region_rd[i].sz,
chip->dt_region_rd[i].vaddr.io, &chip->dt_region_rd[i].paddr);
i, dma_data->dt_rd[i].bar,
dma_data->dt_rd[i].off, chip->dt_region_rd[i].sz,
chip->dt_region_rd[i].vaddr.io,
&chip->dt_region_rd[i].paddr);
}
pci_dbg(pdev, "Nr. IRQs:\t%u\n", chip->nr_irqs);
@@ -555,17 +622,34 @@ static void dw_edma_pcie_remove(struct pci_dev *pdev)
err = dw_edma_remove(chip);
if (err)
pci_warn(pdev, "can't remove device properly: %d\n", err);
/* Freeing IRQs */
pci_free_irq_vectors(pdev);
}
static const struct dw_edma_pcie_match_data snps_edda_match_data = {
.data = &snps_edda_data,
.plat_ops = &dw_edma_pcie_plat_ops,
.parse_caps = dw_edma_pcie_parse_synopsys_caps,
};
static const struct dw_edma_pcie_match_data xilinx_mdb_match_data = {
.data = &xilinx_mdb_data,
.plat_ops = &dw_edma_pcie_plat_ops,
.parse_caps = dw_edma_pcie_parse_xilinx_caps,
.flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF,
};
static const struct dw_edma_pcie_match_data xilinx_cpm6_dma_match_data = {
.data = &xilinx_cpm6_dma_data,
.plat_ops = &dw_edma_pcie_plat_ops,
.parse_caps = dw_edma_pcie_parse_xilinx_caps,
.flags = DW_EDMA_PCIE_F_DEVMEM_PHYS_OFF,
};
static const struct pci_device_id dw_edma_pcie_id_table[] = {
{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_data) },
{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, &snps_edda_match_data) },
{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B054),
(kernel_ulong_t)&xilinx_mdb_data },
.driver_data = (kernel_ulong_t)&xilinx_mdb_match_data },
{ PCI_VDEVICE(XILINX, PCI_DEVICE_ID_XILINX_B00F),
.driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_data },
.driver_data = (kernel_ulong_t)&xilinx_cpm6_dma_match_data },
{ }
};
MODULE_DEVICE_TABLE(pci, dw_edma_pcie_id_table);

382
drivers/dma/dw-edma/dw-edma-v0-core.c
View File

@@ -7,6 +7,7 @@
*/
#include <linux/bitfield.h>
#include <linux/iopoll.h>
#include <linux/irqreturn.h>
#include <linux/io-64-nonatomic-lo-hi.h>
@@ -25,6 +26,8 @@ enum dw_edma_control {
DW_EDMA_V0_LLE = BIT(9),
};
#define EDMA_V0_FUNC_NUM_MASK GENMASK(16, 12)
static inline struct dw_edma_v0_regs __iomem *__dw_regs(struct dw_edma *dw)
{
return dw->chip->reg_base;
@@ -159,7 +162,93 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch,
#define GET_CH_32(dw, dir, ch, name) \
readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name))
static u32 dw_edma_v0_func_num(struct dw_edma_chan *chan)
{
return FIELD_PREP(EDMA_V0_FUNC_NUM_MASK, chan->func_no);
}
/* eDMA management callbacks */
static void dw_edma_v0_core_ch_power(struct dw_edma *dw,
enum dw_edma_dir dir, u16 id, bool enable)
{
u32 value = enable ? BIT(0) : 0;
if (WARN_ON_ONCE(id >= EDMA_V0_MAX_NR_CH))
return;
switch (id) {
case 0:
SET_RW_COMPAT(dw, dir, ch0_pwr_en, value);
break;
case 1:
SET_RW_COMPAT(dw, dir, ch1_pwr_en, value);
break;
case 2:
SET_RW_COMPAT(dw, dir, ch2_pwr_en, value);
break;
case 3:
SET_RW_COMPAT(dw, dir, ch3_pwr_en, value);
break;
case 4:
SET_RW_COMPAT(dw, dir, ch4_pwr_en, value);
break;
case 5:
SET_RW_COMPAT(dw, dir, ch5_pwr_en, value);
break;
case 6:
SET_RW_COMPAT(dw, dir, ch6_pwr_en, value);
break;
case 7:
SET_RW_COMPAT(dw, dir, ch7_pwr_en, value);
break;
}
}
static int dw_edma_v0_core_engine_disable(struct dw_edma *dw,
enum dw_edma_dir dir)
{
u32 value;
int ret;
SET_RW_32(dw, dir, engine_en, 0);
ret = read_poll_timeout(GET_RW_32, value, !(value & BIT(0)), 100,
200000, false, dw, dir, engine_en);
if (ret)
dev_warn(dw->chip->dev, "%s engine did not stop within 200ms\n",
dir == EDMA_DIR_WRITE ? "write" : "read");
return ret;
}
static int dw_edma_v0_core_dir_off(struct dw_edma *dw, enum dw_edma_dir dir)
{
u16 count, id;
int ret = 0;
scoped_guard(raw_spinlock_irqsave, &dw->lock)
SET_RW_32(dw, dir, int_mask,
EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
/*
* DWC PCIe Controller Databook 6.10a-lca06, "Legacy DMA
* and HDMA Software Compatibility": HDMA compatibility mode
* does not implement ENGINE_EN, but retains CHi_PWR_EN for
* per-channel enable and disable.
*/
count = dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt;
for (id = 0; id < count; id++)
dw_edma_v0_core_ch_power(dw, dir, id, false);
} else {
ret = dw_edma_v0_core_engine_disable(dw, dir);
}
SET_RW_32(dw, dir, int_clear,
EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK);
return ret;
}
static void dw_edma_v0_core_off(struct dw_edma *dw)
{
SET_BOTH_32(dw, int_mask,
@@ -169,6 +258,33 @@ static void dw_edma_v0_core_off(struct dw_edma *dw)
SET_BOTH_32(dw, engine_en, 0);
}
static int dw_edma_v0_core_quiesce(struct dw_edma *dw)
{
int ret = 0;
int err;
if (dw->wr_ch_cnt)
ret = dw_edma_v0_core_dir_off(dw, EDMA_DIR_WRITE);
if (dw->rd_ch_cnt) {
err = dw_edma_v0_core_dir_off(dw, EDMA_DIR_READ);
if (!ret)
ret = err;
}
return ret;
}
/*
* The unrolled eDMA and HDMA compatibility register maps share interrupt
* control per direction, so the whole direction is quiesced. Callers must
* own the direction entirely and prevent the peer from programming it after
* this point. Partial ownership mode validates direction granularity.
*/
static int dw_edma_v0_core_ch_quiesce(struct dw_edma_chan *chan)
{
return dw_edma_v0_core_dir_off(chan->dw, chan->dir);
}
static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
{
u32 num_ch;
@@ -218,18 +334,6 @@ static void dw_edma_v0_core_clear_abort_int(struct dw_edma_chan *chan)
FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id)));
}
static u32 dw_edma_v0_core_status_done_int(struct dw_edma *dw, enum dw_edma_dir dir)
{
return FIELD_GET(EDMA_V0_DONE_INT_MASK,
GET_RW_32(dw, dir, int_status));
}
static u32 dw_edma_v0_core_status_abort_int(struct dw_edma *dw, enum dw_edma_dir dir)
{
return FIELD_GET(EDMA_V0_ABORT_INT_MASK,
GET_RW_32(dw, dir, int_status));
}
static irqreturn_t
dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
dw_edma_handler_t done, dw_edma_handler_t abort)
@@ -239,7 +343,8 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
irqreturn_t ret = IRQ_NONE;
struct dw_edma_chan *chan;
unsigned long off;
u32 mask;
unsigned long *mask;
u32 sts;
if (dir == EDMA_DIR_WRITE) {
total = dw->wr_ch_cnt;
@@ -251,22 +356,38 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
mask = dw_irq->rd_mask;
}
val = dw_edma_v0_core_status_done_int(dw, dir);
val &= mask;
/*
* DONE and ABORT status share one register, and on remote setups
* every read is a non-posted round trip across the PCIe link. Take
* one snapshot and derive both views from it. An abort raised
* after the snapshot is deferred, not lost: only bits observed in
* the snapshot are ever cleared below, so its status remains set and
* triggers another handler pass.
*/
sts = GET_RW_32(dw, dir, int_status);
val = FIELD_GET(EDMA_V0_DONE_INT_MASK, sts);
val &= *mask;
for_each_set_bit(pos, &val, total) {
chan = &dw->chan[pos + off];
if (unlikely(dw_edma_core_ch_ignore_irq(chan)))
continue;
dw_edma_v0_core_clear_done_int(chan);
done(chan);
ret = IRQ_HANDLED;
}
val = dw_edma_v0_core_status_abort_int(dw, dir);
val &= mask;
val = FIELD_GET(EDMA_V0_ABORT_INT_MASK, sts);
val &= *mask;
for_each_set_bit(pos, &val, total) {
chan = &dw->chan[pos + off];
if (unlikely(dw_edma_core_ch_ignore_irq(chan)))
continue;
dw_edma_v0_core_clear_abort_int(chan);
abort(chan);
@@ -276,77 +397,90 @@ dw_edma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
return ret;
}
static void dw_edma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
static void dw_edma_v0_write_ll_data(struct dw_edma_chan *chan, int i,
u32 control, u32 size, u64 sar, u64 dar)
{
ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
lli->control = control;
lli->transfer_size = size;
lli->sar.reg = sar;
lli->dar.reg = dar;
dma_wmb();
lli->control = control;
} else {
struct dw_edma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
struct dw_edma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
writel(control, &lli->control);
writel(size, &lli->transfer_size);
writeq(sar, &lli->sar.reg);
writeq(dar, &lli->dar.reg);
writel(control, &lli->control);
}
}
static void dw_edma_v0_write_ll_link(struct dw_edma_chunk *chunk,
static void dw_edma_v0_write_ll_link(struct dw_edma_chan *chan,
int i, u32 control, u64 pointer)
{
ptrdiff_t ofs = i * sizeof(struct dw_edma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_edma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs;
llp->control = control;
llp->llp.reg = pointer;
dma_wmb();
llp->control = control;
} else {
struct dw_edma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
struct dw_edma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs;
writel(control, &llp->control);
writeq(pointer, &llp->llp.reg);
writel(control, &llp->control);
}
}
static void dw_edma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan)
{
struct dw_edma_burst *child;
struct dw_edma_chan *chan = chunk->chan;
u32 control = 0, i = 0;
int j;
struct dw_edma *dw = chan->dw;
unsigned long flags;
u32 tmp;
if (chunk->cb)
control = DW_EDMA_V0_CB;
/* Enable engine */
SET_RW_32(dw, chan->dir, engine_en, BIT(0));
if (dw->chip->mf == EDMA_MF_HDMA_COMPAT)
dw_edma_v0_core_ch_power(dw, chan->dir, chan->id, true);
/* Interrupt mask/unmask - done, abort */
raw_spin_lock_irqsave(&dw->lock, flags);
j = chunk->bursts_alloc;
list_for_each_entry(child, &chunk->burst->list, list) {
j--;
if (!j) {
control |= DW_EDMA_V0_LIE;
if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
control |= DW_EDMA_V0_RIE;
}
dw_edma_v0_write_ll_data(chunk, i++, control, child->sz,
child->sar, child->dar);
tmp = GET_RW_32(dw, chan->dir, int_mask);
if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE) {
tmp |= FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
tmp |= FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
} else {
tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
}
SET_RW_32(dw, chan->dir, int_mask, tmp);
/* Linked list error */
tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
if (!chunk->cb)
control |= DW_EDMA_V0_CB;
raw_spin_unlock_irqrestore(&dw->lock, flags);
dw_edma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
/* Channel control */
SET_CH_32(dw, chan->dir, chan->id, ch_control1,
DW_EDMA_V0_CCS | DW_EDMA_V0_LLE |
dw_edma_v0_func_num(chan));
/* Linked list */
/* llp is not aligned on 64bit -> keep 32bit accesses */
SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
lower_32_bits(chan->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, llp.msb,
upper_32_bits(chan->ll_region.paddr));
}
static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
static void dw_edma_v0_sync_ll_data(struct dw_edma_chan *chan)
{
/*
* In case of remote eDMA engine setup, the DW PCIe RP/EP internal
@@ -356,88 +490,8 @@ static void dw_edma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
* LL memory in a hope that the MRd TLP will return only after the
* last MWr TLP is completed
*/
if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
readl(chunk->ll_region.vaddr.io);
}
static void dw_edma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
{
struct dw_edma_chan *chan = chunk->chan;
struct dw_edma *dw = chan->dw;
unsigned long flags;
u32 tmp;
dw_edma_v0_core_write_chunk(chunk);
if (first) {
/* Enable engine */
SET_RW_32(dw, chan->dir, engine_en, BIT(0));
if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) {
switch (chan->id) {
case 0:
SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en,
BIT(0));
break;
case 1:
SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en,
BIT(0));
break;
case 2:
SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en,
BIT(0));
break;
case 3:
SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en,
BIT(0));
break;
case 4:
SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en,
BIT(0));
break;
case 5:
SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en,
BIT(0));
break;
case 6:
SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en,
BIT(0));
break;
case 7:
SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en,
BIT(0));
break;
}
}
/* Interrupt unmask - done, abort */
raw_spin_lock_irqsave(&dw->lock, flags);
tmp = GET_RW_32(dw, chan->dir, int_mask);
tmp &= ~FIELD_PREP(EDMA_V0_DONE_INT_MASK, BIT(chan->id));
tmp &= ~FIELD_PREP(EDMA_V0_ABORT_INT_MASK, BIT(chan->id));
SET_RW_32(dw, chan->dir, int_mask, tmp);
/* Linked list error */
tmp = GET_RW_32(dw, chan->dir, linked_list_err_en);
tmp |= FIELD_PREP(EDMA_V0_LINKED_LIST_ERR_MASK, BIT(chan->id));
SET_RW_32(dw, chan->dir, linked_list_err_en, tmp);
raw_spin_unlock_irqrestore(&dw->lock, flags);
/* Channel control */
SET_CH_32(dw, chan->dir, chan->id, ch_control1,
(DW_EDMA_V0_CCS | DW_EDMA_V0_LLE));
/* Linked list */
/* llp is not aligned on 64bit -> keep 32bit accesses */
SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
lower_32_bits(chunk->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, llp.msb,
upper_32_bits(chunk->ll_region.paddr));
}
dw_edma_v0_sync_ll_data(chunk);
/* Doorbell */
SET_RW_32(dw, chan->dir, doorbell,
FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
readl(chan->ll_region.vaddr.io);
}
static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
@@ -509,6 +563,59 @@ static void dw_edma_v0_core_ch_config(struct dw_edma_chan *chan)
}
}
static void
dw_edma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq)
{
u32 control = 0;
if (cb)
control |= DW_EDMA_V0_CB;
if (irq) {
control |= DW_EDMA_V0_LIE;
/*
* A local instance never issues transfers on a remote-routed
* channel: on CHIP_LOCAL instances, REMOTE routing denotes a
* channel handed over to the remote side, which programs the
* linked list through its own instance. The remote-only
* recipe (LIE|RIE with the local interrupt masked) is thus
* applied by the instance that owns the transfer, and the
* LIE-only write below never executes for a remote-routed
* channel.
*/
if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) &&
chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
control |= DW_EDMA_V0_RIE;
}
dw_edma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
burst->dar);
}
static void
dw_edma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
{
u32 control = DW_EDMA_V0_LLP | DW_EDMA_V0_TCB;
if (!cb)
control |= DW_EDMA_V0_CB;
dw_edma_v0_write_ll_link(chan, idx, control, addr);
}
static void dw_edma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
dw_edma_v0_sync_ll_data(chan);
/* Doorbell */
SET_RW_32(dw, chan->dir, doorbell,
FIELD_PREP(EDMA_V0_DOORBELL_CH_MASK, chan->id));
}
/* eDMA debugfs callbacks */
static void dw_edma_v0_core_debugfs_on(struct dw_edma *dw)
{
@@ -536,10 +643,15 @@ static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw)
static const struct dw_edma_core_ops dw_edma_v0_core = {
.off = dw_edma_v0_core_off,
.quiesce = dw_edma_v0_core_quiesce,
.ch_quiesce = dw_edma_v0_core_ch_quiesce,
.ch_count = dw_edma_v0_core_ch_count,
.ch_status = dw_edma_v0_core_ch_status,
.handle_int = dw_edma_v0_core_handle_int,
.start = dw_edma_v0_core_start,
.ll_data = dw_edma_v0_core_ll_data,
.ll_link = dw_edma_v0_core_ll_link,
.ch_doorbell = dw_edma_v0_core_ch_doorbell,
.ch_enable = dw_edma_v0_core_ch_enable,
.ch_config = dw_edma_v0_core_ch_config,
.debugfs_on = dw_edma_v0_core_debugfs_on,
.ack_emulated_irq = dw_edma_v0_core_ack_emulated_irq,

265
drivers/dma/dw-edma/dw-hdma-v0-core.c
View File

@@ -49,18 +49,80 @@ __dw_ch_regs(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch)
writel(value, &(__dw_ch_regs(dw, EDMA_DIR_READ, ch)->name)); \
} while (0)
static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val)
{
val &= ~(HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_REMOTE_STOP_INT_EN |
HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK);
/*
* HDMA_INT_STATUS.STOP and .ABORT are latched only when LSIE and
* LAIE are enabled. A remote handler needs those status bits to
* identify the source of the IMWr, so keep local generation enabled
* and mask the local interrupt pins instead.
*/
val |= HDMA_V0_LOCAL_ABORT_INT_EN | HDMA_V0_LOCAL_STOP_INT_EN;
if (chan->irq_mode == DW_EDMA_CH_IRQ_REMOTE)
val |= HDMA_V0_REMOTE_ABORT_INT_EN |
HDMA_V0_REMOTE_STOP_INT_EN |
HDMA_V0_ABORT_INT_MASK | HDMA_V0_STOP_INT_MASK;
return val;
}
/* HDMA management callbacks */
static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir,
u16 id)
{
SET_CH_32(dw, dir, id, int_setup,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
SET_CH_32(dw, dir, id, int_clear,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
}
static void dw_hdma_v0_core_off(struct dw_edma *dw)
{
int id;
enum dw_edma_dir dir;
for (id = 0; id < HDMA_V0_MAX_NR_CH; id++) {
SET_BOTH_CH_32(dw, id, int_setup,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_BOTH_CH_32(dw, id, int_clear,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_BOTH_CH_32(dw, id, ch_en, 0);
dir = EDMA_DIR_WRITE;
for (id = 0; id < dw->wr_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
dir = EDMA_DIR_READ;
for (id = 0; id < dw->rd_ch_cnt; id++) {
SET_CH_32(dw, dir, id, int_setup,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, int_clear,
HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
SET_CH_32(dw, dir, id, ch_en, 0);
}
}
static int dw_hdma_v0_core_quiesce(struct dw_edma *dw)
{
int id;
for (id = 0; id < dw->wr_ch_cnt; id++)
dw_hdma_v0_core_ch_off(dw, EDMA_DIR_WRITE, id);
for (id = 0; id < dw->rd_ch_cnt; id++)
dw_hdma_v0_core_ch_off(dw, EDMA_DIR_READ, id);
return 0;
}
static int dw_hdma_v0_core_ch_quiesce(struct dw_edma_chan *chan)
{
dw_hdma_v0_core_ch_off(chan->dw, chan->dir, chan->id);
return 0;
}
static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir)
@@ -79,7 +141,7 @@ static enum dma_status dw_hdma_v0_core_ch_status(struct dw_edma_chan *chan)
u32 tmp;
tmp = FIELD_GET(HDMA_V0_CH_STATUS_MASK,
GET_CH_32(dw, chan->id, chan->dir, ch_stat));
GET_CH_32(dw, chan->dir, chan->id, ch_stat));
if (tmp == 1)
return DMA_IN_PROGRESS;
@@ -118,7 +180,7 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
unsigned long total, pos, val;
irqreturn_t ret = IRQ_NONE;
struct dw_edma_chan *chan;
unsigned long off, mask;
unsigned long off, *mask;
if (dir == EDMA_DIR_WRITE) {
total = dw->wr_ch_cnt;
@@ -130,8 +192,10 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
mask = dw_irq->rd_mask;
}
for_each_set_bit(pos, &mask, total) {
for_each_set_bit(pos, mask, total) {
chan = &dw->chan[pos + off];
if (unlikely(dw_edma_core_ch_ignore_irq(chan)))
continue;
val = dw_hdma_v0_core_status_int(chan);
if (FIELD_GET(HDMA_V0_STOP_INT_MASK, val)) {
@@ -152,66 +216,73 @@ dw_hdma_v0_core_handle_int(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir,
return ret;
}
static void dw_hdma_v0_write_ll_data(struct dw_edma_chunk *chunk, int i,
static void dw_hdma_v0_write_ll_data(struct dw_edma_chan *chan, int i,
u32 control, u32 size, u64 sar, u64 dar)
{
ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_hdma_v0_lli *lli = chunk->ll_region.vaddr.mem + ofs;
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_hdma_v0_lli *lli = chan->ll_region.vaddr.mem + ofs;
lli->control = control;
lli->transfer_size = size;
lli->sar.reg = sar;
lli->dar.reg = dar;
dma_wmb();
lli->control = control;
} else {
struct dw_hdma_v0_lli __iomem *lli = chunk->ll_region.vaddr.io + ofs;
struct dw_hdma_v0_lli __iomem *lli = chan->ll_region.vaddr.io + ofs;
writel(control, &lli->control);
writel(size, &lli->transfer_size);
writeq(sar, &lli->sar.reg);
writeq(dar, &lli->dar.reg);
writel(control, &lli->control);
}
}
static void dw_hdma_v0_write_ll_link(struct dw_edma_chunk *chunk,
static void dw_hdma_v0_write_ll_link(struct dw_edma_chan *chan,
int i, u32 control, u64 pointer)
{
ptrdiff_t ofs = i * sizeof(struct dw_hdma_v0_lli);
if (chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_hdma_v0_llp *llp = chunk->ll_region.vaddr.mem + ofs;
if (chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL) {
struct dw_hdma_v0_llp *llp = chan->ll_region.vaddr.mem + ofs;
llp->control = control;
llp->llp.reg = pointer;
dma_wmb();
llp->control = control;
} else {
struct dw_hdma_v0_llp __iomem *llp = chunk->ll_region.vaddr.io + ofs;
struct dw_hdma_v0_llp __iomem *llp = chan->ll_region.vaddr.io + ofs;
writel(control, &llp->control);
writeq(pointer, &llp->llp.reg);
writel(control, &llp->control);
}
}
static void dw_hdma_v0_core_write_chunk(struct dw_edma_chunk *chunk)
static void dw_hdma_v0_core_ch_enable(struct dw_edma_chan *chan)
{
struct dw_edma_burst *child;
u32 control = 0, i = 0;
struct dw_edma *dw = chan->dw;
u32 tmp;
if (chunk->cb)
control = DW_HDMA_V0_CB;
list_for_each_entry(child, &chunk->burst->list, list)
dw_hdma_v0_write_ll_data(chunk, i++, control, child->sz,
child->sar, child->dar);
control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
if (!chunk->cb)
control |= DW_HDMA_V0_CB;
dw_hdma_v0_write_ll_link(chunk, i, control, chunk->ll_region.paddr);
/* Enable engine */
SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
/* Interrupt unmask - stop, abort */
tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
tmp = dw_hdma_v0_core_int_setup(chan, tmp);
SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
/* Channel control */
SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
/* Linked list */
/* llp is not aligned on 64bit -> keep 32bit accesses */
SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
lower_32_bits(chan->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, llp.msb,
upper_32_bits(chan->ll_region.paddr));
/* Set consumer cycle */
SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
}
static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
static void dw_hdma_v0_sync_ll_data(struct dw_edma_chan *chan)
{
/*
* In case of remote HDMA engine setup, the DW PCIe RP/EP internal
@@ -221,60 +292,16 @@ static void dw_hdma_v0_sync_ll_data(struct dw_edma_chunk *chunk)
* LL memory in a hope that the MRd TLP will return only after the
* last MWr TLP is completed
*/
if (!(chunk->chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
readl(chunk->ll_region.vaddr.io);
if (!(chan->dw->chip->flags & DW_EDMA_CHIP_LOCAL))
readl(chan->ll_region.vaddr.io);
}
static void dw_hdma_v0_core_ll_start(struct dw_edma_chunk *chunk, bool first)
static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chan *chan,
struct dw_edma_burst *child)
{
struct dw_edma_chan *chan = chunk->chan;
struct dw_edma *dw = chan->dw;
u32 tmp;
dw_hdma_v0_core_write_chunk(chunk);
if (first) {
/* Enable engine */
SET_CH_32(dw, chan->dir, chan->id, ch_en, BIT(0));
/* Interrupt unmask - stop, abort */
tmp = GET_CH_32(dw, chan->dir, chan->id, int_setup);
tmp &= ~(HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK);
/* Interrupt enable - stop, abort */
tmp |= HDMA_V0_LOCAL_STOP_INT_EN | HDMA_V0_LOCAL_ABORT_INT_EN;
if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL))
tmp |= HDMA_V0_REMOTE_STOP_INT_EN | HDMA_V0_REMOTE_ABORT_INT_EN;
SET_CH_32(dw, chan->dir, chan->id, int_setup, tmp);
/* Channel control */
SET_CH_32(dw, chan->dir, chan->id, control1, HDMA_V0_LINKLIST_EN);
/* Linked list */
/* llp is not aligned on 64bit -> keep 32bit accesses */
SET_CH_32(dw, chan->dir, chan->id, llp.lsb,
lower_32_bits(chunk->ll_region.paddr));
SET_CH_32(dw, chan->dir, chan->id, llp.msb,
upper_32_bits(chunk->ll_region.paddr));
/* Set consumer cycle */
SET_CH_32(dw, chan->dir, chan->id, cycle_sync,
HDMA_V0_CONSUMER_CYCLE_STAT | HDMA_V0_CONSUMER_CYCLE_BIT);
}
dw_hdma_v0_sync_ll_data(chunk);
/* Doorbell */
SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
}
static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
{
struct dw_edma_chan *chan = chunk->chan;
struct dw_edma *dw = chan->dw;
struct dw_edma_burst *child;
u32 val;
child = list_first_entry_or_null(&chunk->burst->list,
struct dw_edma_burst, list);
if (!child)
return;
SET_CH_32(dw, chan->dir, chan->id, ch_en, HDMA_V0_CH_EN);
/* Source address */
@@ -293,17 +320,8 @@ static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
SET_CH_32(dw, chan->dir, chan->id, transfer_size, child->sz);
/* Interrupt setup */
val = GET_CH_32(dw, chan->dir, chan->id, int_setup) |
HDMA_V0_STOP_INT_MASK |
HDMA_V0_ABORT_INT_MASK |
HDMA_V0_LOCAL_STOP_INT_EN |
HDMA_V0_LOCAL_ABORT_INT_EN;
if (!(dw->chip->flags & DW_EDMA_CHIP_LOCAL)) {
val |= HDMA_V0_REMOTE_STOP_INT_EN |
HDMA_V0_REMOTE_ABORT_INT_EN;
}
val = GET_CH_32(dw, chan->dir, chan->id, int_setup);
val = dw_hdma_v0_core_int_setup(chan, val);
SET_CH_32(dw, chan->dir, chan->id, int_setup, val);
/* Channel control setup */
@@ -315,16 +333,6 @@ static void dw_hdma_v0_core_non_ll_start(struct dw_edma_chunk *chunk)
HDMA_V0_DOORBELL_START);
}
static void dw_hdma_v0_core_start(struct dw_edma_chunk *chunk, bool first)
{
struct dw_edma_chan *chan = chunk->chan;
if (chan->non_ll)
dw_hdma_v0_core_non_ll_start(chunk);
else
dw_hdma_v0_core_ll_start(chunk, first);
}
static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
@@ -337,6 +345,43 @@ static void dw_hdma_v0_core_ch_config(struct dw_edma_chan *chan)
SET_CH_32(dw, chan->dir, chan->id, msi_abort.msb, chan->msi.address_hi);
/* config MSI data */
SET_CH_32(dw, chan->dir, chan->id, msi_msgdata, chan->msi.data);
/* Configure the requester function number used by outbound TLPs. */
SET_CH_32(dw, chan->dir, chan->id, func_num,
FIELD_PREP(HDMA_V0_FUNC_NUM_PF_MASK, chan->func_no));
}
static void
dw_hdma_v0_core_ll_data(struct dw_edma_chan *chan, struct dw_edma_burst *burst,
u32 idx, bool cb, bool irq)
{
u32 control = 0;
if (cb)
control |= DW_HDMA_V0_CB;
dw_hdma_v0_write_ll_data(chan, idx, control, burst->sz, burst->sar,
burst->dar);
}
static void
dw_hdma_v0_core_ll_link(struct dw_edma_chan *chan, u32 idx, bool cb, u64 addr)
{
u32 control = DW_HDMA_V0_LLP | DW_HDMA_V0_TCB;
if (!cb)
control |= DW_HDMA_V0_CB;
dw_hdma_v0_write_ll_link(chan, idx, control, addr);
}
static void dw_hdma_v0_core_ch_doorbell(struct dw_edma_chan *chan)
{
struct dw_edma *dw = chan->dw;
dw_hdma_v0_sync_ll_data(chan);
/* Doorbell */
SET_CH_32(dw, chan->dir, chan->id, doorbell, HDMA_V0_DOORBELL_START);
}
/* HDMA debugfs callbacks */
@@ -353,10 +398,16 @@ static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw)
static const struct dw_edma_core_ops dw_hdma_v0_core = {
.off = dw_hdma_v0_core_off,
.quiesce = dw_hdma_v0_core_quiesce,
.ch_quiesce = dw_hdma_v0_core_ch_quiesce,
.ch_count = dw_hdma_v0_core_ch_count,
.ch_status = dw_hdma_v0_core_ch_status,
.handle_int = dw_hdma_v0_core_handle_int,
.start = dw_hdma_v0_core_start,
.non_ll_start = dw_hdma_v0_core_non_ll_start,
.ll_data = dw_hdma_v0_core_ll_data,
.ll_link = dw_hdma_v0_core_ll_link,
.ch_doorbell = dw_hdma_v0_core_ch_doorbell,
.ch_enable = dw_hdma_v0_core_ch_enable,
.ch_config = dw_hdma_v0_core_ch_config,
.debugfs_on = dw_hdma_v0_core_debugfs_on,
.db_offset = dw_hdma_v0_core_db_offset,

3
drivers/dma/dw-edma/dw-hdma-v0-regs.h
View File

@@ -11,7 +11,7 @@
#include <linux/dmaengine.h>
#define HDMA_V0_MAX_NR_CH 8
#define HDMA_V0_MAX_NR_CH 64
#define HDMA_V0_CH_EN BIT(0)
#define HDMA_V0_LOCAL_ABORT_INT_EN BIT(6)
#define HDMA_V0_REMOTE_ABORT_INT_EN BIT(5)
@@ -24,6 +24,7 @@
#define HDMA_V0_CONSUMER_CYCLE_BIT BIT(0)
#define HDMA_V0_DOORBELL_START BIT(0)
#define HDMA_V0_CH_STATUS_MASK GENMASK(1, 0)
#define HDMA_V0_FUNC_NUM_PF_MASK GENMASK(7, 0)
struct dw_hdma_v0_ch_regs {
u32 ch_en; /* 0x0000 */

2
drivers/dma/fsl-edma-main.c
View File

@@ -414,6 +414,8 @@ static int fsl_edma3_irq_init(struct platform_device *pdev, struct fsl_edma_engi
errirq_name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s-err",
dev_name(&pdev->dev));
if (!errirq_name)
return -ENOMEM;
ret = devm_request_irq(&pdev->dev, fsl_edma->errirq, fsl_edma3_err_handler_shared,
0, errirq_name, fsl_edma);

4
drivers/dma/fsl-edma-trace.h
View File

@@ -19,14 +19,16 @@ DECLARE_EVENT_CLASS(edma_log_io,
__field(struct fsl_edma_engine *, edma)
__field(void __iomem *, addr)
__field(u32, value)
__field(void __iomem *, membase)
),
TP_fast_assign(
__entry->edma = edma;
__entry->addr = addr;
__entry->value = value;
__entry->membase = edma->membase;
),
TP_printk("offset %08x: value %08x",
(u32)(__entry->addr - __entry->edma->membase), __entry->value)
(u32)(__entry->addr - __entry->membase), __entry->value)
);
DEFINE_EVENT(edma_log_io, edma_readl,

58
drivers/dma/fsl_raid.c
View File

@@ -242,9 +242,9 @@ static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index,
u32 efrl = length & FSL_RE_CF_LENGTH_MASK;
efrl |= final << FSL_RE_CF_FINAL_SHIFT;
cf[index].efrl32 = efrl;
cf[index].addr_high = upper_32_bits(addr);
cf[index].addr_low = lower_32_bits(addr);
cf[index].efrl32 = cpu_to_be32(efrl);
cf[index].addr_high = cpu_to_be32(upper_32_bits(addr));
cf[index].addr_low = cpu_to_be32(lower_32_bits(addr));
}
static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,
@@ -256,9 +256,10 @@ static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan,
dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan);
INIT_LIST_HEAD(&desc->node);
desc->hwdesc.fmt32 = FSL_RE_FRAME_FORMAT << FSL_RE_HWDESC_FMT_SHIFT;
desc->hwdesc.lbea32 = upper_32_bits(paddr);
desc->hwdesc.addr_low = lower_32_bits(paddr);
desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT <<
FSL_RE_HWDESC_FMT_SHIFT);
desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr));
desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr));
desc->cf_addr = cf;
desc->cf_paddr = paddr;
@@ -374,11 +375,11 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq(
for (i = 2, j = 0; j < save_src_cnt; i++, j++)
fill_cfd_frame(cf, i, len, src[j], 0);
/* Fill the last frame and mark it final */
if (cont_q)
fill_cfd_frame(cf, i++, len, dest, 0);
/* Setting the final bit in the last source buffer frame in CFD */
cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
fill_cfd_frame(cf, i, len, dest, 1);
else
fill_cfd_frame(cf, i - 1, len, src[j - 1], 1);
return &desc->async_tx;
}
@@ -504,16 +505,16 @@ static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq(
p[save_src_cnt + 2] = 1;
fill_cfd_frame(cf, i++, len, dest[0], 0);
fill_cfd_frame(cf, i++, len, dest[1], 0);
fill_cfd_frame(cf, i++, len, dest[1], 0);
fill_cfd_frame(cf, i++, len, dest[1], 1);
} else {
dev_err(re_chan->dev, "PQ tx continuation error!\n");
return NULL;
}
} else {
/* Mark the last source buffer frame final */
fill_cfd_frame(cf, i - 1, len, src[j - 1], 1);
}
/* Setting the final bit in the last source buffer frame in CFD */
cf[i - 1].efrl32 |= 1 << FSL_RE_CF_FINAL_SHIFT;
return &desc->async_tx;
}
@@ -656,8 +657,7 @@ static int fsl_re_chan_probe(struct platform_device *ofdev,
goto err_free;
}
chan->jrregs = (struct fsl_re_chan_cfg *)((u8 *)re_priv->re_regs +
off + ptr);
chan->jrregs = re_priv->base + off + ptr;
/* read irq property from dts */
chan->irq = irq_of_parse_and_map(np, 0);
@@ -745,38 +745,36 @@ static int fsl_re_chan_probe(struct platform_device *ofdev,
/* Probe function for RAID Engine */
static int fsl_re_probe(struct platform_device *ofdev)
{
struct fsl_re_ctrl __iomem *re_regs;
struct fsl_re_drv_private *re_priv;
struct device_node *child;
u32 off;
u8 ridx = 0;
struct dma_device *dma_dev;
struct resource *res;
int rc;
struct device *dev = &ofdev->dev;
/* IOMAP the entire RAID Engine region */
re_regs = devm_platform_ioremap_resource(ofdev, 0);
if (IS_ERR(re_regs))
return PTR_ERR(re_regs);
re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL);
if (!re_priv)
return -ENOMEM;
res = platform_get_resource(ofdev, IORESOURCE_MEM, 0);
if (!res)
return -ENODEV;
/* IOMAP the entire RAID Engine region */
re_priv->re_regs = devm_ioremap(dev, res->start, resource_size(res));
if (!re_priv->re_regs)
return -EBUSY;
re_priv->base = re_regs;
/* Program the RE mode */
out_be32(&re_priv->re_regs->global_config, FSL_RE_NON_DPAA_MODE);
out_be32(&re_regs->global_config, FSL_RE_NON_DPAA_MODE);
/* Program Galois Field polynomial */
out_be32(&re_priv->re_regs->galois_field_config, FSL_RE_GFM_POLY);
out_be32(&re_regs->galois_field_config, FSL_RE_GFM_POLY);
dev_info(dev, "version %x, mode %x, gfp %x\n",
in_be32(&re_priv->re_regs->re_version_id),
in_be32(&re_priv->re_regs->global_config),
in_be32(&re_priv->re_regs->galois_field_config));
in_be32(&re_regs->re_version_id),
in_be32(&re_regs->global_config),
in_be32(&re_regs->galois_field_config));
dma_dev = &re_priv->dma_dev;
dma_dev->dev = dev;

4
drivers/dma/fsl_raid.h
View File

@@ -256,7 +256,7 @@ struct fsl_re_hw_desc {
struct fsl_re_drv_private {
u8 total_chans;
struct dma_device dma_dev;
struct fsl_re_ctrl *re_regs;
void __iomem *base;
struct fsl_re_chan *re_jrs[FSL_RE_MAX_CHANS];
struct dma_pool *cf_desc_pool;
struct dma_pool *hw_desc_pool;
@@ -273,7 +273,7 @@ struct fsl_re_chan {
struct device *dev;
struct fsl_re_drv_private *re_dev;
struct dma_chan chan;
struct fsl_re_chan_cfg *jrregs;
struct fsl_re_chan_cfg __iomem *jrregs;
int irq;
struct tasklet_struct irqtask;
u32 alloc_count;

2
drivers/dma/hisi_dma.c
View File

@@ -983,7 +983,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
GFP_KERNEL);
if (!hdma_dev)
return -EINVAL;
return -ENOMEM;
hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
hdma_dev->pdev = pdev;

12
drivers/dma/idxd/defaults.c
View File

@@ -8,6 +8,7 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd)
struct idxd_engine *engine;
struct idxd_group *group;
struct idxd_wq *wq;
int i;
if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags))
return 0;
@@ -41,11 +42,12 @@ int idxd_load_iaa_device_defaults(struct idxd_device *idxd)
/* set driver_name to "crypto" */
strscpy_pad(wq->driver_name, "crypto");
engine = idxd->engines[0];
/* set engine group to 0 */
engine->group = idxd->groups[0];
engine->group->num_engines++;
/* assign all engines to group 0 */
for (i = 0; i < idxd->max_engines; i++) {
engine = idxd->engines[i];
engine->group = group;
group->num_engines++;
}
return 0;
}

2
drivers/dma/mediatek/mtk-uart-apdma.c
View File

@@ -531,7 +531,7 @@ static int mtk_uart_apdma_probe(struct platform_device *pdev)
for (i = 0; i < mtkd->dma_requests; i++) {
c = devm_kzalloc(mtkd->ddev.dev, sizeof(*c), GFP_KERNEL);
if (!c) {
rc = -ENODEV;
rc = -ENOMEM;
goto err_no_dma;
}

19
drivers/dma/pl330.c
View File

@@ -502,6 +502,7 @@ struct pl330_dmac {
struct dma_pl330_chan *peripherals; /* keep at end */
int quirks;
struct dentry *dbgfs;
struct reset_control *rstc;
struct reset_control *rstc_ocp;
};
@@ -2952,14 +2953,24 @@ DEFINE_SHOW_ATTRIBUTE(pl330_debugfs);
static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
{
debugfs_create_file(dev_name(pl330->ddma.dev),
S_IFREG | 0444, NULL, pl330,
&pl330_debugfs_fops);
pl330->dbgfs = debugfs_create_file(dev_name(pl330->ddma.dev),
S_IFREG | 0444, NULL, pl330,
&pl330_debugfs_fops);
}
static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330)
{
debugfs_remove(pl330->dbgfs);
pl330->dbgfs = NULL;
}
#else
static inline void init_pl330_debugfs(struct pl330_dmac *pl330)
{
}
static inline void deinit_pl330_debugfs(struct pl330_dmac *pl330)
{
}
#endif
/*
@@ -3204,6 +3215,8 @@ static void pl330_remove(struct amba_device *adev)
struct dma_pl330_chan *pch, *_p;
int i, irq;
deinit_pl330_debugfs(pl330);
pm_runtime_get_noresume(pl330->ddma.dev);
if (adev->dev.of_node)

1
drivers/dma/qcom/bam_dma.c
View File

@@ -1401,6 +1401,7 @@ static void bam_dma_remove(struct platform_device *pdev)
struct bam_device *bdev = platform_get_drvdata(pdev);
u32 i;
pm_runtime_dont_use_autosuspend(&pdev->dev);
pm_runtime_force_suspend(&pdev->dev);
of_dma_controller_free(pdev->dev.of_node);

6
drivers/dma/qcom/hidma_ll.c
View File

@@ -677,11 +677,7 @@ int hidma_ll_setup(struct hidma_lldev *lldev)
/* configure interrupts */
hidma_ll_setup_irq(lldev, lldev->msi_support);
rc = hidma_ll_enable(lldev);
if (rc)
return rc;
return rc;
return hidma_ll_enable(lldev);
}
void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)

12
drivers/dma/tegra210-adma.c
View File

@@ -1087,15 +1087,9 @@ static int tegra_adma_probe(struct platform_device *pdev)
}
} else {
/* If no 'page' property found, then reg DT binding would be legacy */
res_base = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (res_base) {
tdma->base_addr = devm_ioremap_resource(&pdev->dev, res_base);
if (IS_ERR(tdma->base_addr))
return PTR_ERR(tdma->base_addr);
} else {
return dev_err_probe(&pdev->dev, -ENODEV,
"failed to get memory resource\n");
}
tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
if (IS_ERR(tdma->base_addr))
return PTR_ERR(tdma->base_addr);
tdma->ch_base_addr = tdma->base_addr + cdata->ch_base_offset;
}

2
drivers/dma/ti/k3-udma.c
View File

@@ -3408,7 +3408,7 @@ static int udma_set_metadata_len(struct dma_async_tx_descriptor *desc,
return 0;
}
static struct dma_descriptor_metadata_ops metadata_ops = {
static const struct dma_descriptor_metadata_ops metadata_ops = {
.attach = udma_attach_metadata,
.get_ptr = udma_get_metadata_ptr,
.set_len = udma_set_metadata_len,

66
drivers/dma/xilinx/xilinx_dma.c
View File

@@ -165,8 +165,10 @@
#define XILINX_DMA_FLUSH_MM2S 2
#define XILINX_DMA_FLUSH_BOTH 1
/* Delay loop counter to prevent hardware failure */
#define XILINX_DMA_LOOP_COUNT 1000000
/* Timeout for polling various registers */
#define XILINX_DMA_POLL_TIMEOUT_US 1000000
/* Delay between polls (avoid a delay of 0 to prevent CPU stalls) */
#define XILINX_DMA_POLL_DELAY_US 10
/* AXI DMA Specific Registers/Offsets */
#define XILINX_DMA_REG_SRCDSTADDR 0x18
@@ -653,7 +655,7 @@ static void *xilinx_dma_get_metadata_ptr(struct dma_async_tx_descriptor *tx,
return seg->hw.app;
}
static struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {
static const struct dma_descriptor_metadata_ops xilinx_dma_metadata_ops = {
.get_ptr = xilinx_dma_get_metadata_ptr,
};
@@ -1324,8 +1326,9 @@ static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)
/* Wait for the hardware to halt */
return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
val & XILINX_DMA_DMASR_HALTED, 0,
XILINX_DMA_LOOP_COUNT);
val & XILINX_DMA_DMASR_HALTED,
XILINX_DMA_POLL_DELAY_US,
XILINX_DMA_POLL_TIMEOUT_US);
}
/**
@@ -1339,8 +1342,9 @@ static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)
u32 val;
return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
val & XILINX_DMA_DMASR_IDLE, 0,
XILINX_DMA_LOOP_COUNT);
val & XILINX_DMA_DMASR_IDLE,
XILINX_DMA_POLL_DELAY_US,
XILINX_DMA_POLL_TIMEOUT_US);
}
/**
@@ -1356,8 +1360,9 @@ static void xilinx_dma_start(struct xilinx_dma_chan *chan)
/* Wait for the hardware to start */
err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,
!(val & XILINX_DMA_DMASR_HALTED), 0,
XILINX_DMA_LOOP_COUNT);
!(val & XILINX_DMA_DMASR_HALTED),
XILINX_DMA_POLL_DELAY_US,
XILINX_DMA_POLL_TIMEOUT_US);
if (err) {
dev_err(chan->dev, "Cannot start channel %p: %x\n",
@@ -1580,7 +1585,14 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
return;
}
if (!chan->idle)
/*
* Direct (non-SG) mode has no descriptor queue: writing the BTT
* register launches a transfer immediately, so a new transfer must
* not be programmed while one is in flight. Keep such transfers
* serialized. SG mode supports chaining onto a running transfer via
* tail-pointer extension, so it is allowed to proceed when busy.
*/
if (!chan->has_sg && !chan->idle)
return;
head_desc = list_first_entry(&chan->pending_list,
@@ -1596,10 +1608,9 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
reg &= ~XILINX_DMA_CR_COALESCE_MAX;
reg |= chan->desc_pendingcount <<
XILINX_DMA_CR_COALESCE_SHIFT;
dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
}
if (chan->has_sg)
if (chan->has_sg && list_empty(&chan->active_list))
xilinx_write(chan, XILINX_DMA_REG_CURDESC,
head_desc->async_tx.phys);
reg &= ~XILINX_DMA_CR_DELAY_MAX;
@@ -1607,7 +1618,8 @@ static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)
reg |= XILINX_DMA_DMAXR_ALL_IRQ_MASK;
dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);
xilinx_dma_start(chan);
if (chan->idle)
xilinx_dma_start(chan);
if (chan->err)
return;
@@ -1660,9 +1672,6 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
if (chan->err)
return;
if (!chan->idle)
return;
if (list_empty(&chan->pending_list))
return;
@@ -1685,8 +1694,9 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
/* Program current descriptor */
xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
head_desc->async_tx.phys);
if (chan->has_sg && list_empty(&chan->active_list))
xilinx_write(chan, XILINX_MCDMA_CHAN_CDESC_OFFSET(chan->tdest),
head_desc->async_tx.phys);
/* Program channel enable register */
reg = dma_ctrl_read(chan, XILINX_MCDMA_CHEN_OFFSET);
@@ -1698,7 +1708,8 @@ static void xilinx_mcdma_start_transfer(struct xilinx_dma_chan *chan)
reg |= XILINX_MCDMA_CR_RUNSTOP_MASK;
dma_ctrl_write(chan, XILINX_MCDMA_CHAN_CR_OFFSET(chan->tdest), reg);
xilinx_dma_start(chan);
if (chan->idle)
xilinx_dma_start(chan);
if (chan->err)
return;
@@ -1794,8 +1805,9 @@ static int xilinx_dma_reset(struct xilinx_dma_chan *chan)
/* Wait for the hardware to finish reset */
err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp,
!(tmp & XILINX_DMA_DMACR_RESET), 0,
XILINX_DMA_LOOP_COUNT);
!(tmp & XILINX_DMA_DMACR_RESET),
XILINX_DMA_POLL_DELAY_US,
XILINX_DMA_POLL_TIMEOUT_US);
if (err) {
dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",
@@ -1893,8 +1905,10 @@ static irqreturn_t xilinx_mcdma_irq_handler(int irq, void *data)
if (status & XILINX_MCDMA_IRQ_IOC_MASK) {
spin_lock(&chan->lock);
xilinx_dma_complete_descriptor(chan);
chan->idle = true;
chan->start_transfer(chan);
if (list_empty(&chan->active_list)) {
chan->idle = true;
chan->start_transfer(chan);
}
spin_unlock(&chan->lock);
}
@@ -1950,8 +1964,10 @@ static irqreturn_t xilinx_dma_irq_handler(int irq, void *data)
XILINX_DMA_DMASR_DLY_CNT_IRQ)) {
spin_lock(&chan->lock);
xilinx_dma_complete_descriptor(chan);
chan->idle = true;
chan->start_transfer(chan);
if (list_empty(&chan->active_list)) {
chan->idle = true;
chan->start_transfer(chan);
}
spin_unlock(&chan->lock);
}

6
drivers/dma/xilinx/zynqmp_dma.c
View File

@@ -1159,8 +1159,6 @@ static int zynqmp_dma_probe(struct platform_device *pdev)
/**
* zynqmp_dma_remove - Driver remove function
* @pdev: Pointer to the platform_device structure
*
* Return: Always '0'
*/
static void zynqmp_dma_remove(struct platform_device *pdev)
{
@@ -1170,9 +1168,9 @@ static void zynqmp_dma_remove(struct platform_device *pdev)
dma_async_device_unregister(&zdev->common);
zynqmp_dma_chan_remove(zdev->chan);
if (pm_runtime_active(zdev->dev))
zynqmp_dma_runtime_suspend(zdev->dev);
pm_runtime_disable(zdev->dev);
if (!pm_runtime_status_suspended(zdev->dev))
zynqmp_dma_runtime_suspend(zdev->dev);
}
static const struct of_device_id zynqmp_dma_of_match[] = {

33
drivers/nvme/target/pci-epf.c
View File

@@ -210,9 +210,7 @@ struct nvmet_pci_epf {
bool dma_enabled;
struct dma_chan *dma_tx_chan;
struct mutex dma_tx_lock;
struct dma_chan *dma_rx_chan;
struct mutex dma_rx_lock;
struct mutex mmio_lock;
@@ -295,9 +293,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf)
struct dma_chan *chan;
dma_cap_mask_t mask;
mutex_init(&nvme_epf->dma_rx_lock);
mutex_init(&nvme_epf->dma_tx_lock);
dma_cap_zero(mask);
dma_cap_set(DMA_SLAVE, mask);
@@ -336,8 +331,6 @@ static void nvmet_pci_epf_init_dma(struct nvmet_pci_epf *nvme_epf)
nvme_epf->dma_rx_chan = NULL;
out_dma_no_rx:
mutex_destroy(&nvme_epf->dma_rx_lock);
mutex_destroy(&nvme_epf->dma_tx_lock);
nvme_epf->dma_enabled = false;
dev_info(&epf->dev, "DMA not supported, falling back to MMIO\n");
@@ -352,8 +345,6 @@ static void nvmet_pci_epf_deinit_dma(struct nvmet_pci_epf *nvme_epf)
nvme_epf->dma_tx_chan = NULL;
dma_release_channel(nvme_epf->dma_rx_chan);
nvme_epf->dma_rx_chan = NULL;
mutex_destroy(&nvme_epf->dma_rx_lock);
mutex_destroy(&nvme_epf->dma_tx_lock);
nvme_epf->dma_enabled = false;
}
@@ -368,18 +359,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
struct dma_chan *chan;
dma_cookie_t cookie;
dma_addr_t dma_addr;
struct mutex *lock;
int ret;
switch (dir) {
case DMA_FROM_DEVICE:
lock = &nvme_epf->dma_rx_lock;
chan = nvme_epf->dma_rx_chan;
sconf.direction = DMA_DEV_TO_MEM;
sconf.src_addr = seg->pci_addr;
break;
case DMA_TO_DEVICE:
lock = &nvme_epf->dma_tx_lock;
chan = nvme_epf->dma_tx_chan;
sconf.direction = DMA_MEM_TO_DEV;
sconf.dst_addr = seg->pci_addr;
@@ -388,22 +376,15 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
return -EINVAL;
}
mutex_lock(lock);
dma_dev = dmaengine_get_dma_device(chan);
dma_addr = dma_map_single(dma_dev, seg->buf, seg->length, dir);
ret = dma_mapping_error(dma_dev, dma_addr);
if (ret)
goto unlock;
return ret;
ret = dmaengine_slave_config(chan, &sconf);
if (ret) {
dev_err(dev, "Failed to configure DMA channel\n");
goto unmap;
}
desc = dmaengine_prep_slave_single(chan, dma_addr, seg->length,
sconf.direction, DMA_CTRL_ACK);
desc = dmaengine_prep_config_single_safe(chan, dma_addr, seg->length,
sconf.direction,
DMA_CTRL_ACK, &sconf);
if (!desc) {
dev_err(dev, "Failed to prepare DMA\n");
ret = -EIO;
@@ -420,16 +401,12 @@ static int nvmet_pci_epf_dma_transfer(struct nvmet_pci_epf *nvme_epf,
if (dma_sync_wait(chan, cookie) != DMA_COMPLETE) {
dev_err(dev, "DMA transfer failed\n");
ret = -EIO;
dmaengine_terminate_sync(chan);
}
dmaengine_terminate_sync(chan);
unmap:
dma_unmap_single(dma_dev, dma_addr, seg->length, dir);
unlock:
mutex_unlock(lock);
return ret;
}

52
drivers/pci/endpoint/functions/pci-epf-mhi.c
View File

@@ -328,12 +328,6 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
config.direction = DMA_DEV_TO_MEM;
config.src_addr = buf_info->host_addr;
ret = dmaengine_slave_config(chan, &config);
if (ret) {
dev_err(dev, "Failed to configure DMA channel\n");
goto err_unlock;
}
dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
DMA_FROM_DEVICE);
ret = dma_mapping_error(dma_dev, dst_addr);
@@ -342,9 +336,10 @@ static int pci_epf_mhi_edma_read(struct mhi_ep_cntrl *mhi_cntrl,
goto err_unlock;
}
desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
DMA_DEV_TO_MEM,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size,
DMA_DEV_TO_MEM,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
&config);
if (!desc) {
dev_err(dev, "Failed to prepare DMA\n");
ret = -EIO;
@@ -401,12 +396,6 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
config.direction = DMA_MEM_TO_DEV;
config.dst_addr = buf_info->host_addr;
ret = dmaengine_slave_config(chan, &config);
if (ret) {
dev_err(dev, "Failed to configure DMA channel\n");
goto err_unlock;
}
src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
DMA_TO_DEVICE);
ret = dma_mapping_error(dma_dev, src_addr);
@@ -415,9 +404,10 @@ static int pci_epf_mhi_edma_write(struct mhi_ep_cntrl *mhi_cntrl,
goto err_unlock;
}
desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
DMA_MEM_TO_DEV,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size,
DMA_MEM_TO_DEV,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
&config);
if (!desc) {
dev_err(dev, "Failed to prepare DMA\n");
ret = -EIO;
@@ -506,12 +496,6 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl,
config.direction = DMA_DEV_TO_MEM;
config.src_addr = buf_info->host_addr;
ret = dmaengine_slave_config(chan, &config);
if (ret) {
dev_err(dev, "Failed to configure DMA channel\n");
goto err_unlock;
}
dst_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
DMA_FROM_DEVICE);
ret = dma_mapping_error(dma_dev, dst_addr);
@@ -520,9 +504,10 @@ static int pci_epf_mhi_edma_read_async(struct mhi_ep_cntrl *mhi_cntrl,
goto err_unlock;
}
desc = dmaengine_prep_slave_single(chan, dst_addr, buf_info->size,
DMA_DEV_TO_MEM,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
desc = dmaengine_prep_config_single(chan, dst_addr, buf_info->size,
DMA_DEV_TO_MEM,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
&config);
if (!desc) {
dev_err(dev, "Failed to prepare DMA\n");
ret = -EIO;
@@ -585,12 +570,6 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl,
config.direction = DMA_MEM_TO_DEV;
config.dst_addr = buf_info->host_addr;
ret = dmaengine_slave_config(chan, &config);
if (ret) {
dev_err(dev, "Failed to configure DMA channel\n");
goto err_unlock;
}
src_addr = dma_map_single(dma_dev, buf_info->dev_addr, buf_info->size,
DMA_TO_DEVICE);
ret = dma_mapping_error(dma_dev, src_addr);
@@ -599,9 +578,10 @@ static int pci_epf_mhi_edma_write_async(struct mhi_ep_cntrl *mhi_cntrl,
goto err_unlock;
}
desc = dmaengine_prep_slave_single(chan, src_addr, buf_info->size,
DMA_MEM_TO_DEV,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT);
desc = dmaengine_prep_config_single(chan, src_addr, buf_info->size,
DMA_MEM_TO_DEV,
DMA_CTRL_ACK | DMA_PREP_INTERRUPT,
&config);
if (!desc) {
dev_err(dev, "Failed to prepare DMA\n");
ret = -EIO;

8
drivers/pci/endpoint/functions/pci-epf-test.c
View File

@@ -183,12 +183,8 @@ static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
else
sconf.src_addr = dma_remote;
if (dmaengine_slave_config(chan, &sconf)) {
dev_err(dev, "DMA slave config fail\n");
return -EIO;
}
tx = dmaengine_prep_slave_single(chan, dma_local, len, dir,
flags);
tx = dmaengine_prep_config_single(chan, dma_local, len,
dir, flags, &sconf);
} else {
tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len,
flags);

51
include/linux/dma/edma.h
View File

@@ -14,6 +14,8 @@
#define EDMA_MAX_WR_CH 8
#define EDMA_MAX_RD_CH 8
#define HDMA_MAX_WR_CH 64
#define HDMA_MAX_RD_CH 64
struct dw_edma;
@@ -55,9 +57,48 @@ enum dw_edma_map_format {
/**
* enum dw_edma_chip_flags - Flags specific to an eDMA chip
* @DW_EDMA_CHIP_LOCAL: eDMA is used locally by an endpoint
* @DW_EDMA_CHIP_PARTIAL: Only channels described by this instance are
* owned by this driver. Controller-wide state
* must be preserved, and layouts with shared
* direction-wide registers must only be shared at
* direction granularity. Layouts with per-channel
* registers may be shared at channel granularity.
*/
enum dw_edma_chip_flags {
DW_EDMA_CHIP_LOCAL = BIT(0),
DW_EDMA_CHIP_PARTIAL = BIT(1),
};
/**
* enum dw_edma_ch_irq_mode - per-channel interrupt routing control
* @DW_EDMA_CH_IRQ_LOCAL: local interrupt only (edma_int[])
* @DW_EDMA_CH_IRQ_REMOTE: remote interrupt only (IMWr/MSI), without
* delivering local edma_int[].
*
* DesignWare EP eDMA can signal interrupts locally through the edma_int[]
* bus, and remotely using posted memory writes (IMWr) that may be
* interpreted as MSI/MSI-X by the RC.
*
* For the v0 eDMA linked-list programming path, DMA_*_INT_MASK gates the local
* edma_int[] assertion, while there is no dedicated per-channel mask for IMWr
* generation. To request a remote-only interrupt, Synopsys recommends setting
* both LIE and RIE, and masking the local interrupt in DMA_*_INT_MASK. See the
* DesignWare endpoint databook 6.30a, Linked List Mode interrupt handling
* ("Software Programming of an Endpoint's LIE and RIE Bits for Linked List
* Transfers", Attention).
*
* A local (DW_EDMA_CHIP_LOCAL) instance never issues transfers on a
* remote-routed channel: REMOTE routing on such an instance denotes a channel
* handed over to and driven by the remote side, and the recipe above is
* applied by the driving instance.
*
* HDMA linked-list watermark interrupts have the same LWIE/RWIE guidance. HDMA
* non-linked-list mode has dedicated local and remote stop/abort interrupt
* enables.
*/
enum dw_edma_ch_irq_mode {
DW_EDMA_CH_IRQ_LOCAL = 0,
DW_EDMA_CH_IRQ_REMOTE,
};
/**
@@ -76,6 +117,7 @@ enum dw_edma_chip_flags {
* @db_irq: Virtual IRQ dedicated to interrupt emulation
* @db_offset: Offset from DMA register base
* @mf: DMA register map format
* @func_no: PCI endpoint function number used by DMA TLPs
* @dw: struct dw_edma that is filled by dw_edma_probe()
*/
struct dw_edma_chip {
@@ -89,18 +131,19 @@ struct dw_edma_chip {
u16 ll_wr_cnt;
u16 ll_rd_cnt;
/* link list address */
struct dw_edma_region ll_region_wr[EDMA_MAX_WR_CH];
struct dw_edma_region ll_region_rd[EDMA_MAX_RD_CH];
struct dw_edma_region ll_region_wr[HDMA_MAX_WR_CH];
struct dw_edma_region ll_region_rd[HDMA_MAX_RD_CH];
/* data region */
struct dw_edma_region dt_region_wr[EDMA_MAX_WR_CH];
struct dw_edma_region dt_region_rd[EDMA_MAX_RD_CH];
struct dw_edma_region dt_region_wr[HDMA_MAX_WR_CH];
struct dw_edma_region dt_region_rd[HDMA_MAX_RD_CH];
/* interrupt emulation */
int db_irq;
resource_size_t db_offset;
enum dw_edma_map_format mf;
u8 func_no;
struct dw_edma *dw;
bool cfg_non_ll;

151
include/linux/dmaengine.h
View File

@@ -322,6 +322,8 @@ struct dma_router {
* @slave: ptr to the device using this channel
* @cookie: last cookie value returned to client
* @completed_cookie: last completed cookie for this channel
* @lock: protect between config and prepare transfer when driver have not
* implemented callback device_prep_config_sg().
* @chan_id: channel ID for sysfs
* @dev: class device for sysfs
* @name: backlink name for sysfs
@@ -341,6 +343,12 @@ struct dma_chan {
dma_cookie_t cookie;
dma_cookie_t completed_cookie;
/*
* protect between config and prepare transfer because *_prep() may be
* called from complete callback, which is in GFP_NOSLEEP context.
*/
spinlock_t lock;
/* sysfs */
int chan_id;
struct dma_chan_dev *dev;
@@ -623,7 +631,7 @@ struct dma_async_tx_descriptor {
void *callback_param;
struct dmaengine_unmap_data *unmap;
enum dma_desc_metadata_mode desc_metadata_mode;
struct dma_descriptor_metadata_ops *metadata_ops;
const struct dma_descriptor_metadata_ops *metadata_ops;
#ifdef CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH
struct dma_async_tx_descriptor *next;
struct dma_async_tx_descriptor *parent;
@@ -835,6 +843,7 @@ struct dma_filter {
* where the address and size of each segment is located in one entry of
* the dma_vec array.
* @device_prep_slave_sg: prepares a slave dma operation
* @device_prep_config_sg: prepares a slave DMA operation with dma_slave_config
* @device_prep_dma_cyclic: prepare a cyclic dma operation suitable for audio.
* The function takes a buffer of size buf_len. The callback function will
* be called after period_len bytes have been transferred.
@@ -934,6 +943,10 @@ struct dma_device {
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
unsigned long flags, void *context);
struct dma_async_tx_descriptor *(*device_prep_config_sg)(
struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction direction,
unsigned long flags, struct dma_slave_config *config);
struct dma_async_tx_descriptor *(*device_prep_dma_cyclic)(
struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
size_t period_len, enum dma_transfer_direction direction,
@@ -974,22 +987,44 @@ static inline bool is_slave_direction(enum dma_transfer_direction direction)
(direction == DMA_DEV_TO_DEV);
}
static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_single(
struct dma_chan *chan, dma_addr_t buf, size_t len,
enum dma_transfer_direction dir, unsigned long flags)
static inline struct dma_async_tx_descriptor *
dmaengine_prep_config_single(struct dma_chan *chan, dma_addr_t buf, size_t len,
enum dma_transfer_direction dir,
unsigned long flags,
struct dma_slave_config *config)
{
struct scatterlist sg;
if (!chan || !chan->device)
return NULL;
sg_init_table(&sg, 1);
sg_dma_address(&sg) = buf;
sg_dma_len(&sg) = len;
if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
if (chan->device->device_prep_config_sg)
return chan->device->device_prep_config_sg(chan, &sg, 1, dir,
flags, config);
if (config)
if (dmaengine_slave_config(chan, config))
return NULL;
if (!chan->device->device_prep_slave_sg)
return NULL;
return chan->device->device_prep_slave_sg(chan, &sg, 1,
dir, flags, NULL);
}
static inline struct dma_async_tx_descriptor *
dmaengine_prep_slave_single(struct dma_chan *chan, dma_addr_t buf, size_t len,
enum dma_transfer_direction dir,
unsigned long flags)
{
return dmaengine_prep_config_single(chan, buf, len, dir, flags, NULL);
}
/**
* dmaengine_prep_peripheral_dma_vec() - Prepare a DMA scatter-gather descriptor
* @chan: The channel to be used for this descriptor
@@ -1010,17 +1045,115 @@ static inline struct dma_async_tx_descriptor *dmaengine_prep_peripheral_dma_vec(
dir, flags);
}
static inline struct dma_async_tx_descriptor *dmaengine_prep_slave_sg(
struct dma_chan *chan, struct scatterlist *sgl, unsigned int sg_len,
enum dma_transfer_direction dir, unsigned long flags)
static inline struct dma_async_tx_descriptor *
dmaengine_prep_config_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction dir,
unsigned long flags, struct dma_slave_config *config)
{
if (!chan || !chan->device || !chan->device->device_prep_slave_sg)
if (!chan || !chan->device)
return NULL;
if (chan->device->device_prep_config_sg)
return chan->device->device_prep_config_sg(chan, sgl, sg_len,
dir, flags, config);
if (config)
if (dmaengine_slave_config(chan, config))
return NULL;
if (!chan->device->device_prep_slave_sg)
return NULL;
return chan->device->device_prep_slave_sg(chan, sgl, sg_len,
dir, flags, NULL);
}
static inline struct dma_async_tx_descriptor *
dmaengine_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len, enum dma_transfer_direction dir,
unsigned long flags)
{
return dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, NULL);
}
/**
* dmaengine_prep_config_sg_safe - prepare a scatter-gather DMA transfer
* with atomic slave configuration update
* @chan: DMA channel
* @sgl: scatterlist for the transfer
* @sg_len: number of entries in @sgl
* @dir: DMA transfer direction
* @flags: transfer preparation flags
* @config: DMA slave configuration for this transfer
*
* Prepare a DMA scatter-gather transfer together with a corresponding slave
* configuration update in a re-entrant and race-safe manner.
*
* DMA engine drivers may implement the optional
* device_prep_config_sg() callback to perform both the slave configuration
* and descriptor preparation atomically. In this case, the operation is
* fully handled by the DMA engine driver.
*
* If the DMA engine driver does not implement device_prep_config_sg(), falls
* back to calling dmaengine_slave_config() followed by dmaengine_prep_slave_sg().
* The fallback path is protected by a per-channel spinlock to ensure that
* concurrent callers cannot interleave configuration and descriptor preparation
* on the same DMA channel.
*
* Return: Pointer to a prepared DMA async transaction descriptor on success,
* or %NULL if the transfer could not be prepared.
*/
static inline struct dma_async_tx_descriptor *
dmaengine_prep_config_sg_safe(struct dma_chan *chan, struct scatterlist *sgl,
unsigned int sg_len,
enum dma_transfer_direction dir,
unsigned long flags,
struct dma_slave_config *config)
{
struct dma_async_tx_descriptor *tx;
unsigned long spinlock_flags;
if (!chan || !chan->device)
return NULL;
if (!chan->device->device_prep_config_sg)
spin_lock_irqsave(&chan->lock, spinlock_flags);
tx = dmaengine_prep_config_sg(chan, sgl, sg_len, dir, flags, config);
if (!chan->device->device_prep_config_sg)
spin_unlock_irqrestore(&chan->lock, spinlock_flags);
return tx;
}
/**
* dmaengine_prep_config_single_safe - prepare a single-buffer DMA transfer
* with atomic slave configuration update
* @chan: DMA channel
* @buf: DMA buffer address
* @len: length of the transfer in bytes
* @dir: DMA transfer direction
* @flags: transfer preparation flags
* @config: DMA slave configuration for this transfer
*
* Detail see dmaengine_prep_config_sg_safe().
*/
static inline struct dma_async_tx_descriptor *
dmaengine_prep_config_single_safe(struct dma_chan *chan, dma_addr_t buf,
size_t len, enum dma_transfer_direction dir,
unsigned long flags,
struct dma_slave_config *config)
{
struct scatterlist sg;
sg_init_table(&sg, 1);
sg_dma_address(&sg) = buf;
sg_dma_len(&sg) = len;
return dmaengine_prep_config_sg_safe(chan, &sg, 1, dir, flags, config);
}
#ifdef CONFIG_RAPIDIO_DMA_ENGINE
struct rio_dma_ext;
static inline struct dma_async_tx_descriptor *dmaengine_prep_rio_sg(