diff --git a/drivers/dma/dw-edma/dw-edma-core.h b/drivers/dma/dw-edma/dw-edma-core.h index ff20965db4b2..bd3d532ebc89 100644 --- a/drivers/dma/dw-edma/dw-edma-core.h +++ b/drivers/dma/dw-edma/dw-edma-core.h @@ -127,6 +127,8 @@ typedef void (*dw_edma_handler_t)(struct dw_edma_chan *); struct dw_edma_core_ops { void (*off)(struct dw_edma *dw); + int (*quiesce)(struct dw_edma *dw); + int (*ch_quiesce)(struct dw_edma_chan *chan); u16 (*ch_count)(struct dw_edma *dw, enum dw_edma_dir dir); enum dma_status (*ch_status)(struct dw_edma_chan *chan); irqreturn_t (*handle_int)(struct dw_edma_irq *dw_irq, enum dw_edma_dir dir, @@ -192,6 +194,18 @@ void dw_edma_core_off(struct dw_edma *dw) dw->core->off(dw); } +static inline +int dw_edma_core_quiesce(struct dw_edma *dw) +{ + return dw->core->quiesce(dw); +} + +static inline +int dw_edma_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return chan->dw->core->ch_quiesce(chan); +} + static inline u16 dw_edma_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { diff --git a/drivers/dma/dw-edma/dw-edma-v0-core.c b/drivers/dma/dw-edma/dw-edma-v0-core.c index 66a605f28a12..13e69ce6c41d 100644 --- a/drivers/dma/dw-edma/dw-edma-v0-core.c +++ b/drivers/dma/dw-edma/dw-edma-v0-core.c @@ -7,6 +7,7 @@ */ #include +#include #include #include @@ -160,6 +161,87 @@ static inline u32 readl_ch(struct dw_edma *dw, enum dw_edma_dir dir, u16 ch, readl_ch(dw, dir, ch, &(__dw_ch_regs(dw, dir, ch)->name)) /* eDMA management callbacks */ +static void dw_edma_v0_core_ch_power(struct dw_edma *dw, + enum dw_edma_dir dir, u16 id, bool enable) +{ + u32 value = enable ? BIT(0) : 0; + + if (WARN_ON_ONCE(id >= EDMA_V0_MAX_NR_CH)) + return; + + switch (id) { + case 0: + SET_RW_COMPAT(dw, dir, ch0_pwr_en, value); + break; + case 1: + SET_RW_COMPAT(dw, dir, ch1_pwr_en, value); + break; + case 2: + SET_RW_COMPAT(dw, dir, ch2_pwr_en, value); + break; + case 3: + SET_RW_COMPAT(dw, dir, ch3_pwr_en, value); + break; + case 4: + SET_RW_COMPAT(dw, dir, ch4_pwr_en, value); + break; + case 5: + SET_RW_COMPAT(dw, dir, ch5_pwr_en, value); + break; + case 6: + SET_RW_COMPAT(dw, dir, ch6_pwr_en, value); + break; + case 7: + SET_RW_COMPAT(dw, dir, ch7_pwr_en, value); + break; + } +} + +static int dw_edma_v0_core_engine_disable(struct dw_edma *dw, + enum dw_edma_dir dir) +{ + u32 value; + int ret; + + SET_RW_32(dw, dir, engine_en, 0); + ret = read_poll_timeout(GET_RW_32, value, !(value & BIT(0)), 100, + 200000, false, dw, dir, engine_en); + if (ret) + dev_warn(dw->chip->dev, "%s engine did not stop within 200ms\n", + dir == EDMA_DIR_WRITE ? "write" : "read"); + + return ret; +} + +static int dw_edma_v0_core_dir_off(struct dw_edma *dw, enum dw_edma_dir dir) +{ + u16 count, id; + int ret = 0; + + scoped_guard(raw_spinlock_irqsave, &dw->lock) + SET_RW_32(dw, dir, int_mask, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { + /* + * DWC PCIe Controller Databook 6.10a-lca06, "Legacy DMA + * and HDMA Software Compatibility": HDMA compatibility mode + * does not implement ENGINE_EN, but retains CHi_PWR_EN for + * per-channel enable and disable. + */ + count = dir == EDMA_DIR_WRITE ? dw->wr_ch_cnt : dw->rd_ch_cnt; + for (id = 0; id < count; id++) + dw_edma_v0_core_ch_power(dw, dir, id, false); + } else { + ret = dw_edma_v0_core_engine_disable(dw, dir); + } + + SET_RW_32(dw, dir, int_clear, + EDMA_V0_DONE_INT_MASK | EDMA_V0_ABORT_INT_MASK); + + return ret; +} + static void dw_edma_v0_core_off(struct dw_edma *dw) { SET_BOTH_32(dw, int_mask, @@ -169,6 +251,33 @@ static void dw_edma_v0_core_off(struct dw_edma *dw) SET_BOTH_32(dw, engine_en, 0); } +static int dw_edma_v0_core_quiesce(struct dw_edma *dw) +{ + int ret = 0; + int err; + + if (dw->wr_ch_cnt) + ret = dw_edma_v0_core_dir_off(dw, EDMA_DIR_WRITE); + if (dw->rd_ch_cnt) { + err = dw_edma_v0_core_dir_off(dw, EDMA_DIR_READ); + if (!ret) + ret = err; + } + + return ret; +} + +/* + * The unrolled eDMA and HDMA compatibility register maps share interrupt + * control per direction, so the whole direction is quiesced. Callers must + * own the direction entirely and prevent the peer from programming it after + * this point. Partial ownership mode validates direction granularity. + */ +static int dw_edma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + return dw_edma_v0_core_dir_off(chan->dw, chan->dir); +} + static u16 dw_edma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { u32 num_ch; @@ -331,34 +440,8 @@ static void dw_edma_v0_core_ch_enable(struct dw_edma_chan *chan) /* Enable engine */ SET_RW_32(dw, chan->dir, engine_en, BIT(0)); - if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) { - switch (chan->id) { - case 0: - SET_RW_COMPAT(dw, chan->dir, ch0_pwr_en, BIT(0)); - break; - case 1: - SET_RW_COMPAT(dw, chan->dir, ch1_pwr_en, BIT(0)); - break; - case 2: - SET_RW_COMPAT(dw, chan->dir, ch2_pwr_en, BIT(0)); - break; - case 3: - SET_RW_COMPAT(dw, chan->dir, ch3_pwr_en, BIT(0)); - break; - case 4: - SET_RW_COMPAT(dw, chan->dir, ch4_pwr_en, BIT(0)); - break; - case 5: - SET_RW_COMPAT(dw, chan->dir, ch5_pwr_en, BIT(0)); - break; - case 6: - SET_RW_COMPAT(dw, chan->dir, ch6_pwr_en, BIT(0)); - break; - case 7: - SET_RW_COMPAT(dw, chan->dir, ch7_pwr_en, BIT(0)); - break; - } - } + if (dw->chip->mf == EDMA_MF_HDMA_COMPAT) + dw_edma_v0_core_ch_power(dw, chan->dir, chan->id, true); /* Interrupt mask/unmask - done, abort */ raw_spin_lock_irqsave(&dw->lock, flags); @@ -552,6 +635,8 @@ static resource_size_t dw_edma_v0_core_db_offset(struct dw_edma *dw) static const struct dw_edma_core_ops dw_edma_v0_core = { .off = dw_edma_v0_core_off, + .quiesce = dw_edma_v0_core_quiesce, + .ch_quiesce = dw_edma_v0_core_ch_quiesce, .ch_count = dw_edma_v0_core_ch_count, .ch_status = dw_edma_v0_core_ch_status, .handle_int = dw_edma_v0_core_handle_int, diff --git a/drivers/dma/dw-edma/dw-hdma-v0-core.c b/drivers/dma/dw-edma/dw-hdma-v0-core.c index e9b4bc66886a..8d20bb028b52 100644 --- a/drivers/dma/dw-edma/dw-hdma-v0-core.c +++ b/drivers/dma/dw-edma/dw-hdma-v0-core.c @@ -72,6 +72,16 @@ static u32 dw_hdma_v0_core_int_setup(struct dw_edma_chan *chan, u32 val) } /* HDMA management callbacks */ +static void dw_hdma_v0_core_ch_off(struct dw_edma *dw, enum dw_edma_dir dir, + u16 id) +{ + SET_CH_32(dw, dir, id, int_setup, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); + SET_CH_32(dw, dir, id, ch_en, 0); + SET_CH_32(dw, dir, id, int_clear, + HDMA_V0_STOP_INT_MASK | HDMA_V0_ABORT_INT_MASK); +} + static void dw_hdma_v0_core_off(struct dw_edma *dw) { int id; @@ -96,6 +106,25 @@ static void dw_hdma_v0_core_off(struct dw_edma *dw) } } +static int dw_hdma_v0_core_quiesce(struct dw_edma *dw) +{ + int id; + + for (id = 0; id < dw->wr_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_WRITE, id); + for (id = 0; id < dw->rd_ch_cnt; id++) + dw_hdma_v0_core_ch_off(dw, EDMA_DIR_READ, id); + + return 0; +} + +static int dw_hdma_v0_core_ch_quiesce(struct dw_edma_chan *chan) +{ + dw_hdma_v0_core_ch_off(chan->dw, chan->dir, chan->id); + + return 0; +} + static u16 dw_hdma_v0_core_ch_count(struct dw_edma *dw, enum dw_edma_dir dir) { /* @@ -366,6 +395,8 @@ static resource_size_t dw_hdma_v0_core_db_offset(struct dw_edma *dw) static const struct dw_edma_core_ops dw_hdma_v0_core = { .off = dw_hdma_v0_core_off, + .quiesce = dw_hdma_v0_core_quiesce, + .ch_quiesce = dw_hdma_v0_core_ch_quiesce, .ch_count = dw_hdma_v0_core_ch_count, .ch_status = dw_hdma_v0_core_ch_status, .handle_int = dw_hdma_v0_core_handle_int,