+/* dummy param set used to (re)initialize parameter RAM slots */
+static const struct edmacc_param dummy_paramset = {
+ .link_bcntrld = 0xffff,
+ .ccnt = 1,
+};
+
+#define EDMA_BINDING_LEGACY 0
+#define EDMA_BINDING_TPCC 1
+static const struct of_device_id edma_of_ids[] = {
+ {
+ .compatible = "ti,edma3",
+ .data = (void *)EDMA_BINDING_LEGACY,
+ },
+ {
+ .compatible = "ti,edma3-tpcc",
+ .data = (void *)EDMA_BINDING_TPCC,
+ },
+ {}
+};
+
+static const struct of_device_id edma_tptc_of_ids[] = {
+ { .compatible = "ti,edma3-tptc", },
+ {}
+};
+
+static inline unsigned int edma_read(struct edma_cc *ecc, int offset)
+{
+ return (unsigned int)__raw_readl(ecc->base + offset);
+}
+
+static inline void edma_write(struct edma_cc *ecc, int offset, int val)
+{
+ __raw_writel(val, ecc->base + offset);
+}
+
+static inline void edma_modify(struct edma_cc *ecc, int offset, unsigned and,
+ unsigned or)
+{
+ unsigned val = edma_read(ecc, offset);
+
+ val &= and;
+ val |= or;
+ edma_write(ecc, offset, val);
+}
+
+static inline void edma_and(struct edma_cc *ecc, int offset, unsigned and)
+{
+ unsigned val = edma_read(ecc, offset);
+
+ val &= and;
+ edma_write(ecc, offset, val);
+}
+
+static inline void edma_or(struct edma_cc *ecc, int offset, unsigned or)
+{
+ unsigned val = edma_read(ecc, offset);
+
+ val |= or;
+ edma_write(ecc, offset, val);
+}
+
+static inline unsigned int edma_read_array(struct edma_cc *ecc, int offset,
+ int i)
+{
+ return edma_read(ecc, offset + (i << 2));
+}
+
+static inline void edma_write_array(struct edma_cc *ecc, int offset, int i,
+ unsigned val)
+{
+ edma_write(ecc, offset + (i << 2), val);
+}
+
+static inline void edma_modify_array(struct edma_cc *ecc, int offset, int i,
+ unsigned and, unsigned or)
+{
+ edma_modify(ecc, offset + (i << 2), and, or);
+}
+
+static inline void edma_or_array(struct edma_cc *ecc, int offset, int i,
+ unsigned or)
+{
+ edma_or(ecc, offset + (i << 2), or);
+}
+
+static inline void edma_or_array2(struct edma_cc *ecc, int offset, int i, int j,
+ unsigned or)
+{
+ edma_or(ecc, offset + ((i * 2 + j) << 2), or);
+}
+
+static inline void edma_write_array2(struct edma_cc *ecc, int offset, int i,
+ int j, unsigned val)
+{
+ edma_write(ecc, offset + ((i * 2 + j) << 2), val);
+}
+
+static inline unsigned int edma_shadow0_read(struct edma_cc *ecc, int offset)
+{
+ return edma_read(ecc, EDMA_SHADOW0 + offset);
+}
+
+static inline unsigned int edma_shadow0_read_array(struct edma_cc *ecc,
+ int offset, int i)
+{
+ return edma_read(ecc, EDMA_SHADOW0 + offset + (i << 2));
+}
+
+static inline void edma_shadow0_write(struct edma_cc *ecc, int offset,
+ unsigned val)
+{
+ edma_write(ecc, EDMA_SHADOW0 + offset, val);
+}
+
+static inline void edma_shadow0_write_array(struct edma_cc *ecc, int offset,
+ int i, unsigned val)
+{
+ edma_write(ecc, EDMA_SHADOW0 + offset + (i << 2), val);
+}
+
+static inline unsigned int edma_param_read(struct edma_cc *ecc, int offset,
+ int param_no)
+{
+ return edma_read(ecc, EDMA_PARM + offset + (param_no << 5));
+}
+
+static inline void edma_param_write(struct edma_cc *ecc, int offset,
+ int param_no, unsigned val)
+{
+ edma_write(ecc, EDMA_PARM + offset + (param_no << 5), val);
+}
+
+static inline void edma_param_modify(struct edma_cc *ecc, int offset,
+ int param_no, unsigned and, unsigned or)
+{
+ edma_modify(ecc, EDMA_PARM + offset + (param_no << 5), and, or);
+}
+
+static inline void edma_param_and(struct edma_cc *ecc, int offset, int param_no,
+ unsigned and)
+{
+ edma_and(ecc, EDMA_PARM + offset + (param_no << 5), and);
+}
+
+static inline void edma_param_or(struct edma_cc *ecc, int offset, int param_no,
+ unsigned or)
+{
+ edma_or(ecc, EDMA_PARM + offset + (param_no << 5), or);
+}
+
+static inline void set_bits(int offset, int len, unsigned long *p)
+{
+ for (; len > 0; len--)
+ set_bit(offset + (len - 1), p);
+}
+
+static inline void clear_bits(int offset, int len, unsigned long *p)
+{
+ for (; len > 0; len--)
+ clear_bit(offset + (len - 1), p);
+}
+
+static void edma_assign_priority_to_queue(struct edma_cc *ecc, int queue_no,
+ int priority)
+{
+ int bit = queue_no * 4;
+
+ edma_modify(ecc, EDMA_QUEPRI, ~(0x7 << bit), ((priority & 0x7) << bit));
+}
+
+static void edma_set_chmap(struct edma_chan *echan, int slot)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+ if (ecc->chmap_exist) {
+ slot = EDMA_CHAN_SLOT(slot);
+ edma_write_array(ecc, EDMA_DCHMAP, channel, (slot << 5));
+ }
+}
+
+static void edma_setup_interrupt(struct edma_chan *echan, bool enable)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+ if (enable) {
+ edma_shadow0_write_array(ecc, SH_ICR, channel >> 5,
+ BIT(channel & 0x1f));
+ edma_shadow0_write_array(ecc, SH_IESR, channel >> 5,
+ BIT(channel & 0x1f));
+ } else {
+ edma_shadow0_write_array(ecc, SH_IECR, channel >> 5,
+ BIT(channel & 0x1f));
+ }
+}
+
+/*
+ * paRAM slot management functions
+ */
+static void edma_write_slot(struct edma_cc *ecc, unsigned slot,
+ const struct edmacc_param *param)
+{
+ slot = EDMA_CHAN_SLOT(slot);
+ if (slot >= ecc->num_slots)
+ return;
+ memcpy_toio(ecc->base + PARM_OFFSET(slot), param, PARM_SIZE);
+}
+
+static void edma_read_slot(struct edma_cc *ecc, unsigned slot,
+ struct edmacc_param *param)
+{
+ slot = EDMA_CHAN_SLOT(slot);
+ if (slot >= ecc->num_slots)
+ return;
+ memcpy_fromio(param, ecc->base + PARM_OFFSET(slot), PARM_SIZE);
+}
+
+/**
+ * edma_alloc_slot - allocate DMA parameter RAM
+ * @ecc: pointer to edma_cc struct
+ * @slot: specific slot to allocate; negative for "any unused slot"
+ *
+ * This allocates a parameter RAM slot, initializing it to hold a
+ * dummy transfer. Slots allocated using this routine have not been
+ * mapped to a hardware DMA channel, and will normally be used by
+ * linking to them from a slot associated with a DMA channel.
+ *
+ * Normal use is to pass EDMA_SLOT_ANY as the @slot, but specific
+ * slots may be allocated on behalf of DSP firmware.
+ *
+ * Returns the number of the slot, else negative errno.
+ */
+static int edma_alloc_slot(struct edma_cc *ecc, int slot)
+{
+ if (slot > 0) {
+ slot = EDMA_CHAN_SLOT(slot);
+ /* Requesting entry paRAM slot for a HW triggered channel. */
+ if (ecc->chmap_exist && slot < ecc->num_channels)
+ slot = EDMA_SLOT_ANY;
+ }
+
+ if (slot < 0) {
+ if (ecc->chmap_exist)
+ slot = 0;
+ else
+ slot = ecc->num_channels;
+ for (;;) {
+ slot = find_next_zero_bit(ecc->slot_inuse,
+ ecc->num_slots,
+ slot);
+ if (slot == ecc->num_slots)
+ return -ENOMEM;
+ if (!test_and_set_bit(slot, ecc->slot_inuse))
+ break;
+ }
+ } else if (slot >= ecc->num_slots) {
+ return -EINVAL;
+ } else if (test_and_set_bit(slot, ecc->slot_inuse)) {
+ return -EBUSY;
+ }
+
+ edma_write_slot(ecc, slot, &dummy_paramset);
+
+ return EDMA_CTLR_CHAN(ecc->id, slot);
+}
+
+static void edma_free_slot(struct edma_cc *ecc, unsigned slot)
+{
+ slot = EDMA_CHAN_SLOT(slot);
+ if (slot >= ecc->num_slots)
+ return;
+
+ edma_write_slot(ecc, slot, &dummy_paramset);
+ clear_bit(slot, ecc->slot_inuse);
+}
+
+/**
+ * edma_link - link one parameter RAM slot to another
+ * @ecc: pointer to edma_cc struct
+ * @from: parameter RAM slot originating the link
+ * @to: parameter RAM slot which is the link target
+ *
+ * The originating slot should not be part of any active DMA transfer.
+ */
+static void edma_link(struct edma_cc *ecc, unsigned from, unsigned to)
+{
+ if (unlikely(EDMA_CTLR(from) != EDMA_CTLR(to)))
+ dev_warn(ecc->dev, "Ignoring eDMA instance for linking\n");
+
+ from = EDMA_CHAN_SLOT(from);
+ to = EDMA_CHAN_SLOT(to);
+ if (from >= ecc->num_slots || to >= ecc->num_slots)
+ return;
+
+ edma_param_modify(ecc, PARM_LINK_BCNTRLD, from, 0xffff0000,
+ PARM_OFFSET(to));
+}
+
+/**
+ * edma_get_position - returns the current transfer point
+ * @ecc: pointer to edma_cc struct
+ * @slot: parameter RAM slot being examined
+ * @dst: true selects the dest position, false the source
+ *
+ * Returns the position of the current active slot
+ */
+static dma_addr_t edma_get_position(struct edma_cc *ecc, unsigned slot,
+ bool dst)
+{
+ u32 offs;
+
+ slot = EDMA_CHAN_SLOT(slot);
+ offs = PARM_OFFSET(slot);
+ offs += dst ? PARM_DST : PARM_SRC;
+
+ return edma_read(ecc, offs);
+}
+
+/*
+ * Channels with event associations will be triggered by their hardware
+ * events, and channels without such associations will be triggered by
+ * software. (At this writing there is no interface for using software
+ * triggers except with channels that don't support hardware triggers.)
+ */
+static void edma_start(struct edma_chan *echan)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int j = (channel >> 5);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ if (!echan->hw_triggered) {
+ /* EDMA channels without event association */
+ dev_dbg(ecc->dev, "ESR%d %08x\n", j,
+ edma_shadow0_read_array(ecc, SH_ESR, j));
+ edma_shadow0_write_array(ecc, SH_ESR, j, mask);
+ } else {
+ /* EDMA channel with event association */
+ dev_dbg(ecc->dev, "ER%d %08x\n", j,
+ edma_shadow0_read_array(ecc, SH_ER, j));
+ /* Clear any pending event or error */
+ edma_write_array(ecc, EDMA_ECR, j, mask);
+ edma_write_array(ecc, EDMA_EMCR, j, mask);
+ /* Clear any SER */
+ edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+ edma_shadow0_write_array(ecc, SH_EESR, j, mask);
+ dev_dbg(ecc->dev, "EER%d %08x\n", j,
+ edma_shadow0_read_array(ecc, SH_EER, j));
+ }
+}
+
+static void edma_stop(struct edma_chan *echan)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int j = (channel >> 5);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ edma_shadow0_write_array(ecc, SH_EECR, j, mask);
+ edma_shadow0_write_array(ecc, SH_ECR, j, mask);
+ edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+ edma_write_array(ecc, EDMA_EMCR, j, mask);
+
+ /* clear possibly pending completion interrupt */
+ edma_shadow0_write_array(ecc, SH_ICR, j, mask);
+
+ dev_dbg(ecc->dev, "EER%d %08x\n", j,
+ edma_shadow0_read_array(ecc, SH_EER, j));
+
+ /* REVISIT: consider guarding against inappropriate event
+ * chaining by overwriting with dummy_paramset.
+ */
+}
+
+/*
+ * Temporarily disable EDMA hardware events on the specified channel,
+ * preventing them from triggering new transfers
+ */
+static void edma_pause(struct edma_chan *echan)
+{
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ edma_shadow0_write_array(echan->ecc, SH_EECR, channel >> 5, mask);
+}
+
+/* Re-enable EDMA hardware events on the specified channel. */
+static void edma_resume(struct edma_chan *echan)
+{
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ edma_shadow0_write_array(echan->ecc, SH_EESR, channel >> 5, mask);
+}
+
+static void edma_trigger_channel(struct edma_chan *echan)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ edma_shadow0_write_array(ecc, SH_ESR, (channel >> 5), mask);
+
+ dev_dbg(ecc->dev, "ESR%d %08x\n", (channel >> 5),
+ edma_shadow0_read_array(ecc, SH_ESR, (channel >> 5)));
+}
+
+static void edma_clean_channel(struct edma_chan *echan)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int j = (channel >> 5);
+ unsigned int mask = BIT(channel & 0x1f);
+
+ dev_dbg(ecc->dev, "EMR%d %08x\n", j, edma_read_array(ecc, EDMA_EMR, j));
+ edma_shadow0_write_array(ecc, SH_ECR, j, mask);
+ /* Clear the corresponding EMR bits */
+ edma_write_array(ecc, EDMA_EMCR, j, mask);
+ /* Clear any SER */
+ edma_shadow0_write_array(ecc, SH_SECR, j, mask);
+ edma_write(ecc, EDMA_CCERRCLR, BIT(16) | BIT(1) | BIT(0));
+}
+
+/* Move channel to a specific event queue */
+static void edma_assign_channel_eventq(struct edma_chan *echan,
+ enum dma_event_q eventq_no)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+ int bit = (channel & 0x7) * 4;
+
+ /* default to low priority queue */
+ if (eventq_no == EVENTQ_DEFAULT)
+ eventq_no = ecc->default_queue;
+ if (eventq_no >= ecc->num_tc)
+ return;
+
+ eventq_no &= 7;
+ edma_modify_array(ecc, EDMA_DMAQNUM, (channel >> 3), ~(0x7 << bit),
+ eventq_no << bit);
+}
+
+static int edma_alloc_channel(struct edma_chan *echan,
+ enum dma_event_q eventq_no)
+{
+ struct edma_cc *ecc = echan->ecc;
+ int channel = EDMA_CHAN_SLOT(echan->ch_num);
+
+ /* ensure access through shadow region 0 */
+ edma_or_array2(ecc, EDMA_DRAE, 0, channel >> 5, BIT(channel & 0x1f));
+
+ /* ensure no events are pending */
+ edma_stop(echan);
+
+ edma_setup_interrupt(echan, true);
+
+ edma_assign_channel_eventq(echan, eventq_no);
+
+ return 0;
+}
+
+static void edma_free_channel(struct edma_chan *echan)
+{
+ /* ensure no events are pending */
+ edma_stop(echan);
+ /* REVISIT should probably take out of shadow region 0 */
+ edma_setup_interrupt(echan, false);
+}
+