+
+ if (uart_circ_chars_pending(xmit) < WAKEUP_CHARS)
+ uart_write_wakeup(port);
+
+ msm_handle_tx(port);
+done:
+ spin_unlock_irqrestore(&port->lock, flags);
+}
+
+static int msm_handle_tx_dma(struct msm_port *msm_port, unsigned int count)
+{
+ struct circ_buf *xmit = &msm_port->uart.state->xmit;
+ struct uart_port *port = &msm_port->uart;
+ struct msm_dma *dma = &msm_port->tx_dma;
+ void *cpu_addr;
+ int ret;
+ u32 val;
+
+ cpu_addr = &xmit->buf[xmit->tail];
+
+ dma->phys = dma_map_single(port->dev, cpu_addr, count, dma->dir);
+ ret = dma_mapping_error(port->dev, dma->phys);
+ if (ret)
+ return ret;
+
+ dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
+ count, DMA_MEM_TO_DEV,
+ DMA_PREP_INTERRUPT |
+ DMA_PREP_FENCE);
+ if (!dma->desc) {
+ ret = -EIO;
+ goto unmap;
+ }
+
+ dma->desc->callback = msm_complete_tx_dma;
+ dma->desc->callback_param = msm_port;
+
+ dma->cookie = dmaengine_submit(dma->desc);
+ ret = dma_submit_error(dma->cookie);
+ if (ret)
+ goto unmap;
+
+ /*
+ * Using DMA complete for Tx FIFO reload, no need for
+ * "Tx FIFO below watermark" one, disable it
+ */
+ msm_port->imr &= ~UART_IMR_TXLEV;
+ msm_write(port, msm_port->imr, UART_IMR);
+
+ dma->count = count;
+
+ val = msm_read(port, UARTDM_DMEN);
+ val |= dma->enable_bit;
+
+ if (msm_port->is_uartdm < UARTDM_1P4)
+ msm_write(port, val, UARTDM_DMEN);
+
+ msm_reset_dm_count(port, count);
+
+ if (msm_port->is_uartdm > UARTDM_1P3)
+ msm_write(port, val, UARTDM_DMEN);
+
+ dma_async_issue_pending(dma->chan);
+ return 0;
+unmap:
+ dma_unmap_single(port->dev, dma->phys, count, dma->dir);
+ return ret;
+}
+
+static void msm_complete_rx_dma(void *args)
+{
+ struct msm_port *msm_port = args;
+ struct uart_port *port = &msm_port->uart;
+ struct tty_port *tport = &port->state->port;
+ struct msm_dma *dma = &msm_port->rx_dma;
+ int count = 0, i, sysrq;
+ unsigned long flags;
+ u32 val;
+
+ spin_lock_irqsave(&port->lock, flags);
+
+ /* Already stopped */
+ if (!dma->count)
+ goto done;
+
+ val = msm_read(port, UARTDM_DMEN);
+ val &= ~dma->enable_bit;
+ msm_write(port, val, UARTDM_DMEN);
+
+ /* Restore interrupts */
+ msm_port->imr |= UART_IMR_RXLEV | UART_IMR_RXSTALE;
+ msm_write(port, msm_port->imr, UART_IMR);
+
+ if (msm_read(port, UART_SR) & UART_SR_OVERRUN) {
+ port->icount.overrun++;
+ tty_insert_flip_char(tport, 0, TTY_OVERRUN);
+ msm_write(port, UART_CR_CMD_RESET_ERR, UART_CR);
+ }
+
+ count = msm_read(port, UARTDM_RX_TOTAL_SNAP);
+
+ port->icount.rx += count;
+
+ dma->count = 0;
+
+ dma_unmap_single(port->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);
+
+ for (i = 0; i < count; i++) {
+ char flag = TTY_NORMAL;
+
+ if (msm_port->break_detected && dma->virt[i] == 0) {
+ port->icount.brk++;
+ flag = TTY_BREAK;
+ msm_port->break_detected = false;
+ if (uart_handle_break(port))
+ continue;
+ }
+
+ if (!(port->read_status_mask & UART_SR_RX_BREAK))
+ flag = TTY_NORMAL;
+
+ spin_unlock_irqrestore(&port->lock, flags);
+ sysrq = uart_handle_sysrq_char(port, dma->virt[i]);
+ spin_lock_irqsave(&port->lock, flags);
+ if (!sysrq)
+ tty_insert_flip_char(tport, dma->virt[i], flag);
+ }
+
+ msm_start_rx_dma(msm_port);
+done:
+ spin_unlock_irqrestore(&port->lock, flags);
+
+ if (count)
+ tty_flip_buffer_push(tport);
+}
+
+static void msm_start_rx_dma(struct msm_port *msm_port)
+{
+ struct msm_dma *dma = &msm_port->rx_dma;
+ struct uart_port *uart = &msm_port->uart;
+ u32 val;
+ int ret;
+
+ if (!dma->chan)
+ return;
+
+ dma->phys = dma_map_single(uart->dev, dma->virt,
+ UARTDM_RX_SIZE, dma->dir);
+ ret = dma_mapping_error(uart->dev, dma->phys);
+ if (ret)
+ return;
+
+ dma->desc = dmaengine_prep_slave_single(dma->chan, dma->phys,
+ UARTDM_RX_SIZE, DMA_DEV_TO_MEM,
+ DMA_PREP_INTERRUPT);
+ if (!dma->desc)
+ goto unmap;
+
+ dma->desc->callback = msm_complete_rx_dma;
+ dma->desc->callback_param = msm_port;
+
+ dma->cookie = dmaengine_submit(dma->desc);
+ ret = dma_submit_error(dma->cookie);
+ if (ret)
+ goto unmap;
+ /*
+ * Using DMA for FIFO off-load, no need for "Rx FIFO over
+ * watermark" or "stale" interrupts, disable them
+ */
+ msm_port->imr &= ~(UART_IMR_RXLEV | UART_IMR_RXSTALE);
+
+ /*
+ * Well, when DMA is ADM3 engine(implied by <= UARTDM v1.3),
+ * we need RXSTALE to flush input DMA fifo to memory
+ */
+ if (msm_port->is_uartdm < UARTDM_1P4)
+ msm_port->imr |= UART_IMR_RXSTALE;
+
+ msm_write(uart, msm_port->imr, UART_IMR);
+
+ dma->count = UARTDM_RX_SIZE;
+
+ dma_async_issue_pending(dma->chan);
+
+ msm_write(uart, UART_CR_CMD_RESET_STALE_INT, UART_CR);
+ msm_write(uart, UART_CR_CMD_STALE_EVENT_ENABLE, UART_CR);
+
+ val = msm_read(uart, UARTDM_DMEN);
+ val |= dma->enable_bit;
+
+ if (msm_port->is_uartdm < UARTDM_1P4)
+ msm_write(uart, val, UARTDM_DMEN);
+
+ msm_write(uart, UARTDM_RX_SIZE, UARTDM_DMRX);
+
+ if (msm_port->is_uartdm > UARTDM_1P3)
+ msm_write(uart, val, UARTDM_DMEN);
+
+ return;
+unmap:
+ dma_unmap_single(uart->dev, dma->phys, UARTDM_RX_SIZE, dma->dir);