--- /dev/null
+/*
+ * Copyright (c) 2010 Broadcom Corporation
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <linux/slab.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+#include <net/cfg80211.h>
+#include <net/mac80211.h>
+
+#include <brcmu_utils.h>
+#include <aiutils.h>
+#include "types.h"
+#include "main.h"
+#include "dma.h"
+#include "soc.h"
+#include "scb.h"
+#include "ampdu.h"
+#include "debug.h"
+#include "brcms_trace_events.h"
+
+/*
+ * dma register field offset calculation
+ */
+#define DMA64REGOFFS(field) offsetof(struct dma64regs, field)
+#define DMA64TXREGOFFS(di, field) (di->d64txregbase + DMA64REGOFFS(field))
+#define DMA64RXREGOFFS(di, field) (di->d64rxregbase + DMA64REGOFFS(field))
+
+/*
+ * DMA hardware requires each descriptor ring to be 8kB aligned, and fit within
+ * a contiguous 8kB physical address.
+ */
+#define D64RINGALIGN_BITS 13
+#define D64MAXRINGSZ (1 << D64RINGALIGN_BITS)
+#define D64RINGALIGN (1 << D64RINGALIGN_BITS)
+
+#define D64MAXDD (D64MAXRINGSZ / sizeof(struct dma64desc))
+
+/* transmit channel control */
+#define D64_XC_XE 0x00000001 /* transmit enable */
+#define D64_XC_SE 0x00000002 /* transmit suspend request */
+#define D64_XC_LE 0x00000004 /* loopback enable */
+#define D64_XC_FL 0x00000010 /* flush request */
+#define D64_XC_PD 0x00000800 /* parity check disable */
+#define D64_XC_AE 0x00030000 /* address extension bits */
+#define D64_XC_AE_SHIFT 16
+
+/* transmit descriptor table pointer */
+#define D64_XP_LD_MASK 0x00000fff /* last valid descriptor */
+
+/* transmit channel status */
+#define D64_XS0_CD_MASK 0x00001fff /* current descriptor pointer */
+#define D64_XS0_XS_MASK 0xf0000000 /* transmit state */
+#define D64_XS0_XS_SHIFT 28
+#define D64_XS0_XS_DISABLED 0x00000000 /* disabled */
+#define D64_XS0_XS_ACTIVE 0x10000000 /* active */
+#define D64_XS0_XS_IDLE 0x20000000 /* idle wait */
+#define D64_XS0_XS_STOPPED 0x30000000 /* stopped */
+#define D64_XS0_XS_SUSP 0x40000000 /* suspend pending */
+
+#define D64_XS1_AD_MASK 0x00001fff /* active descriptor */
+#define D64_XS1_XE_MASK 0xf0000000 /* transmit errors */
+#define D64_XS1_XE_SHIFT 28
+#define D64_XS1_XE_NOERR 0x00000000 /* no error */
+#define D64_XS1_XE_DPE 0x10000000 /* descriptor protocol error */
+#define D64_XS1_XE_DFU 0x20000000 /* data fifo underrun */
+#define D64_XS1_XE_DTE 0x30000000 /* data transfer error */
+#define D64_XS1_XE_DESRE 0x40000000 /* descriptor read error */
+#define D64_XS1_XE_COREE 0x50000000 /* core error */
+
+/* receive channel control */
+/* receive enable */
+#define D64_RC_RE 0x00000001
+/* receive frame offset */
+#define D64_RC_RO_MASK 0x000000fe
+#define D64_RC_RO_SHIFT 1
+/* direct fifo receive (pio) mode */
+#define D64_RC_FM 0x00000100
+/* separate rx header descriptor enable */
+#define D64_RC_SH 0x00000200
+/* overflow continue */
+#define D64_RC_OC 0x00000400
+/* parity check disable */
+#define D64_RC_PD 0x00000800
+/* address extension bits */
+#define D64_RC_AE 0x00030000
+#define D64_RC_AE_SHIFT 16
+
+/* flags for dma controller */
+/* partity enable */
+#define DMA_CTRL_PEN (1 << 0)
+/* rx overflow continue */
+#define DMA_CTRL_ROC (1 << 1)
+/* allow rx scatter to multiple descriptors */
+#define DMA_CTRL_RXMULTI (1 << 2)
+/* Unframed Rx/Tx data */
+#define DMA_CTRL_UNFRAMED (1 << 3)
+
+/* receive descriptor table pointer */
+#define D64_RP_LD_MASK 0x00000fff /* last valid descriptor */
+
+/* receive channel status */
+#define D64_RS0_CD_MASK 0x00001fff /* current descriptor pointer */
+#define D64_RS0_RS_MASK 0xf0000000 /* receive state */
+#define D64_RS0_RS_SHIFT 28
+#define D64_RS0_RS_DISABLED 0x00000000 /* disabled */
+#define D64_RS0_RS_ACTIVE 0x10000000 /* active */
+#define D64_RS0_RS_IDLE 0x20000000 /* idle wait */
+#define D64_RS0_RS_STOPPED 0x30000000 /* stopped */
+#define D64_RS0_RS_SUSP 0x40000000 /* suspend pending */
+
+#define D64_RS1_AD_MASK 0x0001ffff /* active descriptor */
+#define D64_RS1_RE_MASK 0xf0000000 /* receive errors */
+#define D64_RS1_RE_SHIFT 28
+#define D64_RS1_RE_NOERR 0x00000000 /* no error */
+#define D64_RS1_RE_DPO 0x10000000 /* descriptor protocol error */
+#define D64_RS1_RE_DFU 0x20000000 /* data fifo overflow */
+#define D64_RS1_RE_DTE 0x30000000 /* data transfer error */
+#define D64_RS1_RE_DESRE 0x40000000 /* descriptor read error */
+#define D64_RS1_RE_COREE 0x50000000 /* core error */
+
+/* fifoaddr */
+#define D64_FA_OFF_MASK 0xffff /* offset */
+#define D64_FA_SEL_MASK 0xf0000 /* select */
+#define D64_FA_SEL_SHIFT 16
+#define D64_FA_SEL_XDD 0x00000 /* transmit dma data */
+#define D64_FA_SEL_XDP 0x10000 /* transmit dma pointers */
+#define D64_FA_SEL_RDD 0x40000 /* receive dma data */
+#define D64_FA_SEL_RDP 0x50000 /* receive dma pointers */
+#define D64_FA_SEL_XFD 0x80000 /* transmit fifo data */
+#define D64_FA_SEL_XFP 0x90000 /* transmit fifo pointers */
+#define D64_FA_SEL_RFD 0xc0000 /* receive fifo data */
+#define D64_FA_SEL_RFP 0xd0000 /* receive fifo pointers */
+#define D64_FA_SEL_RSD 0xe0000 /* receive frame status data */
+#define D64_FA_SEL_RSP 0xf0000 /* receive frame status pointers */
+
+/* descriptor control flags 1 */
+#define D64_CTRL_COREFLAGS 0x0ff00000 /* core specific flags */
+#define D64_CTRL1_EOT ((u32)1 << 28) /* end of descriptor table */
+#define D64_CTRL1_IOC ((u32)1 << 29) /* interrupt on completion */
+#define D64_CTRL1_EOF ((u32)1 << 30) /* end of frame */
+#define D64_CTRL1_SOF ((u32)1 << 31) /* start of frame */
+
+/* descriptor control flags 2 */
+/* buffer byte count. real data len must <= 16KB */
+#define D64_CTRL2_BC_MASK 0x00007fff
+/* address extension bits */
+#define D64_CTRL2_AE 0x00030000
+#define D64_CTRL2_AE_SHIFT 16
+/* parity bit */
+#define D64_CTRL2_PARITY 0x00040000
+
+/* control flags in the range [27:20] are core-specific and not defined here */
+#define D64_CTRL_CORE_MASK 0x0ff00000
+
+#define D64_RX_FRM_STS_LEN 0x0000ffff /* frame length mask */
+#define D64_RX_FRM_STS_OVFL 0x00800000 /* RxOverFlow */
+#define D64_RX_FRM_STS_DSCRCNT 0x0f000000 /* no. of descriptors used - 1 */
+#define D64_RX_FRM_STS_DATATYPE 0xf0000000 /* core-dependent data type */
+
+/*
+ * packet headroom necessary to accommodate the largest header
+ * in the system, (i.e TXOFF). By doing, we avoid the need to
+ * allocate an extra buffer for the header when bridging to WL.
+ * There is a compile time check in wlc.c which ensure that this
+ * value is at least as big as TXOFF. This value is used in
+ * dma_rxfill().
+ */
+
+#define BCMEXTRAHDROOM 172
+
+#define MAXNAMEL 8 /* 8 char names */
+
+/* macros to convert between byte offsets and indexes */
+#define B2I(bytes, type) ((bytes) / sizeof(type))
+#define I2B(index, type) ((index) * sizeof(type))
+
+#define PCI32ADDR_HIGH 0xc0000000 /* address[31:30] */
+#define PCI32ADDR_HIGH_SHIFT 30 /* address[31:30] */
+
+#define PCI64ADDR_HIGH 0x80000000 /* address[63] */
+#define PCI64ADDR_HIGH_SHIFT 31 /* address[63] */
+
+/*
+ * DMA Descriptor
+ * Descriptors are only read by the hardware, never written back.
+ */
+struct dma64desc {
+ __le32 ctrl1; /* misc control bits & bufcount */
+ __le32 ctrl2; /* buffer count and address extension */
+ __le32 addrlow; /* memory address of the date buffer, bits 31:0 */
+ __le32 addrhigh; /* memory address of the date buffer, bits 63:32 */
+};
+
+/* dma engine software state */
+struct dma_info {
+ struct dma_pub dma; /* exported structure */
+ char name[MAXNAMEL]; /* callers name for diag msgs */
+
+ struct bcma_device *core;
+ struct device *dmadev;
+
+ /* session information for AMPDU */
+ struct brcms_ampdu_session ampdu_session;
+
+ bool dma64; /* this dma engine is operating in 64-bit mode */
+ bool addrext; /* this dma engine supports DmaExtendedAddrChanges */
+
+ /* 64-bit dma tx engine registers */
+ uint d64txregbase;
+ /* 64-bit dma rx engine registers */
+ uint d64rxregbase;
+ /* pointer to dma64 tx descriptor ring */
+ struct dma64desc *txd64;
+ /* pointer to dma64 rx descriptor ring */
+ struct dma64desc *rxd64;
+
+ u16 dmadesc_align; /* alignment requirement for dma descriptors */
+
+ u16 ntxd; /* # tx descriptors tunable */
+ u16 txin; /* index of next descriptor to reclaim */
+ u16 txout; /* index of next descriptor to post */
+ /* pointer to parallel array of pointers to packets */
+ struct sk_buff **txp;
+ /* Aligned physical address of descriptor ring */
+ dma_addr_t txdpa;
+ /* Original physical address of descriptor ring */
+ dma_addr_t txdpaorig;
+ u16 txdalign; /* #bytes added to alloc'd mem to align txd */
+ u32 txdalloc; /* #bytes allocated for the ring */
+ u32 xmtptrbase; /* When using unaligned descriptors, the ptr register
+ * is not just an index, it needs all 13 bits to be
+ * an offset from the addr register.
+ */
+
+ u16 nrxd; /* # rx descriptors tunable */
+ u16 rxin; /* index of next descriptor to reclaim */
+ u16 rxout; /* index of next descriptor to post */
+ /* pointer to parallel array of pointers to packets */
+ struct sk_buff **rxp;
+ /* Aligned physical address of descriptor ring */
+ dma_addr_t rxdpa;
+ /* Original physical address of descriptor ring */
+ dma_addr_t rxdpaorig;
+ u16 rxdalign; /* #bytes added to alloc'd mem to align rxd */
+ u32 rxdalloc; /* #bytes allocated for the ring */
+ u32 rcvptrbase; /* Base for ptr reg when using unaligned descriptors */
+
+ /* tunables */
+ unsigned int rxbufsize; /* rx buffer size in bytes, not including
+ * the extra headroom
+ */
+ uint rxextrahdrroom; /* extra rx headroom, reverseved to assist upper
+ * stack, e.g. some rx pkt buffers will be
+ * bridged to tx side without byte copying.
+ * The extra headroom needs to be large enough
+ * to fit txheader needs. Some dongle driver may
+ * not need it.
+ */
+ uint nrxpost; /* # rx buffers to keep posted */
+ unsigned int rxoffset; /* rxcontrol offset */
+ /* add to get dma address of descriptor ring, low 32 bits */
+ uint ddoffsetlow;
+ /* high 32 bits */
+ uint ddoffsethigh;
+ /* add to get dma address of data buffer, low 32 bits */
+ uint dataoffsetlow;
+ /* high 32 bits */
+ uint dataoffsethigh;
+ /* descriptor base need to be aligned or not */
+ bool aligndesc_4k;
+};
+
+/* Check for odd number of 1's */
+static u32 parity32(__le32 data)
+{
+ /* no swap needed for counting 1's */
+ u32 par_data = *(u32 *)&data;
+
+ par_data ^= par_data >> 16;
+ par_data ^= par_data >> 8;
+ par_data ^= par_data >> 4;
+ par_data ^= par_data >> 2;
+ par_data ^= par_data >> 1;
+
+ return par_data & 1;
+}
+
+static bool dma64_dd_parity(struct dma64desc *dd)
+{
+ return parity32(dd->addrlow ^ dd->addrhigh ^ dd->ctrl1 ^ dd->ctrl2);
+}
+
+/* descriptor bumping functions */
+
+static uint xxd(uint x, uint n)
+{
+ return x & (n - 1); /* faster than %, but n must be power of 2 */
+}
+
+static uint txd(struct dma_info *di, uint x)
+{
+ return xxd(x, di->ntxd);
+}
+
+static uint rxd(struct dma_info *di, uint x)
+{
+ return xxd(x, di->nrxd);
+}
+
+static uint nexttxd(struct dma_info *di, uint i)
+{
+ return txd(di, i + 1);
+}
+
+static uint prevtxd(struct dma_info *di, uint i)
+{
+ return txd(di, i - 1);
+}
+
+static uint nextrxd(struct dma_info *di, uint i)
+{
+ return rxd(di, i + 1);
+}
+
+static uint ntxdactive(struct dma_info *di, uint h, uint t)
+{
+ return txd(di, t-h);
+}
+
+static uint nrxdactive(struct dma_info *di, uint h, uint t)
+{
+ return rxd(di, t-h);
+}
+
+static uint _dma_ctrlflags(struct dma_info *di, uint mask, uint flags)
+{
+ uint dmactrlflags;
+
+ if (di == NULL)
+ return 0;
+
+ dmactrlflags = di->dma.dmactrlflags;
+ dmactrlflags &= ~mask;
+ dmactrlflags |= flags;
+
+ /* If trying to enable parity, check if parity is actually supported */
+ if (dmactrlflags & DMA_CTRL_PEN) {
+ u32 control;
+
+ control = bcma_read32(di->core, DMA64TXREGOFFS(di, control));
+ bcma_write32(di->core, DMA64TXREGOFFS(di, control),
+ control | D64_XC_PD);
+ if (bcma_read32(di->core, DMA64TXREGOFFS(di, control)) &
+ D64_XC_PD)
+ /* We *can* disable it so it is supported,
+ * restore control register
+ */
+ bcma_write32(di->core, DMA64TXREGOFFS(di, control),
+ control);
+ else
+ /* Not supported, don't allow it to be enabled */
+ dmactrlflags &= ~DMA_CTRL_PEN;
+ }
+
+ di->dma.dmactrlflags = dmactrlflags;
+
+ return dmactrlflags;
+}
+
+static bool _dma64_addrext(struct dma_info *di, uint ctrl_offset)
+{
+ u32 w;
+ bcma_set32(di->core, ctrl_offset, D64_XC_AE);
+ w = bcma_read32(di->core, ctrl_offset);
+ bcma_mask32(di->core, ctrl_offset, ~D64_XC_AE);
+ return (w & D64_XC_AE) == D64_XC_AE;
+}
+
+/*
+ * return true if this dma engine supports DmaExtendedAddrChanges,
+ * otherwise false
+ */
+static bool _dma_isaddrext(struct dma_info *di)
+{
+ /* DMA64 supports full 32- or 64-bit operation. AE is always valid */
+
+ /* not all tx or rx channel are available */
+ if (di->d64txregbase != 0) {
+ if (!_dma64_addrext(di, DMA64TXREGOFFS(di, control)))
+ brcms_dbg_dma(di->core,
+ "%s: DMA64 tx doesn't have AE set\n",
+ di->name);
+ return true;
+ } else if (di->d64rxregbase != 0) {
+ if (!_dma64_addrext(di, DMA64RXREGOFFS(di, control)))
+ brcms_dbg_dma(di->core,
+ "%s: DMA64 rx doesn't have AE set\n",
+ di->name);
+ return true;
+ }
+
+ return false;
+}
+
+static bool _dma_descriptor_align(struct dma_info *di)
+{
+ u32 addrl;
+
+ /* Check to see if the descriptors need to be aligned on 4K/8K or not */
+ if (di->d64txregbase != 0) {
+ bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow), 0xff0);
+ addrl = bcma_read32(di->core, DMA64TXREGOFFS(di, addrlow));
+ if (addrl != 0)
+ return false;
+ } else if (di->d64rxregbase != 0) {
+ bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow), 0xff0);
+ addrl = bcma_read32(di->core, DMA64RXREGOFFS(di, addrlow));
+ if (addrl != 0)
+ return false;
+ }
+ return true;
+}
+
+/*
+ * Descriptor table must start at the DMA hardware dictated alignment, so
+ * allocated memory must be large enough to support this requirement.
+ */
+static void *dma_alloc_consistent(struct dma_info *di, uint size,
+ u16 align_bits, uint *alloced,
+ dma_addr_t *pap)
+{
+ if (align_bits) {
+ u16 align = (1 << align_bits);
+ if (!IS_ALIGNED(PAGE_SIZE, align))
+ size += align;
+ *alloced = size;
+ }
+ return dma_alloc_coherent(di->dmadev, size, pap, GFP_ATOMIC);
+}
+
+static
+u8 dma_align_sizetobits(uint size)
+{
+ u8 bitpos = 0;
+ while (size >>= 1)
+ bitpos++;
+ return bitpos;
+}
+
+/* This function ensures that the DMA descriptor ring will not get allocated
+ * across Page boundary. If the allocation is done across the page boundary
+ * at the first time, then it is freed and the allocation is done at
+ * descriptor ring size aligned location. This will ensure that the ring will
+ * not cross page boundary
+ */
+static void *dma_ringalloc(struct dma_info *di, u32 boundary, uint size,
+ u16 *alignbits, uint *alloced,
+ dma_addr_t *descpa)
+{
+ void *va;
+ u32 desc_strtaddr;
+ u32 alignbytes = 1 << *alignbits;
+
+ va = dma_alloc_consistent(di, size, *alignbits, alloced, descpa);
+
+ if (NULL == va)
+ return NULL;
+
+ desc_strtaddr = (u32) roundup((unsigned long)va, alignbytes);
+ if (((desc_strtaddr + size - 1) & boundary) != (desc_strtaddr
+ & boundary)) {
+ *alignbits = dma_align_sizetobits(size);
+ dma_free_coherent(di->dmadev, size, va, *descpa);
+ va = dma_alloc_consistent(di, size, *alignbits,
+ alloced, descpa);
+ }
+ return va;
+}
+
+static bool dma64_alloc(struct dma_info *di, uint direction)
+{
+ u16 size;
+ uint ddlen;
+ void *va;
+ uint alloced = 0;
+ u16 align;
+ u16 align_bits;
+
+ ddlen = sizeof(struct dma64desc);
+
+ size = (direction == DMA_TX) ? (di->ntxd * ddlen) : (di->nrxd * ddlen);
+ align_bits = di->dmadesc_align;
+ align = (1 << align_bits);
+
+ if (direction == DMA_TX) {
+ va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
+ &alloced, &di->txdpaorig);
+ if (va == NULL) {
+ brcms_dbg_dma(di->core,
+ "%s: DMA_ALLOC_CONSISTENT(ntxd) failed\n",
+ di->name);
+ return false;
+ }
+ align = (1 << align_bits);
+ di->txd64 = (struct dma64desc *)
+ roundup((unsigned long)va, align);
+ di->txdalign = (uint) ((s8 *)di->txd64 - (s8 *) va);
+ di->txdpa = di->txdpaorig + di->txdalign;
+ di->txdalloc = alloced;
+ } else {
+ va = dma_ringalloc(di, D64RINGALIGN, size, &align_bits,
+ &alloced, &di->rxdpaorig);
+ if (va == NULL) {
+ brcms_dbg_dma(di->core,
+ "%s: DMA_ALLOC_CONSISTENT(nrxd) failed\n",
+ di->name);
+ return false;
+ }
+ align = (1 << align_bits);
+ di->rxd64 = (struct dma64desc *)
+ roundup((unsigned long)va, align);
+ di->rxdalign = (uint) ((s8 *)di->rxd64 - (s8 *) va);
+ di->rxdpa = di->rxdpaorig + di->rxdalign;
+ di->rxdalloc = alloced;
+ }
+
+ return true;
+}
+
+static bool _dma_alloc(struct dma_info *di, uint direction)
+{
+ return dma64_alloc(di, direction);
+}
+
+struct dma_pub *dma_attach(char *name, struct brcms_c_info *wlc,
+ uint txregbase, uint rxregbase, uint ntxd, uint nrxd,
+ uint rxbufsize, int rxextheadroom,
+ uint nrxpost, uint rxoffset)
+{
+ struct si_pub *sih = wlc->hw->sih;
+ struct bcma_device *core = wlc->hw->d11core;
+ struct dma_info *di;
+ u8 rev = core->id.rev;
+ uint size;
+ struct si_info *sii = container_of(sih, struct si_info, pub);
+
+ /* allocate private info structure */
+ di = kzalloc(sizeof(struct dma_info), GFP_ATOMIC);
+ if (di == NULL)
+ return NULL;
+
+ di->dma64 =
+ ((bcma_aread32(core, BCMA_IOST) & SISF_DMA64) == SISF_DMA64);
+
+ /* init dma reg info */
+ di->core = core;
+ di->d64txregbase = txregbase;
+ di->d64rxregbase = rxregbase;
+
+ /*
+ * Default flags (which can be changed by the driver calling
+ * dma_ctrlflags before enable): For backwards compatibility
+ * both Rx Overflow Continue and Parity are DISABLED.
+ */
+ _dma_ctrlflags(di, DMA_CTRL_ROC | DMA_CTRL_PEN, 0);
+
+ brcms_dbg_dma(di->core, "%s: %s flags 0x%x ntxd %d nrxd %d "
+ "rxbufsize %d rxextheadroom %d nrxpost %d rxoffset %d "
+ "txregbase %u rxregbase %u\n", name, "DMA64",
+ di->dma.dmactrlflags, ntxd, nrxd, rxbufsize,
+ rxextheadroom, nrxpost, rxoffset, txregbase, rxregbase);
+
+ /* make a private copy of our callers name */
+ strncpy(di->name, name, MAXNAMEL);
+ di->name[MAXNAMEL - 1] = '\0';
+
+ di->dmadev = core->dma_dev;
+
+ /* save tunables */
+ di->ntxd = (u16) ntxd;
+ di->nrxd = (u16) nrxd;
+
+ /* the actual dma size doesn't include the extra headroom */
+ di->rxextrahdrroom =
+ (rxextheadroom == -1) ? BCMEXTRAHDROOM : rxextheadroom;
+ if (rxbufsize > BCMEXTRAHDROOM)
+ di->rxbufsize = (u16) (rxbufsize - di->rxextrahdrroom);
+ else
+ di->rxbufsize = (u16) rxbufsize;
+
+ di->nrxpost = (u16) nrxpost;
+ di->rxoffset = (u8) rxoffset;
+
+ /*
+ * figure out the DMA physical address offset for dd and data
+ * PCI/PCIE: they map silicon backplace address to zero
+ * based memory, need offset
+ * Other bus: use zero SI_BUS BIGENDIAN kludge: use sdram
+ * swapped region for data buffer, not descriptor
+ */
+ di->ddoffsetlow = 0;
+ di->dataoffsetlow = 0;
+ /* for pci bus, add offset */
+ if (sii->icbus->hosttype == BCMA_HOSTTYPE_PCI) {
+ /* add offset for pcie with DMA64 bus */
+ di->ddoffsetlow = 0;
+ di->ddoffsethigh = SI_PCIE_DMA_H32;
+ }
+ di->dataoffsetlow = di->ddoffsetlow;
+ di->dataoffsethigh = di->ddoffsethigh;
+
+ /* WAR64450 : DMACtl.Addr ext fields are not supported in SDIOD core. */
+ if ((core->id.id == BCMA_CORE_SDIO_DEV)
+ && ((rev > 0) && (rev <= 2)))
+ di->addrext = false;
+ else if ((core->id.id == BCMA_CORE_I2S) &&
+ ((rev == 0) || (rev == 1)))
+ di->addrext = false;
+ else
+ di->addrext = _dma_isaddrext(di);
+
+ /* does the descriptor need to be aligned and if yes, on 4K/8K or not */
+ di->aligndesc_4k = _dma_descriptor_align(di);
+ if (di->aligndesc_4k) {
+ di->dmadesc_align = D64RINGALIGN_BITS;
+ if ((ntxd < D64MAXDD / 2) && (nrxd < D64MAXDD / 2))
+ /* for smaller dd table, HW relax alignment reqmnt */
+ di->dmadesc_align = D64RINGALIGN_BITS - 1;
+ } else {
+ di->dmadesc_align = 4; /* 16 byte alignment */
+ }
+
+ brcms_dbg_dma(di->core, "DMA descriptor align_needed %d, align %d\n",
+ di->aligndesc_4k, di->dmadesc_align);
+
+ /* allocate tx packet pointer vector */
+ if (ntxd) {
+ size = ntxd * sizeof(void *);
+ di->txp = kzalloc(size, GFP_ATOMIC);
+ if (di->txp == NULL)
+ goto fail;
+ }
+
+ /* allocate rx packet pointer vector */
+ if (nrxd) {
+ size = nrxd * sizeof(void *);
+ di->rxp = kzalloc(size, GFP_ATOMIC);
+ if (di->rxp == NULL)
+ goto fail;
+ }
+
+ /*
+ * allocate transmit descriptor ring, only need ntxd descriptors
+ * but it must be aligned
+ */
+ if (ntxd) {
+ if (!_dma_alloc(di, DMA_TX))
+ goto fail;
+ }
+
+ /*
+ * allocate receive descriptor ring, only need nrxd descriptors
+ * but it must be aligned
+ */
+ if (nrxd) {
+ if (!_dma_alloc(di, DMA_RX))
+ goto fail;
+ }
+
+ if ((di->ddoffsetlow != 0) && !di->addrext) {
+ if (di->txdpa > SI_PCI_DMA_SZ) {
+ brcms_dbg_dma(di->core,
+ "%s: txdpa 0x%x: addrext not supported\n",
+ di->name, (u32)di->txdpa);
+ goto fail;
+ }
+ if (di->rxdpa > SI_PCI_DMA_SZ) {
+ brcms_dbg_dma(di->core,
+ "%s: rxdpa 0x%x: addrext not supported\n",
+ di->name, (u32)di->rxdpa);
+ goto fail;
+ }
+ }
+
+ /* Initialize AMPDU session */
+ brcms_c_ampdu_reset_session(&di->ampdu_session, wlc);
+
+ brcms_dbg_dma(di->core,
+ "ddoffsetlow 0x%x ddoffsethigh 0x%x dataoffsetlow 0x%x dataoffsethigh 0x%x addrext %d\n",
+ di->ddoffsetlow, di->ddoffsethigh,
+ di->dataoffsetlow, di->dataoffsethigh,
+ di->addrext);
+
+ return (struct dma_pub *) di;
+
+ fail:
+ dma_detach((struct dma_pub *)di);
+ return NULL;
+}
+
+static inline void
+dma64_dd_upd(struct dma_info *di, struct dma64desc *ddring,
+ dma_addr_t pa, uint outidx, u32 *flags, u32 bufcount)
+{
+ u32 ctrl2 = bufcount & D64_CTRL2_BC_MASK;
+
+ /* PCI bus with big(>1G) physical address, use address extension */
+ if ((di->dataoffsetlow == 0) || !(pa & PCI32ADDR_HIGH)) {
+ ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
+ ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
+ ddring[outidx].ctrl1 = cpu_to_le32(*flags);
+ ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
+ } else {
+ /* address extension for 32-bit PCI */
+ u32 ae;
+
+ ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
+ pa &= ~PCI32ADDR_HIGH;
+
+ ctrl2 |= (ae << D64_CTRL2_AE_SHIFT) & D64_CTRL2_AE;
+ ddring[outidx].addrlow = cpu_to_le32(pa + di->dataoffsetlow);
+ ddring[outidx].addrhigh = cpu_to_le32(di->dataoffsethigh);
+ ddring[outidx].ctrl1 = cpu_to_le32(*flags);
+ ddring[outidx].ctrl2 = cpu_to_le32(ctrl2);
+ }
+ if (di->dma.dmactrlflags & DMA_CTRL_PEN) {
+ if (dma64_dd_parity(&ddring[outidx]))
+ ddring[outidx].ctrl2 =
+ cpu_to_le32(ctrl2 | D64_CTRL2_PARITY);
+ }
+}
+
+/* !! may be called with core in reset */
+void dma_detach(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ /* free dma descriptor rings */
+ if (di->txd64)
+ dma_free_coherent(di->dmadev, di->txdalloc,
+ ((s8 *)di->txd64 - di->txdalign),
+ (di->txdpaorig));
+ if (di->rxd64)
+ dma_free_coherent(di->dmadev, di->rxdalloc,
+ ((s8 *)di->rxd64 - di->rxdalign),
+ (di->rxdpaorig));
+
+ /* free packet pointer vectors */
+ kfree(di->txp);
+ kfree(di->rxp);
+
+ /* free our private info structure */
+ kfree(di);
+
+}
+
+/* initialize descriptor table base address */
+static void
+_dma_ddtable_init(struct dma_info *di, uint direction, dma_addr_t pa)
+{
+ if (!di->aligndesc_4k) {
+ if (direction == DMA_TX)
+ di->xmtptrbase = pa;
+ else
+ di->rcvptrbase = pa;
+ }
+
+ if ((di->ddoffsetlow == 0)
+ || !(pa & PCI32ADDR_HIGH)) {
+ if (direction == DMA_TX) {
+ bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
+ pa + di->ddoffsetlow);
+ bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
+ di->ddoffsethigh);
+ } else {
+ bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
+ pa + di->ddoffsetlow);
+ bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
+ di->ddoffsethigh);
+ }
+ } else {
+ /* DMA64 32bits address extension */
+ u32 ae;
+
+ /* shift the high bit(s) from pa to ae */
+ ae = (pa & PCI32ADDR_HIGH) >> PCI32ADDR_HIGH_SHIFT;
+ pa &= ~PCI32ADDR_HIGH;
+
+ if (direction == DMA_TX) {
+ bcma_write32(di->core, DMA64TXREGOFFS(di, addrlow),
+ pa + di->ddoffsetlow);
+ bcma_write32(di->core, DMA64TXREGOFFS(di, addrhigh),
+ di->ddoffsethigh);
+ bcma_maskset32(di->core, DMA64TXREGOFFS(di, control),
+ D64_XC_AE, (ae << D64_XC_AE_SHIFT));
+ } else {
+ bcma_write32(di->core, DMA64RXREGOFFS(di, addrlow),
+ pa + di->ddoffsetlow);
+ bcma_write32(di->core, DMA64RXREGOFFS(di, addrhigh),
+ di->ddoffsethigh);
+ bcma_maskset32(di->core, DMA64RXREGOFFS(di, control),
+ D64_RC_AE, (ae << D64_RC_AE_SHIFT));
+ }
+ }
+}
+
+static void _dma_rxenable(struct dma_info *di)
+{
+ uint dmactrlflags = di->dma.dmactrlflags;
+ u32 control;
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ control = D64_RC_RE | (bcma_read32(di->core,
+ DMA64RXREGOFFS(di, control)) &
+ D64_RC_AE);
+
+ if ((dmactrlflags & DMA_CTRL_PEN) == 0)
+ control |= D64_RC_PD;
+
+ if (dmactrlflags & DMA_CTRL_ROC)
+ control |= D64_RC_OC;
+
+ bcma_write32(di->core, DMA64RXREGOFFS(di, control),
+ ((di->rxoffset << D64_RC_RO_SHIFT) | control));
+}
+
+void dma_rxinit(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ if (di->nrxd == 0)
+ return;
+
+ di->rxin = di->rxout = 0;
+
+ /* clear rx descriptor ring */
+ memset(di->rxd64, '\0', di->nrxd * sizeof(struct dma64desc));
+
+ /* DMA engine with out alignment requirement requires table to be inited
+ * before enabling the engine
+ */
+ if (!di->aligndesc_4k)
+ _dma_ddtable_init(di, DMA_RX, di->rxdpa);
+
+ _dma_rxenable(di);
+
+ if (di->aligndesc_4k)
+ _dma_ddtable_init(di, DMA_RX, di->rxdpa);
+}
+
+static struct sk_buff *dma64_getnextrxp(struct dma_info *di, bool forceall)
+{
+ uint i, curr;
+ struct sk_buff *rxp;
+ dma_addr_t pa;
+
+ i = di->rxin;
+
+ /* return if no packets posted */
+ if (i == di->rxout)
+ return NULL;
+
+ curr =
+ B2I(((bcma_read32(di->core,
+ DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) -
+ di->rcvptrbase) & D64_RS0_CD_MASK, struct dma64desc);
+
+ /* ignore curr if forceall */
+ if (!forceall && (i == curr))
+ return NULL;
+
+ /* get the packet pointer that corresponds to the rx descriptor */
+ rxp = di->rxp[i];
+ di->rxp[i] = NULL;
+
+ pa = le32_to_cpu(di->rxd64[i].addrlow) - di->dataoffsetlow;
+
+ /* clear this packet from the descriptor ring */
+ dma_unmap_single(di->dmadev, pa, di->rxbufsize, DMA_FROM_DEVICE);
+
+ di->rxd64[i].addrlow = cpu_to_le32(0xdeadbeef);
+ di->rxd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
+
+ di->rxin = nextrxd(di, i);
+
+ return rxp;
+}
+
+static struct sk_buff *_dma_getnextrxp(struct dma_info *di, bool forceall)
+{
+ if (di->nrxd == 0)
+ return NULL;
+
+ return dma64_getnextrxp(di, forceall);
+}
+
+/*
+ * !! rx entry routine
+ * returns the number packages in the next frame, or 0 if there are no more
+ * if DMA_CTRL_RXMULTI is defined, DMA scattering(multiple buffers) is
+ * supported with pkts chain
+ * otherwise, it's treated as giant pkt and will be tossed.
+ * The DMA scattering starts with normal DMA header, followed by first
+ * buffer data. After it reaches the max size of buffer, the data continues
+ * in next DMA descriptor buffer WITHOUT DMA header
+ */
+int dma_rx(struct dma_pub *pub, struct sk_buff_head *skb_list)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct sk_buff_head dma_frames;
+ struct sk_buff *p, *next;
+ uint len;
+ uint pkt_len;
+ int resid = 0;
+ int pktcnt = 1;
+
+ skb_queue_head_init(&dma_frames);
+ next_frame:
+ p = _dma_getnextrxp(di, false);
+ if (p == NULL)
+ return 0;
+
+ len = le16_to_cpu(*(__le16 *) (p->data));
+ brcms_dbg_dma(di->core, "%s: dma_rx len %d\n", di->name, len);
+ dma_spin_for_len(len, p);
+
+ /* set actual length */
+ pkt_len = min((di->rxoffset + len), di->rxbufsize);
+ __skb_trim(p, pkt_len);
+ skb_queue_tail(&dma_frames, p);
+ resid = len - (di->rxbufsize - di->rxoffset);
+
+ /* check for single or multi-buffer rx */
+ if (resid > 0) {
+ while ((resid > 0) && (p = _dma_getnextrxp(di, false))) {
+ pkt_len = min_t(uint, resid, di->rxbufsize);
+ __skb_trim(p, pkt_len);
+ skb_queue_tail(&dma_frames, p);
+ resid -= di->rxbufsize;
+ pktcnt++;
+ }
+
+#ifdef DEBUG
+ if (resid > 0) {
+ uint cur;
+ cur =
+ B2I(((bcma_read32(di->core,
+ DMA64RXREGOFFS(di, status0)) &
+ D64_RS0_CD_MASK) - di->rcvptrbase) &
+ D64_RS0_CD_MASK, struct dma64desc);
+ brcms_dbg_dma(di->core,
+ "rxin %d rxout %d, hw_curr %d\n",
+ di->rxin, di->rxout, cur);
+ }
+#endif /* DEBUG */
+
+ if ((di->dma.dmactrlflags & DMA_CTRL_RXMULTI) == 0) {
+ brcms_dbg_dma(di->core, "%s: bad frame length (%d)\n",
+ di->name, len);
+ skb_queue_walk_safe(&dma_frames, p, next) {
+ skb_unlink(p, &dma_frames);
+ brcmu_pkt_buf_free_skb(p);
+ }
+ di->dma.rxgiants++;
+ pktcnt = 1;
+ goto next_frame;
+ }
+ }
+
+ skb_queue_splice_tail(&dma_frames, skb_list);
+ return pktcnt;
+}
+
+static bool dma64_rxidle(struct dma_info *di)
+{
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ if (di->nrxd == 0)
+ return true;
+
+ return ((bcma_read32(di->core,
+ DMA64RXREGOFFS(di, status0)) & D64_RS0_CD_MASK) ==
+ (bcma_read32(di->core, DMA64RXREGOFFS(di, ptr)) &
+ D64_RS0_CD_MASK));
+}
+
+static bool dma64_txidle(struct dma_info *di)
+{
+ if (di->ntxd == 0)
+ return true;
+
+ return ((bcma_read32(di->core,
+ DMA64TXREGOFFS(di, status0)) & D64_XS0_CD_MASK) ==
+ (bcma_read32(di->core, DMA64TXREGOFFS(di, ptr)) &
+ D64_XS0_CD_MASK));
+}
+
+/*
+ * post receive buffers
+ * Return false if refill failed completely or dma mapping failed. The ring
+ * is empty, which will stall the rx dma and user might want to call rxfill
+ * again asap. This is unlikely to happen on a memory-rich NIC, but often on
+ * memory-constrained dongle.
+ */
+bool dma_rxfill(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct sk_buff *p;
+ u16 rxin, rxout;
+ u32 flags = 0;
+ uint n;
+ uint i;
+ dma_addr_t pa;
+ uint extra_offset = 0;
+ bool ring_empty;
+
+ ring_empty = false;
+
+ /*
+ * Determine how many receive buffers we're lacking
+ * from the full complement, allocate, initialize,
+ * and post them, then update the chip rx lastdscr.
+ */
+
+ rxin = di->rxin;
+ rxout = di->rxout;
+
+ n = di->nrxpost - nrxdactive(di, rxin, rxout);
+
+ brcms_dbg_dma(di->core, "%s: post %d\n", di->name, n);
+
+ if (di->rxbufsize > BCMEXTRAHDROOM)
+ extra_offset = di->rxextrahdrroom;
+
+ for (i = 0; i < n; i++) {
+ /*
+ * the di->rxbufsize doesn't include the extra headroom,
+ * we need to add it to the size to be allocated
+ */
+ p = brcmu_pkt_buf_get_skb(di->rxbufsize + extra_offset);
+
+ if (p == NULL) {
+ brcms_dbg_dma(di->core, "%s: out of rxbufs\n",
+ di->name);
+ if (i == 0 && dma64_rxidle(di)) {
+ brcms_dbg_dma(di->core, "%s: ring is empty !\n",
+ di->name);
+ ring_empty = true;
+ }
+ di->dma.rxnobuf++;
+ break;
+ }
+ /* reserve an extra headroom, if applicable */
+ if (extra_offset)
+ skb_pull(p, extra_offset);
+
+ /* Do a cached write instead of uncached write since DMA_MAP
+ * will flush the cache.
+ */
+ *(u32 *) (p->data) = 0;
+
+ pa = dma_map_single(di->dmadev, p->data, di->rxbufsize,
+ DMA_FROM_DEVICE);
+ if (dma_mapping_error(di->dmadev, pa))
+ return false;
+
+ /* save the free packet pointer */
+ di->rxp[rxout] = p;
+
+ /* reset flags for each descriptor */
+ flags = 0;
+ if (rxout == (di->nrxd - 1))
+ flags = D64_CTRL1_EOT;
+
+ dma64_dd_upd(di, di->rxd64, pa, rxout, &flags,
+ di->rxbufsize);
+ rxout = nextrxd(di, rxout);
+ }
+
+ di->rxout = rxout;
+
+ /* update the chip lastdscr pointer */
+ bcma_write32(di->core, DMA64RXREGOFFS(di, ptr),
+ di->rcvptrbase + I2B(rxout, struct dma64desc));
+
+ return ring_empty;
+}
+
+void dma_rxreclaim(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct sk_buff *p;
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ while ((p = _dma_getnextrxp(di, true)))
+ brcmu_pkt_buf_free_skb(p);
+}
+
+void dma_counterreset(struct dma_pub *pub)
+{
+ /* reset all software counters */
+ pub->rxgiants = 0;
+ pub->rxnobuf = 0;
+ pub->txnobuf = 0;
+}
+
+/* get the address of the var in order to change later */
+unsigned long dma_getvar(struct dma_pub *pub, const char *name)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ if (!strcmp(name, "&txavail"))
+ return (unsigned long)&(di->dma.txavail);
+ return 0;
+}
+
+/* 64-bit DMA functions */
+
+void dma_txinit(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ u32 control = D64_XC_XE;
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ if (di->ntxd == 0)
+ return;
+
+ di->txin = di->txout = 0;
+ di->dma.txavail = di->ntxd - 1;
+
+ /* clear tx descriptor ring */
+ memset(di->txd64, '\0', (di->ntxd * sizeof(struct dma64desc)));
+
+ /* DMA engine with out alignment requirement requires table to be inited
+ * before enabling the engine
+ */
+ if (!di->aligndesc_4k)
+ _dma_ddtable_init(di, DMA_TX, di->txdpa);
+
+ if ((di->dma.dmactrlflags & DMA_CTRL_PEN) == 0)
+ control |= D64_XC_PD;
+ bcma_set32(di->core, DMA64TXREGOFFS(di, control), control);
+
+ /* DMA engine with alignment requirement requires table to be inited
+ * before enabling the engine
+ */
+ if (di->aligndesc_4k)
+ _dma_ddtable_init(di, DMA_TX, di->txdpa);
+}
+
+void dma_txsuspend(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ if (di->ntxd == 0)
+ return;
+
+ bcma_set32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
+}
+
+void dma_txresume(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ brcms_dbg_dma(di->core, "%s:\n", di->name);
+
+ if (di->ntxd == 0)
+ return;
+
+ bcma_mask32(di->core, DMA64TXREGOFFS(di, control), ~D64_XC_SE);
+}
+
+bool dma_txsuspended(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+
+ return (di->ntxd == 0) ||
+ ((bcma_read32(di->core,
+ DMA64TXREGOFFS(di, control)) & D64_XC_SE) ==
+ D64_XC_SE);
+}
+
+void dma_txreclaim(struct dma_pub *pub, enum txd_range range)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct sk_buff *p;
+
+ brcms_dbg_dma(di->core, "%s: %s\n",
+ di->name,
+ range == DMA_RANGE_ALL ? "all" :
+ range == DMA_RANGE_TRANSMITTED ? "transmitted" :
+ "transferred");
+
+ if (di->txin == di->txout)
+ return;
+
+ while ((p = dma_getnexttxp(pub, range))) {
+ /* For unframed data, we don't have any packets to free */
+ if (!(di->dma.dmactrlflags & DMA_CTRL_UNFRAMED))
+ brcmu_pkt_buf_free_skb(p);
+ }
+}
+
+bool dma_txreset(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ u32 status;
+
+ if (di->ntxd == 0)
+ return true;
+
+ /* suspend tx DMA first */
+ bcma_write32(di->core, DMA64TXREGOFFS(di, control), D64_XC_SE);
+ SPINWAIT(((status =
+ (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
+ D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED) &&
+ (status != D64_XS0_XS_IDLE) && (status != D64_XS0_XS_STOPPED),
+ 10000);
+
+ bcma_write32(di->core, DMA64TXREGOFFS(di, control), 0);
+ SPINWAIT(((status =
+ (bcma_read32(di->core, DMA64TXREGOFFS(di, status0)) &
+ D64_XS0_XS_MASK)) != D64_XS0_XS_DISABLED), 10000);
+
+ /* wait for the last transaction to complete */
+ udelay(300);
+
+ return status == D64_XS0_XS_DISABLED;
+}
+
+bool dma_rxreset(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ u32 status;
+
+ if (di->nrxd == 0)
+ return true;
+
+ bcma_write32(di->core, DMA64RXREGOFFS(di, control), 0);
+ SPINWAIT(((status =
+ (bcma_read32(di->core, DMA64RXREGOFFS(di, status0)) &
+ D64_RS0_RS_MASK)) != D64_RS0_RS_DISABLED), 10000);
+
+ return status == D64_RS0_RS_DISABLED;
+}
+
+static void dma_txenq(struct dma_info *di, struct sk_buff *p)
+{
+ unsigned char *data;
+ uint len;
+ u16 txout;
+ u32 flags = 0;
+ dma_addr_t pa;
+
+ txout = di->txout;
+
+ if (WARN_ON(nexttxd(di, txout) == di->txin))
+ return;
+
+ /*
+ * obtain and initialize transmit descriptor entry.
+ */
+ data = p->data;
+ len = p->len;
+
+ /* get physical address of buffer start */
+ pa = dma_map_single(di->dmadev, data, len, DMA_TO_DEVICE);
+ /* if mapping failed, free skb */
+ if (dma_mapping_error(di->dmadev, pa)) {
+ brcmu_pkt_buf_free_skb(p);
+ return;
+ }
+ /* With a DMA segment list, Descriptor table is filled
+ * using the segment list instead of looping over
+ * buffers in multi-chain DMA. Therefore, EOF for SGLIST
+ * is when end of segment list is reached.
+ */
+ flags = D64_CTRL1_SOF | D64_CTRL1_IOC | D64_CTRL1_EOF;
+ if (txout == (di->ntxd - 1))
+ flags |= D64_CTRL1_EOT;
+
+ dma64_dd_upd(di, di->txd64, pa, txout, &flags, len);
+
+ txout = nexttxd(di, txout);
+
+ /* save the packet */
+ di->txp[prevtxd(di, txout)] = p;
+
+ /* bump the tx descriptor index */
+ di->txout = txout;
+}
+
+static void ampdu_finalize(struct dma_info *di)
+{
+ struct brcms_ampdu_session *session = &di->ampdu_session;
+ struct sk_buff *p;
+
+ trace_brcms_ampdu_session(&session->wlc->hw->d11core->dev,
+ session->max_ampdu_len,
+ session->max_ampdu_frames,
+ session->ampdu_len,
+ skb_queue_len(&session->skb_list),
+ session->dma_len);
+
+ if (WARN_ON(skb_queue_empty(&session->skb_list)))
+ return;
+
+ brcms_c_ampdu_finalize(session);
+
+ while (!skb_queue_empty(&session->skb_list)) {
+ p = skb_dequeue(&session->skb_list);
+ dma_txenq(di, p);
+ }
+
+ bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
+ di->xmtptrbase + I2B(di->txout, struct dma64desc));
+ brcms_c_ampdu_reset_session(session, session->wlc);
+}
+
+static void prep_ampdu_frame(struct dma_info *di, struct sk_buff *p)
+{
+ struct brcms_ampdu_session *session = &di->ampdu_session;
+ int ret;
+
+ ret = brcms_c_ampdu_add_frame(session, p);
+ if (ret == -ENOSPC) {
+ /*
+ * AMPDU cannot accomodate this frame. Close out the in-
+ * progress AMPDU session and start a new one.
+ */
+ ampdu_finalize(di);
+ ret = brcms_c_ampdu_add_frame(session, p);
+ }
+
+ WARN_ON(ret);
+}
+
+/* Update count of available tx descriptors based on current DMA state */
+static void dma_update_txavail(struct dma_info *di)
+{
+ /*
+ * Available space is number of descriptors less the number of
+ * active descriptors and the number of queued AMPDU frames.
+ */
+ di->dma.txavail = di->ntxd - ntxdactive(di, di->txin, di->txout) -
+ skb_queue_len(&di->ampdu_session.skb_list) - 1;
+}
+
+/*
+ * !! tx entry routine
+ * WARNING: call must check the return value for error.
+ * the error(toss frames) could be fatal and cause many subsequent hard
+ * to debug problems
+ */
+int dma_txfast(struct brcms_c_info *wlc, struct dma_pub *pub,
+ struct sk_buff *p)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct brcms_ampdu_session *session = &di->ampdu_session;
+ struct ieee80211_tx_info *tx_info;
+ bool is_ampdu;
+
+ /* no use to transmit a zero length packet */
+ if (p->len == 0)
+ return 0;
+
+ /* return nonzero if out of tx descriptors */
+ if (di->dma.txavail == 0 || nexttxd(di, di->txout) == di->txin)
+ goto outoftxd;
+
+ tx_info = IEEE80211_SKB_CB(p);
+ is_ampdu = tx_info->flags & IEEE80211_TX_CTL_AMPDU;
+ if (is_ampdu)
+ prep_ampdu_frame(di, p);
+ else
+ dma_txenq(di, p);
+
+ /* tx flow control */
+ dma_update_txavail(di);
+
+ /* kick the chip */
+ if (is_ampdu) {
+ /*
+ * Start sending data if we've got a full AMPDU, there's
+ * no more space in the DMA ring, or the ring isn't
+ * currently transmitting.
+ */
+ if (skb_queue_len(&session->skb_list) == session->max_ampdu_frames ||
+ di->dma.txavail == 0 || dma64_txidle(di))
+ ampdu_finalize(di);
+ } else {
+ bcma_write32(di->core, DMA64TXREGOFFS(di, ptr),
+ di->xmtptrbase + I2B(di->txout, struct dma64desc));
+ }
+
+ return 0;
+
+ outoftxd:
+ brcms_dbg_dma(di->core, "%s: out of txds !!!\n", di->name);
+ brcmu_pkt_buf_free_skb(p);
+ di->dma.txavail = 0;
+ di->dma.txnobuf++;
+ return -ENOSPC;
+}
+
+void dma_txflush(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct brcms_ampdu_session *session = &di->ampdu_session;
+
+ if (!skb_queue_empty(&session->skb_list))
+ ampdu_finalize(di);
+}
+
+int dma_txpending(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ return ntxdactive(di, di->txin, di->txout);
+}
+
+/*
+ * If we have an active AMPDU session and are not transmitting,
+ * this function will force tx to start.
+ */
+void dma_kick_tx(struct dma_pub *pub)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ struct brcms_ampdu_session *session = &di->ampdu_session;
+
+ if (!skb_queue_empty(&session->skb_list) && dma64_txidle(di))
+ ampdu_finalize(di);
+}
+
+/*
+ * Reclaim next completed txd (txds if using chained buffers) in the range
+ * specified and return associated packet.
+ * If range is DMA_RANGE_TRANSMITTED, reclaim descriptors that have be
+ * transmitted as noted by the hardware "CurrDescr" pointer.
+ * If range is DMA_RANGE_TRANSFERED, reclaim descriptors that have be
+ * transferred by the DMA as noted by the hardware "ActiveDescr" pointer.
+ * If range is DMA_RANGE_ALL, reclaim all txd(s) posted to the ring and
+ * return associated packet regardless of the value of hardware pointers.
+ */
+struct sk_buff *dma_getnexttxp(struct dma_pub *pub, enum txd_range range)
+{
+ struct dma_info *di = container_of(pub, struct dma_info, dma);
+ u16 start, end, i;
+ u16 active_desc;
+ struct sk_buff *txp;
+
+ brcms_dbg_dma(di->core, "%s: %s\n",
+ di->name,
+ range == DMA_RANGE_ALL ? "all" :
+ range == DMA_RANGE_TRANSMITTED ? "transmitted" :
+ "transferred");
+
+ if (di->ntxd == 0)
+ return NULL;
+
+ txp = NULL;
+
+ start = di->txin;
+ if (range == DMA_RANGE_ALL)
+ end = di->txout;
+ else {
+ end = (u16) (B2I(((bcma_read32(di->core,
+ DMA64TXREGOFFS(di, status0)) &
+ D64_XS0_CD_MASK) - di->xmtptrbase) &
+ D64_XS0_CD_MASK, struct dma64desc));
+
+ if (range == DMA_RANGE_TRANSFERED) {
+ active_desc =
+ (u16)(bcma_read32(di->core,
+ DMA64TXREGOFFS(di, status1)) &
+ D64_XS1_AD_MASK);
+ active_desc =
+ (active_desc - di->xmtptrbase) & D64_XS0_CD_MASK;
+ active_desc = B2I(active_desc, struct dma64desc);
+ if (end != active_desc)
+ end = prevtxd(di, active_desc);
+ }
+ }
+
+ if ((start == 0) && (end > di->txout))
+ goto bogus;
+
+ for (i = start; i != end && !txp; i = nexttxd(di, i)) {
+ dma_addr_t pa;
+ uint size;
+
+ pa = le32_to_cpu(di->txd64[i].addrlow) - di->dataoffsetlow;
+
+ size =
+ (le32_to_cpu(di->txd64[i].ctrl2) &
+ D64_CTRL2_BC_MASK);
+
+ di->txd64[i].addrlow = cpu_to_le32(0xdeadbeef);
+ di->txd64[i].addrhigh = cpu_to_le32(0xdeadbeef);
+
+ txp = di->txp[i];
+ di->txp[i] = NULL;
+
+ dma_unmap_single(di->dmadev, pa, size, DMA_TO_DEVICE);
+ }
+
+ di->txin = i;
+
+ /* tx flow control */
+ dma_update_txavail(di);
+
+ return txp;
+
+ bogus:
+ brcms_dbg_dma(di->core, "bogus curr: start %d end %d txout %d\n",
+ start, end, di->txout);
+ return NULL;
+}
+
+/*
+ * Mac80211 initiated actions sometimes require packets in the DMA queue to be
+ * modified. The modified portion of the packet is not under control of the DMA
+ * engine. This function calls a caller-supplied function for each packet in
+ * the caller specified dma chain.
+ */
+void dma_walk_packets(struct dma_pub *dmah, void (*callback_fnc)
+ (void *pkt, void *arg_a), void *arg_a)
+{
+ struct dma_info *di = container_of(dmah, struct dma_info, dma);
+ uint i = di->txin;
+ uint end = di->txout;
+ struct sk_buff *skb;
+ struct ieee80211_tx_info *tx_info;
+
+ while (i != end) {
+ skb = di->txp[i];
+ if (skb != NULL) {
+ tx_info = (struct ieee80211_tx_info *)skb->cb;
+ (callback_fnc)(tx_info, arg_a);
+ }
+ i = nexttxd(di, i);
+ }
+}
Code Review / kvmfornfv.git / blobdiff