Re: [PATCH v2] soc: fsl: dpio: Add support for QBMan ring bulk enqueue.
From: Roy Pledge
Date: Fri Sep 06 2019 - 15:00:19 EST
On 9/5/2019 3:01 PM, Youri Querry wrote:
> The QBMan frame descriptor enqueuing is changed from array
> mode (a single frame enqueue at a time) to bulk ring mode.
>
> This new mode allows the enqueuing of multiple frames in one operation.
> The original interface is kept but use the bulk enqueue of one frame
>
> Signed-off-by: Youri Querry <youri.querry_1@xxxxxxx>
Acked-by: Roy Pledge <roy.pledge@xxxxxxx>
> ---
> drivers/soc/fsl/dpio/dpio-service.c | 69 +++-
> drivers/soc/fsl/dpio/qbman-portal.c | 772 ++++++++++++++++++++++++++++++++----
> drivers/soc/fsl/dpio/qbman-portal.h | 175 +++++++-
> 3 files changed, 935 insertions(+), 81 deletions(-)
>
> diff --git a/drivers/soc/fsl/dpio/dpio-service.c b/drivers/soc/fsl/dpio/dpio-service.c
> index b9539ef..4eb53ee 100644
> --- a/drivers/soc/fsl/dpio/dpio-service.c
> +++ b/drivers/soc/fsl/dpio/dpio-service.c
> @@ -1,7 +1,7 @@
> // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
> /*
> * Copyright 2014-2016 Freescale Semiconductor Inc.
> - * Copyright 2016 NXP
> + * Copyright 2016-2019 NXP
> *
> */
> #include <linux/types.h>
> @@ -435,6 +435,69 @@ int dpaa2_io_service_enqueue_fq(struct dpaa2_io *d,
> EXPORT_SYMBOL(dpaa2_io_service_enqueue_fq);
>
> /**
> + * dpaa2_io_service_enqueue_multiple_fq() - Enqueue multiple frames
> + * to a frame queue using one fqid.
> + * @d: the given DPIO service.
> + * @fqid: the given frame queue id.
> + * @fd: the list of frame descriptors enqueued.
> + * @nb: number of frames to be enqueued
> + *
> + * Return the number of enqueued frames (0 if EQCR is busy)
> + * or -ENODEV if there is no dpio service.
> + */
> +int dpaa2_io_service_enqueue_multiple_fq(struct dpaa2_io *d,
> + u32 fqid,
> + const struct dpaa2_fd *fd,
> + int nb)
> +{
> + struct qbman_eq_desc ed;
> +
> + d = service_select(d);
> + if (!d)
> + return -ENODEV;
> +
> + qbman_eq_desc_clear(&ed);
> + qbman_eq_desc_set_no_orp(&ed, 0);
> + qbman_eq_desc_set_fq(&ed, fqid);
> +
> + return qbman_swp_enqueue_multiple(d->swp, &ed, fd, 0, nb);
> +}
> +EXPORT_SYMBOL(dpaa2_io_service_enqueue_multiple_fq);
> +
> +/**
> + * dpaa2_io_service_enqueue_multiple_desc_fq() - Enqueue multiple frames
> + * to different frame queue using a list of fqids.
> + * @d: the given DPIO service.
> + * @fqid: the given list of frame queue ids.
> + * @fd: the list of frame descriptors enqueued.
> + * @nb: number of frames to be enqueued
> + *
> + * Return the number of enqueued frames (0 if EQCR is busy)
> + * or -ENODEV if there is no dpio service.
> + */
> +int dpaa2_io_service_enqueue_multiple_desc_fq(struct dpaa2_io *d,
> + u32 *fqid,
> + const struct dpaa2_fd *fd,
> + int nb)
> +{
> + int i;
> + struct qbman_eq_desc_min ed[32];
> +
> + d = service_select(d);
> + if (!d)
> + return -ENODEV;
> +
> + for (i = 0; i < nb; i++) {
> + qbman_eq_desc_min_clear(&ed[i]);
> + qbman_eq_desc_set_no_orp_min(&ed[i], 0);
> + qbman_eq_desc_set_min_fq(&ed[i], fqid[i]);
> + }
> +
> + return qbman_swp_enqueue_multiple_desc(d->swp, &ed[0], fd, nb);
> +}
> +EXPORT_SYMBOL(dpaa2_io_service_enqueue_multiple_desc_fq);
> +
> +/**
> * dpaa2_io_service_enqueue_qd() - Enqueue a frame to a QD.
> * @d: the given DPIO service.
> * @qdid: the given queuing destination id.
> @@ -528,7 +591,7 @@ EXPORT_SYMBOL_GPL(dpaa2_io_service_acquire);
>
> /**
> * dpaa2_io_store_create() - Create the dma memory storage for dequeue result.
> - * @max_frames: the maximum number of dequeued result for frames, must be <= 16.
> + * @max_frames: the maximum number of dequeued result for frames, must be <= 32.
> * @dev: the device to allow mapping/unmapping the DMAable region.
> *
> * The size of the storage is "max_frames*sizeof(struct dpaa2_dq)".
> @@ -543,7 +606,7 @@ struct dpaa2_io_store *dpaa2_io_store_create(unsigned int max_frames,
> struct dpaa2_io_store *ret;
> size_t size;
>
> - if (!max_frames || (max_frames > 16))
> + if (!max_frames || (max_frames > 32))
> return NULL;
>
> ret = kmalloc(sizeof(*ret), GFP_KERNEL);
> diff --git a/drivers/soc/fsl/dpio/qbman-portal.c b/drivers/soc/fsl/dpio/qbman-portal.c
> index c66f5b7..0ed2c8f 100644
> --- a/drivers/soc/fsl/dpio/qbman-portal.c
> +++ b/drivers/soc/fsl/dpio/qbman-portal.c
> @@ -1,13 +1,14 @@
> // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
> /*
> * Copyright (C) 2014-2016 Freescale Semiconductor, Inc.
> - * Copyright 2016 NXP
> + * Copyright 2016-2019 NXP
> *
> */
>
> #include <asm/cacheflush.h>
> #include <linux/io.h>
> #include <linux/slab.h>
> +#include <linux/spinlock.h>
> #include <soc/fsl/dpaa2-global.h>
>
> #include "qbman-portal.h"
> @@ -28,6 +29,7 @@
>
> /* CINH register offsets */
> #define QBMAN_CINH_SWP_EQCR_PI 0x800
> +#define QBMAN_CINH_SWP_EQCR_CI 0x840
> #define QBMAN_CINH_SWP_EQAR 0x8c0
> #define QBMAN_CINH_SWP_CR_RT 0x900
> #define QBMAN_CINH_SWP_VDQCR_RT 0x940
> @@ -51,6 +53,8 @@
> #define QBMAN_CENA_SWP_CR 0x600
> #define QBMAN_CENA_SWP_RR(vb) (0x700 + ((u32)(vb) >> 1))
> #define QBMAN_CENA_SWP_VDQCR 0x780
> +#define QBMAN_CENA_SWP_EQCR_CI 0x840
> +#define QBMAN_CENA_SWP_EQCR_CI_MEMBACK 0x1840
>
> /* CENA register offsets in memory-backed mode */
> #define QBMAN_CENA_SWP_DQRR_MEM(n) (0x800 + ((u32)(n) << 6))
> @@ -78,6 +82,12 @@
> /* opaque token for static dequeues */
> #define QMAN_SDQCR_TOKEN 0xbb
>
> +#define QBMAN_EQCR_DCA_IDXMASK 0x0f
> +#define QBMAN_ENQUEUE_FLAG_DCA (1ULL << 31)
> +
> +#define EQ_DESC_SIZE_WITHOUT_FD 29
> +#define EQ_DESC_SIZE_FD_START 32
> +
> enum qbman_sdqcr_dct {
> qbman_sdqcr_dct_null = 0,
> qbman_sdqcr_dct_prio_ics,
> @@ -90,6 +100,82 @@ enum qbman_sdqcr_fc {
> qbman_sdqcr_fc_up_to_3 = 1
> };
>
> +/* Internal Function declaration */
> +static int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd);
> +static int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd);
> +static int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames);
> +static int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames);
> +static int
> +qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames);
> +static
> +int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames);
> +static int qbman_swp_pull_direct(struct qbman_swp *s,
> + struct qbman_pull_desc *d);
> +static int qbman_swp_pull_mem_back(struct qbman_swp *s,
> + struct qbman_pull_desc *d);
> +
> +const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s);
> +const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s);
> +
> +static int qbman_swp_release_direct(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers,
> + unsigned int num_buffers);
> +static int qbman_swp_release_mem_back(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers,
> + unsigned int num_buffers);
> +
> +/* Function pointers */
> +int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd)
> + = qbman_swp_enqueue_ring_mode_direct;
> +
> +int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames)
> + = qbman_swp_enqueue_multiple_direct;
> +
> +int
> +(*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames)
> + = qbman_swp_enqueue_multiple_desc_direct;
> +
> +int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d)
> + = qbman_swp_pull_direct;
> +
> +const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s)
> + = qbman_swp_dqrr_next_direct;
> +
> +int (*qbman_swp_release_ptr)(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers,
> + unsigned int num_buffers)
> + = qbman_swp_release_direct;
> +
> /* Portal Access */
>
> static inline u32 qbman_read_register(struct qbman_swp *p, u32 offset)
> @@ -146,6 +232,15 @@ static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm,
>
> #define QMAN_RT_MODE 0x00000100
>
> +static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
> +{
> + /* 'first' is included, 'last' is excluded */
> + if (first <= last)
> + return last - first;
> + else
> + return (2 * ringsize) - (first - last);
> +}
> +
> /**
> * qbman_swp_init() - Create a functional object representing the given
> * QBMan portal descriptor.
> @@ -156,8 +251,12 @@ static inline u32 qbman_set_swp_cfg(u8 max_fill, u8 wn, u8 est, u8 rpm, u8 dcm,
> */
> struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
> {
> - struct qbman_swp *p = kmalloc(sizeof(*p), GFP_KERNEL);
> + struct qbman_swp *p = kzalloc(sizeof(*p), GFP_KERNEL);
> u32 reg;
> + u32 mask_size;
> + u32 eqcr_pi;
> +
> + spin_lock_init(&p->access_spinlock);
>
> if (!p)
> return NULL;
> @@ -189,22 +288,39 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
> if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
> memset(p->addr_cena, 0, 64 * 1024);
>
> - reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
> - 1, /* Writes Non-cacheable */
> - 0, /* EQCR_CI stashing threshold */
> - 3, /* RPM: Valid bit mode, RCR in array mode */
> - 2, /* DCM: Discrete consumption ack mode */
> - 3, /* EPM: Valid bit mode, EQCR in array mode */
> - 1, /* mem stashing drop enable == TRUE */
> - 1, /* mem stashing priority == TRUE */
> - 1, /* mem stashing enable == TRUE */
> - 1, /* dequeue stashing priority == TRUE */
> - 0, /* dequeue stashing enable == FALSE */
> - 0); /* EQCR_CI stashing priority == FALSE */
> - if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000)
> + if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
> +
> + reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
> + 0, /* Writes Non-cacheable */
> + 1, /* EQCR_CI stashing threshold */
> + 3, /* RPM: RCR in array mode */
> + 2, /* DCM: Discrete consumption ack */
> + 0, /* EPM: EQCR in ring mode */
> + 1, /* mem stashing drop enable */
> + 1, /* mem stashing priority enable */
> + 1, /* mem stashing enable */
> + 1, /* dequeue stashing priority enable */
> + 0, /* dequeue stashing enable */
> + 0); /* EQCR_CI stashing priority enable */
> + } else {
> + reg = qbman_set_swp_cfg(p->dqrr.dqrr_size,
> + 1, /* Writes Non-cacheable */
> + 0, /* EQCR_CI stashing threshold */
> + 3, /* RPM: RCR in array mode */
> + 2, /* DCM: Discrete consumption ack */
> + 2, /* EPM: EQCR in ring mode */
> + 1, /* mem stashing drop enable enable */
> + 1, /* mem stashing priority enable */
> + 1, /* mem stashing enable */
> + 1, /* dequeue stashing priority enable */
> + 0, /* dequeue stashing enable enable */
> + 0); /* EQCR_CI stashing priority enable */
> + }
> + if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
> reg |= 1 << SWP_CFG_CPBS_SHIFT | /* memory-backed mode */
> 1 << SWP_CFG_VPM_SHIFT | /* VDQCR read triggered mode */
> 1 << SWP_CFG_CPM_SHIFT; /* CR read triggered mode */
> + }
>
> qbman_write_register(p, QBMAN_CINH_SWP_CFG, reg);
> reg = qbman_read_register(p, QBMAN_CINH_SWP_CFG);
> @@ -225,6 +341,30 @@ struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d)
> * applied when dequeues from a specific channel are enabled.
> */
> qbman_write_register(p, QBMAN_CINH_SWP_SDQCR, 0);
> +
> + p->eqcr.pi_ring_size = 8;
> + if ((p->desc->qman_version & QMAN_REV_MASK) >= QMAN_REV_5000) {
> + p->eqcr.pi_ring_size = 32;
> + qbman_swp_enqueue_ring_mode_ptr =
> + qbman_swp_enqueue_ring_mode_mem_back;
> + qbman_swp_enqueue_multiple_ptr =
> + qbman_swp_enqueue_multiple_mem_back;
> + qbman_swp_enqueue_multiple_desc_ptr =
> + qbman_swp_enqueue_multiple_desc_mem_back;
> + qbman_swp_pull_ptr = qbman_swp_pull_mem_back;
> + qbman_swp_dqrr_next_ptr = qbman_swp_dqrr_next_mem_back;
> + qbman_swp_release_ptr = qbman_swp_release_mem_back;
> + }
> +
> + for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
> + p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;
> + eqcr_pi = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_PI);
> + p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
> + p->eqcr.pi_vb = eqcr_pi & QB_VALID_BIT;
> + p->eqcr.ci = qbman_read_register(p, QBMAN_CINH_SWP_EQCR_CI)
> + & p->eqcr.pi_ci_mask;
> + p->eqcr.available = p->eqcr.pi_ring_size;
> +
> return p;
> }
>
> @@ -378,6 +518,7 @@ enum qb_enqueue_commands {
> #define QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT 2
> #define QB_ENQUEUE_CMD_IRQ_ON_DISPATCH_SHIFT 3
> #define QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT 4
> +#define QB_ENQUEUE_CMD_DCA_EN_SHIFT 7
>
> /**
> * qbman_eq_desc_clear() - Clear the contents of a descriptor to
> @@ -389,6 +530,16 @@ void qbman_eq_desc_clear(struct qbman_eq_desc *d)
> }
>
> /**
> + * qbman_eq_desc_min_clear() - Clear the contents of a minimal
> + * enqueue descriptor to
> + * default/starting state.
> + */
> +void qbman_eq_desc_min_clear(struct qbman_eq_desc_min *d)
> +{
> + memset(d, 0, sizeof(*d));
> +}
> +
> +/**
> * qbman_eq_desc_set_no_orp() - Set enqueue descriptor without orp
> * @d: the enqueue descriptor.
> * @response_success: 1 = enqueue with response always; 0 = enqueue with
> @@ -403,6 +554,22 @@ void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success)
> d->verb |= enqueue_rejects_to_fq;
> }
>
> +/**
> + * qbman_eq_desc_set_no_orp_min() - Set minimal enqueue descriptor without orp
> + * @d: the enqueue descriptor.
> + * @response_success: 1 = enqueue with response always; 0 = enqueue with
> + * rejections returned on a FQ.
> + */
> +void qbman_eq_desc_set_no_orp_min(struct qbman_eq_desc_min *d,
> + int respond_success)
> +{
> + d->verb &= ~(1 << QB_ENQUEUE_CMD_ORP_ENABLE_SHIFT);
> + if (respond_success)
> + d->verb |= enqueue_response_always;
> + else
> + d->verb |= enqueue_rejects_to_fq;
> +}
> +
> /*
> * Exactly one of the following descriptor "targets" should be set. (Calling any
> * one of these will replace the effect of any prior call to one of these.)
> @@ -422,6 +589,17 @@ void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid)
> }
>
> /**
> + * qbman_eq_desc_set_min_fq() - set the FQ for the minimal enqueue command
> + * @d: the enqueue descriptor
> + * @fqid: the id of the frame queue to be enqueued
> + */
> +void qbman_eq_desc_set_min_fq(struct qbman_eq_desc_min *d, u32 fqid)
> +{
> + d->verb &= ~(1 << QB_ENQUEUE_CMD_TARGET_TYPE_SHIFT);
> + d->tgtid = cpu_to_le32(fqid);
> +}
> +
> +/**
> * qbman_eq_desc_set_qd() - Set Queuing Destination for the enqueue command
> * @d: the enqueue descriptor
> * @qdid: the id of the queuing destination to be enqueued
> @@ -453,41 +631,340 @@ static inline void qbman_write_eqcr_am_rt_register(struct qbman_swp *p,
> QMAN_RT_MODE);
> }
>
> +#define QB_RT_BIT ((u32)0x100)
> /**
> - * qbman_swp_enqueue() - Issue an enqueue command
> + * qbman_swp_enqueue_ring_mode_direct() - Issue an enqueue command
> * @s: the software portal used for enqueue
> * @d: the enqueue descriptor
> * @fd: the frame descriptor to be enqueued
> *
> - * Please note that 'fd' should only be NULL if the "action" of the
> - * descriptor is "orp_hole" or "orp_nesn".
> + * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
> + */
> +static
> +int qbman_swp_enqueue_ring_mode_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd)
> +{
> + int flags = 0;
> + int ret = qbman_swp_enqueue_multiple_direct(s, d, fd, &flags, 1);
> +
> + if (ret >= 0)
> + ret = 0;
> + else
> + ret = -EBUSY;
> + return ret;
> +}
> +
> +/**
> + * qbman_swp_enqueue_ring_mode_mem_back() - Issue an enqueue command
> + * @s: the software portal used for enqueue
> + * @d: the enqueue descriptor
> + * @fd: the frame descriptor to be enqueued
> *
> * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
> */
> -int qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
> - const struct dpaa2_fd *fd)
> +static
> +int qbman_swp_enqueue_ring_mode_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd)
> {
> - struct qbman_eq_desc *p;
> - u32 eqar = qbman_read_register(s, QBMAN_CINH_SWP_EQAR);
> + int flags = 0;
> + int ret = qbman_swp_enqueue_multiple_mem_back(s, d, fd, &flags, 1);
>
> - if (!EQAR_SUCCESS(eqar))
> - return -EBUSY;
> + if (ret >= 0)
> + ret = 0;
> + else
> + ret = -EBUSY;
> + return ret;
> +}
>
> - p = qbman_get_cmd(s, QBMAN_CENA_SWP_EQCR(EQAR_IDX(eqar)));
> - memcpy(&p->dca, &d->dca, 31);
> - memcpy(&p->fd, fd, sizeof(*fd));
> +/**
> + * qbman_swp_enqueue_multiple_direct() - Issue a multi enqueue command
> + * using one enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: the enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static
> +int qbman_swp_enqueue_multiple_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames)
> +{
> + uint32_t *p = NULL;
> + const uint32_t *cl = (uint32_t *)d;
> + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
> + int i, num_enqueued = 0;
> +
> + half_mask = (s->eqcr.pi_ci_mask>>1);
> + full_mask = s->eqcr.pi_ci_mask;
> +
> + if (!s->eqcr.available) {
> + eqcr_ci = s->eqcr.ci;
> + p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
> + s->eqcr.ci = __raw_readl(p) & full_mask;
> +
> + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
> + eqcr_ci, s->eqcr.ci);
> + if (!s->eqcr.available)
> + return 0;
> + }
>
> - if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
> - /* Set the verb byte, have to substitute in the valid-bit */
> - dma_wmb();
> - p->verb = d->verb | EQAR_VB(eqar);
> - } else {
> - p->verb = d->verb | EQAR_VB(eqar);
> - dma_wmb();
> - qbman_write_eqcr_am_rt_register(s, EQAR_IDX(eqar));
> + eqcr_pi = s->eqcr.pi;
> + num_enqueued = (s->eqcr.available < num_frames) ?
> + s->eqcr.available : num_frames;
> + s->eqcr.available -= num_enqueued;
> + /* Fill in the EQCR ring */
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + /* Skip copying the verb */
> + memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
> + memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
> + &fd[i], sizeof(*fd));
> + eqcr_pi++;
> }
>
> - return 0;
> + dma_wmb();
> +
> + /* Set the verb byte, have to substitute in the valid-bit */
> + eqcr_pi = s->eqcr.pi;
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + p[0] = cl[0] | s->eqcr.pi_vb;
> + if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
> + struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
> +
> + d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
> + ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
> + }
> + eqcr_pi++;
> + if (!(eqcr_pi & half_mask))
> + s->eqcr.pi_vb ^= QB_VALID_BIT;
> + }
> +
> + s->eqcr.pi = (s->eqcr.pi + num_enqueued) & full_mask;
> +
> + return num_enqueued;
> +}
> +
> +/**
> + * qbman_swp_enqueue_multiple_mem_back() - Issue a multi enqueue command
> + * using one enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: the enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static
> +int qbman_swp_enqueue_multiple_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames)
> +{
> + uint32_t *p = NULL;
> + const uint32_t *cl = (uint32_t *)(d);
> + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
> + int i, num_enqueued = 0;
> + unsigned long irq_flags;
> +
> + spin_lock(&s->access_spinlock);
> + local_irq_save(irq_flags);
> +
> + half_mask = (s->eqcr.pi_ci_mask>>1);
> + full_mask = s->eqcr.pi_ci_mask;
> + if (!s->eqcr.available) {
> + eqcr_ci = s->eqcr.ci;
> + p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
> + s->eqcr.ci = __raw_readl(p) & full_mask;
> + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
> + eqcr_ci, s->eqcr.ci);
> + if (!s->eqcr.available) {
> + local_irq_restore(irq_flags);
> + spin_unlock(&s->access_spinlock);
> + return 0;
> + }
> + }
> +
> + eqcr_pi = s->eqcr.pi;
> + num_enqueued = (s->eqcr.available < num_frames) ?
> + s->eqcr.available : num_frames;
> + s->eqcr.available -= num_enqueued;
> + /* Fill in the EQCR ring */
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + /* Skip copying the verb */
> + memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
> + memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
> + &fd[i], sizeof(*fd));
> + eqcr_pi++;
> + }
> +
> + /* Set the verb byte, have to substitute in the valid-bit */
> + eqcr_pi = s->eqcr.pi;
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + p[0] = cl[0] | s->eqcr.pi_vb;
> + if (flags && (flags[i] & QBMAN_ENQUEUE_FLAG_DCA)) {
> + struct qbman_eq_desc *d = (struct qbman_eq_desc *)p;
> +
> + d->dca = (1 << QB_ENQUEUE_CMD_DCA_EN_SHIFT) |
> + ((flags[i]) & QBMAN_EQCR_DCA_IDXMASK);
> + }
> + eqcr_pi++;
> + if (!(eqcr_pi & half_mask))
> + s->eqcr.pi_vb ^= QB_VALID_BIT;
> + }
> + s->eqcr.pi = eqcr_pi & full_mask;
> +
> + dma_wmb();
> + qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
> + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
> + local_irq_restore(irq_flags);
> + spin_unlock(&s->access_spinlock);
> +
> + return num_enqueued;
> +}
> +
> +/**
> + * qbman_swp_enqueue_multiple_desc_direct() - Issue a multi enqueue command
> + * using multiple enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: table of minimal enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static
> +int qbman_swp_enqueue_multiple_desc_direct(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames)
> +{
> + uint32_t *p;
> + const uint32_t *cl;
> + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
> + int i, num_enqueued = 0;
> +
> + half_mask = (s->eqcr.pi_ci_mask>>1);
> + full_mask = s->eqcr.pi_ci_mask;
> + if (!s->eqcr.available) {
> + eqcr_ci = s->eqcr.ci;
> + p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI;
> + s->eqcr.ci = __raw_readl(p) & full_mask;
> + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
> + eqcr_ci, s->eqcr.ci);
> + if (!s->eqcr.available)
> + return 0;
> + }
> +
> + eqcr_pi = s->eqcr.pi;
> + num_enqueued = (s->eqcr.available < num_frames) ?
> + s->eqcr.available : num_frames;
> + s->eqcr.available -= num_enqueued;
> + /* Fill in the EQCR ring */
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + cl = (uint32_t *)(&d[i]);
> + /* Skip copying the verb */
> + memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
> + memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
> + &fd[i], sizeof(*fd));
> + eqcr_pi++;
> + }
> +
> + dma_wmb();
> +
> + /* Set the verb byte, have to substitute in the valid-bit */
> + eqcr_pi = s->eqcr.pi;
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + cl = (uint32_t *)(&d[i]);
> + p[0] = cl[0] | s->eqcr.pi_vb;
> + eqcr_pi++;
> + if (!(eqcr_pi & half_mask))
> + s->eqcr.pi_vb ^= QB_VALID_BIT;
> + }
> + s->eqcr.pi = (s->eqcr.pi + num_enqueued) & full_mask;
> +
> + return num_enqueued;
> +}
> +
> +/**
> + * qbman_swp_enqueue_multiple_desc_mem_back() - Issue a multi enqueue command
> + * using multiple enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: table of minimal enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static
> +int qbman_swp_enqueue_multiple_desc_mem_back(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames)
> +{
> + uint32_t *p;
> + const uint32_t *cl;
> + uint32_t eqcr_ci, eqcr_pi, half_mask, full_mask;
> + int i, num_enqueued = 0;
> +
> + half_mask = (s->eqcr.pi_ci_mask>>1);
> + full_mask = s->eqcr.pi_ci_mask;
> + if (!s->eqcr.available) {
> + eqcr_ci = s->eqcr.ci;
> + p = s->addr_cena + QBMAN_CENA_SWP_EQCR_CI_MEMBACK;
> + s->eqcr.ci = __raw_readl(p) & full_mask;
> + s->eqcr.available = qm_cyc_diff(s->eqcr.pi_ring_size,
> + eqcr_ci, s->eqcr.ci);
> + if (!s->eqcr.available)
> + return 0;
> + }
> +
> + eqcr_pi = s->eqcr.pi;
> + num_enqueued = (s->eqcr.available < num_frames) ?
> + s->eqcr.available : num_frames;
> + s->eqcr.available -= num_enqueued;
> + /* Fill in the EQCR ring */
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + cl = (uint32_t *)(&d[i]);
> + /* Skip copying the verb */
> + memcpy(&p[1], &cl[1], EQ_DESC_SIZE_WITHOUT_FD - 1);
> + memcpy(&p[EQ_DESC_SIZE_FD_START/sizeof(uint32_t)],
> + &fd[i], sizeof(*fd));
> + eqcr_pi++;
> + }
> +
> + /* Set the verb byte, have to substitute in the valid-bit */
> + eqcr_pi = s->eqcr.pi;
> + for (i = 0; i < num_enqueued; i++) {
> + p = (s->addr_cena + QBMAN_CENA_SWP_EQCR(eqcr_pi & half_mask));
> + cl = (uint32_t *)(&d[i]);
> + p[0] = cl[0] | s->eqcr.pi_vb;
> + eqcr_pi++;
> + if (!(eqcr_pi & half_mask))
> + s->eqcr.pi_vb ^= QB_VALID_BIT;
> + }
> +
> + s->eqcr.pi = eqcr_pi & full_mask;
> +
> + dma_wmb();
> + qbman_write_register(s, QBMAN_CINH_SWP_EQCR_PI,
> + (QB_RT_BIT)|(s->eqcr.pi)|s->eqcr.pi_vb);
> +
> + return num_enqueued;
> }
>
> /* Static (push) dequeue */
> @@ -645,7 +1122,7 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
> }
>
> /**
> - * qbman_swp_pull() - Issue the pull dequeue command
> + * qbman_swp_pull_direct() - Issue the pull dequeue command
> * @s: the software portal object
> * @d: the software portal descriptor which has been configured with
> * the set of qbman_pull_desc_set_*() calls
> @@ -653,7 +1130,7 @@ void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
> * Return 0 for success, and -EBUSY if the software portal is not ready
> * to do pull dequeue.
> */
> -int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
> +static int qbman_swp_pull_direct(struct qbman_swp *s, struct qbman_pull_desc *d)
> {
> struct qbman_pull_desc *p;
>
> @@ -671,18 +1148,48 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
> p->dq_src = d->dq_src;
> p->rsp_addr = d->rsp_addr;
> p->rsp_addr_virt = d->rsp_addr_virt;
> + dma_wmb();
> + /* Set the verb byte, have to substitute in the valid-bit */
> + p->verb = d->verb | s->vdq.valid_bit;
> + s->vdq.valid_bit ^= QB_VALID_BIT;
>
> - if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
> - dma_wmb();
> - /* Set the verb byte, have to substitute in the valid-bit */
> - p->verb = d->verb | s->vdq.valid_bit;
> - s->vdq.valid_bit ^= QB_VALID_BIT;
> - } else {
> - p->verb = d->verb | s->vdq.valid_bit;
> - s->vdq.valid_bit ^= QB_VALID_BIT;
> - dma_wmb();
> - qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
> + return 0;
> +}
> +
> +/**
> + * qbman_swp_pull_mem_back() - Issue the pull dequeue command
> + * @s: the software portal object
> + * @d: the software portal descriptor which has been configured with
> + * the set of qbman_pull_desc_set_*() calls
> + *
> + * Return 0 for success, and -EBUSY if the software portal is not ready
> + * to do pull dequeue.
> + */
> +static int qbman_swp_pull_mem_back(struct qbman_swp *s,
> + struct qbman_pull_desc *d)
> +{
> + struct qbman_pull_desc *p;
> +
> + if (!atomic_dec_and_test(&s->vdq.available)) {
> + atomic_inc(&s->vdq.available);
> + return -EBUSY;
> }
> + s->vdq.storage = (void *)(uintptr_t)d->rsp_addr_virt;
> + if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR);
> + else
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_VDQCR_MEM);
> + p->numf = d->numf;
> + p->tok = QMAN_DQ_TOKEN_VALID;
> + p->dq_src = d->dq_src;
> + p->rsp_addr = d->rsp_addr;
> + p->rsp_addr_virt = d->rsp_addr_virt;
> +
> + /* Set the verb byte, have to substitute in the valid-bit */
> + p->verb = d->verb | s->vdq.valid_bit;
> + s->vdq.valid_bit ^= QB_VALID_BIT;
> + dma_wmb();
> + qbman_write_register(s, QBMAN_CINH_SWP_VDQCR_RT, QMAN_RT_MODE);
>
> return 0;
> }
> @@ -690,14 +1197,14 @@ int qbman_swp_pull(struct qbman_swp *s, struct qbman_pull_desc *d)
> #define QMAN_DQRR_PI_MASK 0xf
>
> /**
> - * qbman_swp_dqrr_next() - Get an valid DQRR entry
> + * qbman_swp_dqrr_next_direct() - Get an valid DQRR entry
> * @s: the software portal object
> *
> * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
> * only once, so repeated calls can return a sequence of DQRR entries, without
> * requiring they be consumed immediately or in any particular order.
> */
> -const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s)
> +const struct dpaa2_dq *qbman_swp_dqrr_next_direct(struct qbman_swp *s)
> {
> u32 verb;
> u32 response_verb;
> @@ -740,10 +1247,7 @@ const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s)
> QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
> }
>
> - if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
> - p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
> - else
> - p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx));
> verb = p->dq.verb;
>
> /*
> @@ -785,6 +1289,98 @@ const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s)
> }
>
> /**
> + * qbman_swp_dqrr_next_mem_back() - Get an valid DQRR entry
> + * @s: the software portal object
> + *
> + * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
> + * only once, so repeated calls can return a sequence of DQRR entries, without
> + * requiring they be consumed immediately or in any particular order.
> + */
> +const struct dpaa2_dq *qbman_swp_dqrr_next_mem_back(struct qbman_swp *s)
> +{
> + u32 verb;
> + u32 response_verb;
> + u32 flags;
> + struct dpaa2_dq *p;
> +
> + /* Before using valid-bit to detect if something is there, we have to
> + * handle the case of the DQRR reset bug...
> + */
> + if (unlikely(s->dqrr.reset_bug)) {
> + /*
> + * We pick up new entries by cache-inhibited producer index,
> + * which means that a non-coherent mapping would require us to
> + * invalidate and read *only* once that PI has indicated that
> + * there's an entry here. The first trip around the DQRR ring
> + * will be much less efficient than all subsequent trips around
> + * it...
> + */
> + u8 pi = qbman_read_register(s, QBMAN_CINH_SWP_DQPI) &
> + QMAN_DQRR_PI_MASK;
> +
> + /* there are new entries if pi != next_idx */
> + if (pi == s->dqrr.next_idx)
> + return NULL;
> +
> + /*
> + * if next_idx is/was the last ring index, and 'pi' is
> + * different, we can disable the workaround as all the ring
> + * entries have now been DMA'd to so valid-bit checking is
> + * repaired. Note: this logic needs to be based on next_idx
> + * (which increments one at a time), rather than on pi (which
> + * can burst and wrap-around between our snapshots of it).
> + */
> + if (s->dqrr.next_idx == (s->dqrr.dqrr_size - 1)) {
> + pr_debug("next_idx=%d, pi=%d, clear reset bug\n",
> + s->dqrr.next_idx, pi);
> + s->dqrr.reset_bug = 0;
> + }
> + prefetch(qbman_get_cmd(s,
> + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
> + }
> +
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR_MEM(s->dqrr.next_idx));
> + verb = p->dq.verb;
> +
> + /*
> + * If the valid-bit isn't of the expected polarity, nothing there. Note,
> + * in the DQRR reset bug workaround, we shouldn't need to skip these
> + * check, because we've already determined that a new entry is available
> + * and we've invalidated the cacheline before reading it, so the
> + * valid-bit behaviour is repaired and should tell us what we already
> + * knew from reading PI.
> + */
> + if ((verb & QB_VALID_BIT) != s->dqrr.valid_bit) {
> + prefetch(qbman_get_cmd(s,
> + QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
> + return NULL;
> + }
> + /*
> + * There's something there. Move "next_idx" attention to the next ring
> + * entry (and prefetch it) before returning what we found.
> + */
> + s->dqrr.next_idx++;
> + s->dqrr.next_idx &= s->dqrr.dqrr_size - 1; /* Wrap around */
> + if (!s->dqrr.next_idx)
> + s->dqrr.valid_bit ^= QB_VALID_BIT;
> +
> + /*
> + * If this is the final response to a volatile dequeue command
> + * indicate that the vdq is available
> + */
> + flags = p->dq.stat;
> + response_verb = verb & QBMAN_RESULT_MASK;
> + if (response_verb == QBMAN_RESULT_DQ &&
> + (flags & DPAA2_DQ_STAT_VOLATILE) &&
> + (flags & DPAA2_DQ_STAT_EXPIRED))
> + atomic_inc(&s->vdq.available);
> +
> + prefetch(qbman_get_cmd(s, QBMAN_CENA_SWP_DQRR(s->dqrr.next_idx)));
> +
> + return p;
> +}
> +
> +/**
> * qbman_swp_dqrr_consume() - Consume DQRR entries previously returned from
> * qbman_swp_dqrr_next().
> * @s: the software portal object
> @@ -872,7 +1468,7 @@ void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
> #define RAR_SUCCESS(rar) ((rar) & 0x100)
>
> /**
> - * qbman_swp_release() - Issue a buffer release command
> + * qbman_swp_release_direct() - Issue a buffer release command
> * @s: the software portal object
> * @d: the release descriptor
> * @buffers: a pointer pointing to the buffer address to be released
> @@ -880,8 +1476,53 @@ void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable)
> *
> * Return 0 for success, -EBUSY if the release command ring is not ready.
> */
> -int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
> - const u64 *buffers, unsigned int num_buffers)
> +int qbman_swp_release_direct(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers, unsigned int num_buffers)
> +{
> + int i;
> + struct qbman_release_desc *p;
> + u32 rar;
> +
> + if (!num_buffers || num_buffers > 7)
> + return -EINVAL;
> +
> + rar = qbman_read_register(s, QBMAN_CINH_SWP_RAR);
> + if (!RAR_SUCCESS(rar))
> + return -EBUSY;
> +
> + /* Start the release command */
> + if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000)
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR(RAR_IDX(rar)));
> + else
> + p = qbman_get_cmd(s, QBMAN_CENA_SWP_RCR_MEM(RAR_IDX(rar)));
> + /* Copy the caller's buffer pointers to the command */
> + for (i = 0; i < num_buffers; i++)
> + p->buf[i] = cpu_to_le64(buffers[i]);
> + p->bpid = d->bpid;
> +
> + /*
> + * Set the verb byte, have to substitute in the valid-bit
> + * and the number of buffers.
> + */
> + dma_wmb();
> + p->verb = d->verb | RAR_VB(rar) | num_buffers;
> +
> + return 0;
> +}
> +
> +/**
> + * qbman_swp_release_mem_back() - Issue a buffer release command
> + * @s: the software portal object
> + * @d: the release descriptor
> + * @buffers: a pointer pointing to the buffer address to be released
> + * @num_buffers: number of buffers to be released, must be less than 8
> + *
> + * Return 0 for success, -EBUSY if the release command ring is not ready.
> + */
> +int qbman_swp_release_mem_back(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers, unsigned int num_buffers)
> {
> int i;
> struct qbman_release_desc *p;
> @@ -904,19 +1545,10 @@ int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
> p->buf[i] = cpu_to_le64(buffers[i]);
> p->bpid = d->bpid;
>
> - if ((s->desc->qman_version & QMAN_REV_MASK) < QMAN_REV_5000) {
> - /*
> - * Set the verb byte, have to substitute in the valid-bit
> - * and the number of buffers.
> - */
> - dma_wmb();
> - p->verb = d->verb | RAR_VB(rar) | num_buffers;
> - } else {
> - p->verb = d->verb | RAR_VB(rar) | num_buffers;
> - dma_wmb();
> - qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
> - RAR_IDX(rar) * 4, QMAN_RT_MODE);
> - }
> + p->verb = d->verb | RAR_VB(rar) | num_buffers;
> + dma_wmb();
> + qbman_write_register(s, QBMAN_CINH_SWP_RCR_AM_RT +
> + RAR_IDX(rar) * 4, QMAN_RT_MODE);
>
> return 0;
> }
> diff --git a/drivers/soc/fsl/dpio/qbman-portal.h b/drivers/soc/fsl/dpio/qbman-portal.h
> index f3ec5d2..f4b6e47 100644
> --- a/drivers/soc/fsl/dpio/qbman-portal.h
> +++ b/drivers/soc/fsl/dpio/qbman-portal.h
> @@ -9,6 +9,13 @@
>
> #include <soc/fsl/dpaa2-fd.h>
>
> +#define QMAN_REV_4000 0x04000000
> +#define QMAN_REV_4100 0x04010000
> +#define QMAN_REV_4101 0x04010001
> +#define QMAN_REV_5000 0x05000000
> +
> +#define QMAN_REV_MASK 0xffff0000
> +
> struct dpaa2_dq;
> struct qbman_swp;
>
> @@ -67,6 +74,22 @@ enum qbman_pull_type_e {
> #define QBMAN_FQ_XOFF 0x4e
>
> /* structure of enqueue descriptor */
> +struct qbman_eq_desc_min {
> + u8 verb;
> + u8 dca;
> + __le16 seqnum;
> + __le16 orpid;
> + __le16 reserved1;
> + __le32 tgtid;
> + __le32 tag;
> + __le16 qdbin;
> + u8 qpri;
> + u8 reserved[3];
> + u8 wae;
> + u8 rspid;
> + __le64 rsp_addr;
> +};
> +
> struct qbman_eq_desc {
> u8 verb;
> u8 dca;
> @@ -132,8 +155,48 @@ struct qbman_swp {
> u8 dqrr_size;
> int reset_bug; /* indicates dqrr reset workaround is needed */
> } dqrr;
> +
> + struct {
> + u32 pi;
> + u32 pi_vb;
> + u32 pi_ring_size;
> + u32 pi_ci_mask;
> + u32 ci;
> + int available;
> + u32 pend;
> + u32 no_pfdr;
> + } eqcr;
> +
> + spinlock_t access_spinlock;
> };
>
> +/* Function pointers */
> +extern
> +int (*qbman_swp_enqueue_ring_mode_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd);
> +extern
> +int (*qbman_swp_enqueue_multiple_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames);
> +extern
> +int (*qbman_swp_enqueue_multiple_desc_ptr)(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames);
> +extern
> +int (*qbman_swp_pull_ptr)(struct qbman_swp *s, struct qbman_pull_desc *d);
> +extern
> +const struct dpaa2_dq *(*qbman_swp_dqrr_next_ptr)(struct qbman_swp *s);
> +extern
> +int (*qbman_swp_release_ptr)(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers,
> + unsigned int num_buffers);
> +
> +/* Functions */
> struct qbman_swp *qbman_swp_init(const struct qbman_swp_desc *d);
> void qbman_swp_finish(struct qbman_swp *p);
> u32 qbman_swp_interrupt_read_status(struct qbman_swp *p);
> @@ -158,29 +221,25 @@ void qbman_pull_desc_set_wq(struct qbman_pull_desc *d, u32 wqid,
> void qbman_pull_desc_set_channel(struct qbman_pull_desc *d, u32 chid,
> enum qbman_pull_type_e dct);
>
> -int qbman_swp_pull(struct qbman_swp *p, struct qbman_pull_desc *d);
> -
> -const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s);
> void qbman_swp_dqrr_consume(struct qbman_swp *s, const struct dpaa2_dq *dq);
>
> int qbman_result_has_new_result(struct qbman_swp *p, const struct dpaa2_dq *dq);
>
> void qbman_eq_desc_clear(struct qbman_eq_desc *d);
> +void qbman_eq_desc_min_clear(struct qbman_eq_desc_min *d);
> void qbman_eq_desc_set_no_orp(struct qbman_eq_desc *d, int respond_success);
> +void qbman_eq_desc_set_no_orp_min(struct qbman_eq_desc_min *d,
> + int respond_success);
> void qbman_eq_desc_set_token(struct qbman_eq_desc *d, u8 token);
> void qbman_eq_desc_set_fq(struct qbman_eq_desc *d, u32 fqid);
> +void qbman_eq_desc_set_min_fq(struct qbman_eq_desc_min *d, u32 fqid);
> void qbman_eq_desc_set_qd(struct qbman_eq_desc *d, u32 qdid,
> u32 qd_bin, u32 qd_prio);
>
> -int qbman_swp_enqueue(struct qbman_swp *p, const struct qbman_eq_desc *d,
> - const struct dpaa2_fd *fd);
> -
> void qbman_release_desc_clear(struct qbman_release_desc *d);
> void qbman_release_desc_set_bpid(struct qbman_release_desc *d, u16 bpid);
> void qbman_release_desc_set_rcdi(struct qbman_release_desc *d, int enable);
>
> -int qbman_swp_release(struct qbman_swp *s, const struct qbman_release_desc *d,
> - const u64 *buffers, unsigned int num_buffers);
> int qbman_swp_acquire(struct qbman_swp *s, u16 bpid, u64 *buffers,
> unsigned int num_buffers);
> int qbman_swp_alt_fq_state(struct qbman_swp *s, u32 fqid,
> @@ -194,6 +253,61 @@ void qbman_swp_mc_submit(struct qbman_swp *p, void *cmd, u8 cmd_verb);
> void *qbman_swp_mc_result(struct qbman_swp *p);
>
> /**
> + * qbman_swp_enqueue() - Issue an enqueue command
> + * @s: the software portal used for enqueue
> + * @d: the enqueue descriptor
> + * @fd: the frame descriptor to be enqueued
> + *
> + * Return 0 for successful enqueue, -EBUSY if the EQCR is not ready.
> + */
> +static inline int
> +qbman_swp_enqueue(struct qbman_swp *s, const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd)
> +{
> + return qbman_swp_enqueue_ring_mode_ptr(s, d, fd);
> +}
> +
> +/**
> + * qbman_swp_enqueue_multiple() - Issue a multi enqueue command
> + * using one enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: the enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @flags: table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static inline int
> +qbman_swp_enqueue_multiple(struct qbman_swp *s,
> + const struct qbman_eq_desc *d,
> + const struct dpaa2_fd *fd,
> + uint32_t *flags,
> + int num_frames)
> +{
> + return qbman_swp_enqueue_multiple_ptr(s, d, fd, flags, num_frames);
> +}
> +
> +/**
> + * qbman_swp_enqueue_multiple_desc() - Issue a multi enqueue command
> + * using multiple enqueue descriptor
> + * @s: the software portal used for enqueue
> + * @d: table of minimal enqueue descriptor
> + * @fd: table pointer of frame descriptor table to be enqueued
> + * @num_frames: number of fd to be enqueued
> + *
> + * Return the number of fd enqueued, or a negative error number.
> + */
> +static inline int
> +qbman_swp_enqueue_multiple_desc(struct qbman_swp *s,
> + const struct qbman_eq_desc_min *d,
> + const struct dpaa2_fd *fd,
> + int num_frames)
> +{
> + return qbman_swp_enqueue_multiple_desc_ptr(s, d, fd, num_frames);
> +}
> +
> +/**
> * qbman_result_is_DQ() - check if the dequeue result is a dequeue response
> * @dq: the dequeue result to be checked
> *
> @@ -504,4 +618,49 @@ int qbman_bp_query(struct qbman_swp *s, u16 bpid,
>
> u32 qbman_bp_info_num_free_bufs(struct qbman_bp_query_rslt *a);
>
> +/**
> + * qbman_swp_release() - Issue a buffer release command
> + * @s: the software portal object
> + * @d: the release descriptor
> + * @buffers: a pointer pointing to the buffer address to be released
> + * @num_buffers: number of buffers to be released, must be less than 8
> + *
> + * Return 0 for success, -EBUSY if the release command ring is not ready.
> + */
> +static inline int qbman_swp_release(struct qbman_swp *s,
> + const struct qbman_release_desc *d,
> + const u64 *buffers,
> + unsigned int num_buffers)
> +{
> + return qbman_swp_release_ptr(s, d, buffers, num_buffers);
> +}
> +
> +/**
> + * qbman_swp_pull() - Issue the pull dequeue command
> + * @s: the software portal object
> + * @d: the software portal descriptor which has been configured with
> + * the set of qbman_pull_desc_set_*() calls
> + *
> + * Return 0 for success, and -EBUSY if the software portal is not ready
> + * to do pull dequeue.
> + */
> +static inline int qbman_swp_pull(struct qbman_swp *s,
> + struct qbman_pull_desc *d)
> +{
> + return qbman_swp_pull_ptr(s, d);
> +}
> +
> +/**
> + * qbman_swp_dqrr_next() - Get an valid DQRR entry
> + * @s: the software portal object
> + *
> + * Return NULL if there are no unconsumed DQRR entries. Return a DQRR entry
> + * only once, so repeated calls can return a sequence of DQRR entries, without
> + * requiring they be consumed immediately or in any particular order.
> + */
> +static inline const struct dpaa2_dq *qbman_swp_dqrr_next(struct qbman_swp *s)
> +{
> + return qbman_swp_dqrr_next_ptr(s);
> +}
> +
> #endif /* __FSL_QBMAN_PORTAL_H */