openwrt_lede/target/linux/layerscape/patches-4.4/7017-fsl_qbman-add-qbman-driver.patch

From f6f8ed4784936724154832ff9e4c5afe8caa63e4 Mon Sep 17 00:00:00 2001
From: Zhao Qiang <qiang.zhao@nxp.com>
Date: Mon, 11 Jul 2016 14:39:18 +0800
Subject: [PATCH 17/70] fsl_qbman: add qbman driver

The QMan and BMan are infrastructure components of dpaa, which are used
by both software and hardware for queuing and memory allocation/deallocation.

Signed-off-by: Roy Pledge <Roy.Pledge@freescale.com> Signed-off-by:
Camelia Groza <camelia.groza@freescale.com> Signed-off-by: Geoff Thorpe
<Geoff.Thorpe@freescale.com> Signed-off-by: Ahmed Mansour
<Ahmed.Mansour@freescale.com> Signed-off-by: Alex Porosanu
<alexandru.porosanu@nxp.com> Signed-off-by: Pan Jiafei
<Jiafei.Pan@nxp.com> Signed-off-by: Haiying Wang
<Haiying.wang@freescale.com>
Signed-off-by: Xie Jianhua-B29408 <Jianhua.Xie@freescale.com>
Signed-off-by: Zhao Qiang <qiang.zhao@nxp.com>
---
 arch/arm/Kconfig                                |    5 +
 arch/powerpc/Kconfig                            |    9 +-
 drivers/misc/Kconfig                            |   17 +
 drivers/staging/Kconfig                         |    2 +
 drivers/staging/Makefile                        |    1 +
 drivers/staging/fsl_qbman/Kconfig               |  211 +
 drivers/staging/fsl_qbman/Makefile              |   28 +
 drivers/staging/fsl_qbman/bman_config.c         |  705 +++
 drivers/staging/fsl_qbman/bman_debugfs.c        |  119 +
 drivers/staging/fsl_qbman/bman_driver.c         |  574 +++
 drivers/staging/fsl_qbman/bman_high.c           | 1141 +++++
 drivers/staging/fsl_qbman/bman_low.h            |  559 +++
 drivers/staging/fsl_qbman/bman_private.h        |  166 +
 drivers/staging/fsl_qbman/bman_test.c           |   56 +
 drivers/staging/fsl_qbman/bman_test.h           |   44 +
 drivers/staging/fsl_qbman/bman_test_high.c      |  183 +
 drivers/staging/fsl_qbman/bman_test_thresh.c    |  196 +
 drivers/staging/fsl_qbman/dpa_alloc.c           |  706 +++
 drivers/staging/fsl_qbman/dpa_sys.h             |  259 ++
 drivers/staging/fsl_qbman/dpa_sys_arm.h         |   95 +
 drivers/staging/fsl_qbman/dpa_sys_arm64.h       |  102 +
 drivers/staging/fsl_qbman/dpa_sys_ppc32.h       |   70 +
 drivers/staging/fsl_qbman/dpa_sys_ppc64.h       |   79 +
 drivers/staging/fsl_qbman/fsl_usdpaa.c          | 1982 ++++++++
 drivers/staging/fsl_qbman/fsl_usdpaa_irq.c      |  289 ++
 drivers/staging/fsl_qbman/qbman_driver.c        |   88 +
 drivers/staging/fsl_qbman/qman_config.c         | 1199 +++++
 drivers/staging/fsl_qbman/qman_debugfs.c        | 1594 +++++++
 drivers/staging/fsl_qbman/qman_driver.c         |  980 ++++
 drivers/staging/fsl_qbman/qman_high.c           | 5568 +++++++++++++++++++++++
 drivers/staging/fsl_qbman/qman_low.h            | 1407 ++++++
 drivers/staging/fsl_qbman/qman_private.h        |  398 ++
 drivers/staging/fsl_qbman/qman_test.c           |   57 +
 drivers/staging/fsl_qbman/qman_test.h           |   45 +
 drivers/staging/fsl_qbman/qman_test_high.c      |  216 +
 drivers/staging/fsl_qbman/qman_test_hotpotato.c |  499 ++
 drivers/staging/fsl_qbman/qman_utility.c        |  129 +
 include/linux/fsl_bman.h                        |  532 +++
 include/linux/fsl_qman.h                        | 3889 ++++++++++++++++
 include/linux/fsl_usdpaa.h                      |  372 ++
 40 files changed, 24569 insertions(+), 2 deletions(-)
 create mode 100644 drivers/staging/fsl_qbman/Kconfig
 create mode 100644 drivers/staging/fsl_qbman/Makefile
 create mode 100644 drivers/staging/fsl_qbman/bman_config.c
 create mode 100644 drivers/staging/fsl_qbman/bman_debugfs.c
 create mode 100644 drivers/staging/fsl_qbman/bman_driver.c
 create mode 100644 drivers/staging/fsl_qbman/bman_high.c
 create mode 100644 drivers/staging/fsl_qbman/bman_low.h
 create mode 100644 drivers/staging/fsl_qbman/bman_private.h
 create mode 100644 drivers/staging/fsl_qbman/bman_test.c
 create mode 100644 drivers/staging/fsl_qbman/bman_test.h
 create mode 100644 drivers/staging/fsl_qbman/bman_test_high.c
 create mode 100644 drivers/staging/fsl_qbman/bman_test_thresh.c
 create mode 100644 drivers/staging/fsl_qbman/dpa_alloc.c
 create mode 100644 drivers/staging/fsl_qbman/dpa_sys.h
 create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm.h
 create mode 100644 drivers/staging/fsl_qbman/dpa_sys_arm64.h
 create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc32.h
 create mode 100644 drivers/staging/fsl_qbman/dpa_sys_ppc64.h
 create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa.c
 create mode 100644 drivers/staging/fsl_qbman/fsl_usdpaa_irq.c
 create mode 100644 drivers/staging/fsl_qbman/qbman_driver.c
 create mode 100644 drivers/staging/fsl_qbman/qman_config.c
 create mode 100644 drivers/staging/fsl_qbman/qman_debugfs.c
 create mode 100644 drivers/staging/fsl_qbman/qman_driver.c
 create mode 100644 drivers/staging/fsl_qbman/qman_high.c
 create mode 100644 drivers/staging/fsl_qbman/qman_low.h
 create mode 100644 drivers/staging/fsl_qbman/qman_private.h
 create mode 100644 drivers/staging/fsl_qbman/qman_test.c
 create mode 100644 drivers/staging/fsl_qbman/qman_test.h
 create mode 100644 drivers/staging/fsl_qbman/qman_test_high.c
 create mode 100644 drivers/staging/fsl_qbman/qman_test_hotpotato.c
 create mode 100644 drivers/staging/fsl_qbman/qman_utility.c
 create mode 100644 include/linux/fsl_bman.h
 create mode 100644 include/linux/fsl_qman.h
 create mode 100644 include/linux/fsl_usdpaa.h

--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1250,6 +1250,11 @@ source "arch/arm/common/Kconfig"
 
 menu "Bus support"
 
+config HAS_FSL_QBMAN
+	bool "Datapath Acceleration Queue and Buffer management"
+	help
+	  Datapath Acceleration Queue and Buffer management
+
 config ISA
 	bool
 	help
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -792,6 +792,11 @@ config FSL_GTM
 	help
 	  Freescale General-purpose Timers support
 
+config HAS_FSL_QBMAN
+        bool "Datapath Acceleration Queue and Buffer management"
+        help
+          Datapath Acceleration Queue and Buffer management
+
 # Yes MCA RS/6000s exist but Linux-PPC does not currently support any
 config MCA
 	bool
@@ -924,14 +929,14 @@ config DYNAMIC_MEMSTART
 	select NONSTATIC_KERNEL
 	help
 	  This option enables the kernel to be loaded at any page aligned
-	  physical address. The kernel creates a mapping from KERNELBASE to 
+	  physical address. The kernel creates a mapping from KERNELBASE to
 	  the address where the kernel is loaded. The page size here implies
 	  the TLB page size of the mapping for kernel on the particular platform.
 	  Please refer to the init code for finding the TLB page size.
 
 	  DYNAMIC_MEMSTART is an easy way of implementing pseudo-RELOCATABLE
 	  kernel image, where the only restriction is the page aligned kernel
-	  load address. When this option is enabled, the compile time physical 
+	  load address. When this option is enabled, the compile time physical
 	  address CONFIG_PHYSICAL_START is ignored.
 
 	  This option is overridden by CONFIG_RELOCATABLE
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -236,6 +236,23 @@ config SGI_XP
 	  this feature will allow for direct communication between SSIs
 	  based on a network adapter and DMA messaging.
 
+config FSL_USDPAA
+        bool "Freescale USDPAA process driver"
+        depends on FSL_DPA
+        default y
+        help
+          This driver provides user-space access to kernel-managed
+          resource interfaces for USDPAA applications, on the assumption
+          that each process will open this device once. Specifically, this
+          device exposes functionality that would be awkward if exposed
+          via the portal devices - ie. this device exposes functionality
+          that is inherently process-wide rather than portal-specific.
+          This device is necessary for obtaining access to DMA memory and
+          for allocation of Qman and Bman resources. In short, if you wish
+          to use USDPAA applications, you need this.
+
+          If unsure, say Y.
+
 config CS5535_MFGPT
 	tristate "CS5535/CS5536 Geode Multi-Function General Purpose Timer (MFGPT) support"
 	depends on MFD_CS5535
--- a/drivers/staging/Kconfig
+++ b/drivers/staging/Kconfig
@@ -106,6 +106,8 @@ source "drivers/staging/fbtft/Kconfig"
 
 source "drivers/staging/fsl-mc/Kconfig"
 
+source "drivers/staging/fsl_qbman/Kconfig"
+
 source "drivers/staging/wilc1000/Kconfig"
 
 source "drivers/staging/most/Kconfig"
--- a/drivers/staging/Makefile
+++ b/drivers/staging/Makefile
@@ -45,5 +45,6 @@ obj-$(CONFIG_UNISYSSPAR)	+= unisys/
 obj-$(CONFIG_COMMON_CLK_XLNX_CLKWZRD)	+= clocking-wizard/
 obj-$(CONFIG_FB_TFT)		+= fbtft/
 obj-$(CONFIG_FSL_MC_BUS)	+= fsl-mc/
+obj-$(CONFIG_FSL_DPA)           += fsl_qbman/
 obj-$(CONFIG_WILC1000)		+= wilc1000/
 obj-$(CONFIG_MOST)		+= most/
--- /dev/null
+++ b/drivers/staging/fsl_qbman/Kconfig
@@ -0,0 +1,211 @@
+config FSL_DPA
+	bool "Freescale Datapath Queue and Buffer management"
+	depends on HAS_FSL_QBMAN
+	default y
+	select FSL_QMAN_FQ_LOOKUP if PPC64
+	select FSL_QMAN_FQ_LOOKUP if ARM64
+
+
+menu "Freescale Datapath QMan/BMan options"
+	depends on FSL_DPA
+
+config FSL_DPA_CHECKING
+	bool "additional driver checking"
+	default n
+	---help---
+	  Compiles in additional checks to sanity-check the drivers and any
+	  use of it by other code. Not recommended for performance.
+
+config FSL_DPA_CAN_WAIT
+	bool
+	default y
+
+config FSL_DPA_CAN_WAIT_SYNC
+	bool
+	default y
+
+config FSL_DPA_PIRQ_FAST
+	bool
+	default y
+
+config FSL_DPA_PIRQ_SLOW
+	bool
+	default y
+
+config FSL_DPA_PORTAL_SHARE
+	bool
+	default y
+
+config FSL_BMAN
+	bool "Freescale Buffer Manager (BMan) support"
+	default y
+
+if FSL_BMAN
+
+config FSL_BMAN_CONFIG
+	bool "BMan device management"
+	default y
+	---help---
+	  If this linux image is running natively, you need this option. If this
+	  linux image is running as a guest OS under the hypervisor, only one
+	  guest OS ("the control plane") needs this option.
+
+config FSL_BMAN_TEST
+	tristate "BMan self-tests"
+	default n
+	---help---
+	  This option compiles self-test code for BMan.
+
+config FSL_BMAN_TEST_HIGH
+	bool "BMan high-level self-test"
+	depends on FSL_BMAN_TEST
+	default y
+	---help---
+	  This requires the presence of cpu-affine portals, and performs
+	  high-level API testing with them (whichever portal(s) are affine to
+	  the cpu(s) the test executes on).
+
+config FSL_BMAN_TEST_THRESH
+	bool "BMan threshold test"
+	depends on FSL_BMAN_TEST
+	default y
+	---help---
+	  Multi-threaded (SMP) test of BMan pool depletion. A pool is seeded
+	  before multiple threads (one per cpu) create pool objects to track
+	  depletion state changes. The pool is then drained to empty by a
+	  "drainer" thread, and the other threads that they observe exactly
+	  the depletion state changes that are expected.
+
+config FSL_BMAN_DEBUGFS
+	tristate "BMan debugfs interface"
+	depends on DEBUG_FS
+	default y
+	---help---
+	  This option compiles debugfs code for BMan.
+
+endif # FSL_BMAN
+
+config FSL_QMAN
+	bool "Freescale Queue Manager (QMan) support"
+	default y
+
+if FSL_QMAN
+
+config FSL_QMAN_POLL_LIMIT
+	int
+	default 32
+
+config FSL_QMAN_CONFIG
+	bool "QMan device management"
+	default y
+	---help---
+	  If this linux image is running natively, you need this option. If this
+	  linux image is running as a guest OS under the hypervisor, only one
+	  guest OS ("the control plane") needs this option.
+
+config FSL_QMAN_TEST
+	tristate "QMan self-tests"
+	default n
+	---help---
+	  This option compiles self-test code for QMan.
+
+config FSL_QMAN_TEST_STASH_POTATO
+	bool "QMan 'hot potato' data-stashing self-test"
+	depends on FSL_QMAN_TEST
+	default y
+	---help---
+	  This performs a "hot potato" style test enqueuing/dequeuing a frame
+	  across a series of FQs scheduled to different portals (and cpus), with
+	  DQRR, data and context stashing always on.
+
+config FSL_QMAN_TEST_HIGH
+	bool "QMan high-level self-test"
+	depends on FSL_QMAN_TEST
+	default y
+	---help---
+	  This requires the presence of cpu-affine portals, and performs
+	  high-level API testing with them (whichever portal(s) are affine to
+	  the cpu(s) the test executes on).
+
+config FSL_QMAN_DEBUGFS
+	tristate "QMan debugfs interface"
+	depends on DEBUG_FS
+	default y
+	---help---
+	  This option compiles debugfs code for QMan.
+
+# H/w settings that can be hard-coded for now.
+config FSL_QMAN_FQD_SZ
+	int "size of Frame Queue Descriptor region"
+	default 10
+	---help---
+	  This is the size of the FQD region defined as: PAGE_SIZE * (2^value)
+	  ex: 10 => PAGE_SIZE * (2^10)
+	  Note: Default device-trees now require minimum Kconfig setting of 10.
+
+config FSL_QMAN_PFDR_SZ
+	int "size of the PFDR pool"
+	default 13
+	---help---
+	  This is the size of the PFDR pool defined as: PAGE_SIZE * (2^value)
+	  ex: 13 => PAGE_SIZE * (2^13)
+
+# Corenet initiator settings. Stash request queues are 4-deep to match cores'
+# ability to snart. Stash priority is 3, other priorities are 2.
+config FSL_QMAN_CI_SCHED_CFG_SRCCIV
+	int
+	depends on FSL_QMAN_CONFIG
+	default 4
+config FSL_QMAN_CI_SCHED_CFG_SRQ_W
+	int
+	depends on FSL_QMAN_CONFIG
+	default 3
+config FSL_QMAN_CI_SCHED_CFG_RW_W
+	int
+	depends on FSL_QMAN_CONFIG
+	default 2
+config FSL_QMAN_CI_SCHED_CFG_BMAN_W
+	int
+	depends on FSL_QMAN_CONFIG
+	default 2
+
+# portal interrupt settings
+config FSL_QMAN_PIRQ_DQRR_ITHRESH
+	int
+	default 12
+config FSL_QMAN_PIRQ_MR_ITHRESH
+	int
+	default 4
+config FSL_QMAN_PIRQ_IPERIOD
+	int
+	default 100
+
+# 64 bit kernel support
+config FSL_QMAN_FQ_LOOKUP
+	bool
+	default n
+
+config QMAN_CEETM_UPDATE_PERIOD
+	int "Token update period for shaping, in nanoseconds"
+	default 1000
+	 ---help---
+	Traffic shaping works by performing token calculations (using
+	credits) on shaper instances periodically. This update period
+	sets the granularity for how often those token rate credit
+	updates are performed, and thus determines the accuracy and
+	range of traffic rates that can be configured by users. The
+	reference manual recommends a 1 microsecond period as providing
+	a good balance between granularity and range.
+
+	Unless you know what you are doing, leave this value at its default.
+
+config FSL_QMAN_INIT_TIMEOUT
+	int "timeout for qman init stage, in seconds"
+	default 10
+	---help---
+	The timeout setting to quit the initialization loop for non-control
+	partition in case the control partition fails to boot-up.
+
+endif # FSL_QMAN
+
+endmenu
--- /dev/null
+++ b/drivers/staging/fsl_qbman/Makefile
@@ -0,0 +1,28 @@
+subdir-ccflags-y := -Werror
+
+# Common
+obj-$(CONFIG_FSL_DPA)		+= dpa_alloc.o
+obj-$(CONFIG_HAS_FSL_QBMAN)	+= qbman_driver.o
+
+# Bman
+obj-$(CONFIG_FSL_BMAN)		+= bman_high.o
+obj-$(CONFIG_FSL_BMAN_CONFIG)	+= bman_config.o bman_driver.o
+obj-$(CONFIG_FSL_BMAN_TEST)	+= bman_tester.o
+obj-$(CONFIG_FSL_BMAN_DEBUGFS)  += bman_debugfs_interface.o
+bman_tester-y			 = bman_test.o
+bman_tester-$(CONFIG_FSL_BMAN_TEST_HIGH) += bman_test_high.o
+bman_tester-$(CONFIG_FSL_BMAN_TEST_THRESH) += bman_test_thresh.o
+bman_debugfs_interface-y	 = bman_debugfs.o
+
+# Qman
+obj-$(CONFIG_FSL_QMAN)		+= qman_high.o qman_utility.o
+obj-$(CONFIG_FSL_QMAN_CONFIG)	+= qman_config.o qman_driver.o
+obj-$(CONFIG_FSL_QMAN_TEST)	+= qman_tester.o
+qman_tester-y			 = qman_test.o
+qman_tester-$(CONFIG_FSL_QMAN_TEST_STASH_POTATO) += qman_test_hotpotato.o
+qman_tester-$(CONFIG_FSL_QMAN_TEST_HIGH) += qman_test_high.o
+obj-$(CONFIG_FSL_QMAN_DEBUGFS)	+= qman_debugfs_interface.o
+qman_debugfs_interface-y	 = qman_debugfs.o
+
+# USDPAA
+obj-$(CONFIG_FSL_USDPAA)	+= fsl_usdpaa.o fsl_usdpaa_irq.o
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_config.c
@@ -0,0 +1,705 @@
+/* Copyright (c) 2009-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/cacheflush.h>
+#include "bman_private.h"
+#include <linux/of_reserved_mem.h>
+
+/* Last updated for v00.79 of the BG */
+
+struct bman;
+
+/* Register offsets */
+#define REG_POOL_SWDET(n)	(0x0000 + ((n) * 0x04))
+#define REG_POOL_HWDET(n)	(0x0100 + ((n) * 0x04))
+#define REG_POOL_SWDXT(n)	(0x0200 + ((n) * 0x04))
+#define REG_POOL_HWDXT(n)	(0x0300 + ((n) * 0x04))
+#define REG_POOL_CONTENT(n)	(0x0600 + ((n) * 0x04))
+#define REG_FBPR_FPC		0x0800
+#define REG_STATE_IDLE		0x960
+#define REG_STATE_STOP		0x964
+#define REG_ECSR		0x0a00
+#define REG_ECIR		0x0a04
+#define REG_EADR		0x0a08
+#define REG_EDATA(n)		(0x0a10 + ((n) * 0x04))
+#define REG_SBEC(n)		(0x0a80 + ((n) * 0x04))
+#define REG_IP_REV_1		0x0bf8
+#define REG_IP_REV_2		0x0bfc
+#define REG_FBPR_BARE		0x0c00
+#define REG_FBPR_BAR		0x0c04
+#define REG_FBPR_AR		0x0c10
+#define REG_SRCIDR		0x0d04
+#define REG_LIODNR		0x0d08
+#define REG_ERR_ISR		0x0e00	/* + "enum bm_isr_reg" */
+
+/* Used by all error interrupt registers except 'inhibit' */
+#define BM_EIRQ_IVCI	0x00000010	/* Invalid Command Verb */
+#define BM_EIRQ_FLWI	0x00000008	/* FBPR Low Watermark */
+#define BM_EIRQ_MBEI	0x00000004	/* Multi-bit ECC Error */
+#define BM_EIRQ_SBEI	0x00000002	/* Single-bit ECC Error */
+#define BM_EIRQ_BSCN	0x00000001	/* pool State Change Notification */
+
+/* BMAN_ECIR valid error bit */
+#define PORTAL_ECSR_ERR	(BM_EIRQ_IVCI)
+
+union bman_ecir {
+	u32 ecir_raw;
+	struct {
+		u32 __reserved1:4;
+		u32 portal_num:4;
+		u32 __reserved2:12;
+		u32 numb:4;
+		u32 __reserved3:2;
+		u32 pid:6;
+	} __packed info;
+};
+
+union bman_eadr {
+	u32 eadr_raw;
+	struct {
+		u32 __reserved1:5;
+		u32 memid:3;
+		u32 __reserved2:14;
+		u32 eadr:10;
+	} __packed info;
+};
+
+struct bman_hwerr_txt {
+	u32 mask;
+	const char *txt;
+};
+
+#define BMAN_HWE_TXT(a, b) { .mask = BM_EIRQ_##a, .txt = b }
+
+static const struct bman_hwerr_txt bman_hwerr_txts[] = {
+	BMAN_HWE_TXT(IVCI, "Invalid Command Verb"),
+	BMAN_HWE_TXT(FLWI, "FBPR Low Watermark"),
+	BMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"),
+	BMAN_HWE_TXT(SBEI, "Single-bit ECC Error"),
+	BMAN_HWE_TXT(BSCN, "Pool State Change Notification"),
+};
+#define BMAN_HWE_COUNT (sizeof(bman_hwerr_txts)/sizeof(struct bman_hwerr_txt))
+
+struct bman_error_info_mdata {
+	u16 addr_mask;
+	u16 bits;
+	const char *txt;
+};
+
+#define BMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c}
+static const struct bman_error_info_mdata error_mdata[] = {
+	BMAN_ERR_MDATA(0x03FF, 192, "Stockpile memory"),
+	BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 1"),
+	BMAN_ERR_MDATA(0x00FF, 256, "SW portal ring memory port 2"),
+};
+#define BMAN_ERR_MDATA_COUNT \
+	(sizeof(error_mdata)/sizeof(struct bman_error_info_mdata))
+
+/* Add this in Kconfig */
+#define BMAN_ERRS_TO_UNENABLE (BM_EIRQ_FLWI)
+
+/**
+ * bm_err_isr_<reg>_<verb> - Manipulate global interrupt registers
+ * @v: for accessors that write values, this is the 32-bit value
+ *
+ * Manipulates BMAN_ERR_ISR, BMAN_ERR_IER, BMAN_ERR_ISDR, BMAN_ERR_IIR. All
+ * manipulations except bm_err_isr_[un]inhibit() use 32-bit masks composed of
+ * the BM_EIRQ_*** definitions. Note that "bm_err_isr_enable_write" means
+ * "write the enable register" rather than "enable the write register"!
+ */
+#define bm_err_isr_status_read(bm)	\
+		__bm_err_isr_read(bm, bm_isr_status)
+#define bm_err_isr_status_clear(bm, m)	\
+		__bm_err_isr_write(bm, bm_isr_status, m)
+#define bm_err_isr_enable_read(bm)	\
+		__bm_err_isr_read(bm, bm_isr_enable)
+#define bm_err_isr_enable_write(bm, v)	\
+		__bm_err_isr_write(bm, bm_isr_enable, v)
+#define bm_err_isr_disable_read(bm)	\
+		__bm_err_isr_read(bm, bm_isr_disable)
+#define bm_err_isr_disable_write(bm, v)	\
+		__bm_err_isr_write(bm, bm_isr_disable, v)
+#define bm_err_isr_inhibit(bm)		\
+		__bm_err_isr_write(bm, bm_isr_inhibit, 1)
+#define bm_err_isr_uninhibit(bm)	\
+		__bm_err_isr_write(bm, bm_isr_inhibit, 0)
+
+/*
+ * TODO: unimplemented registers
+ *
+ * BMAN_POOLk_SDCNT, BMAN_POOLk_HDCNT, BMAN_FULT,
+ * BMAN_VLDPL, BMAN_EECC, BMAN_SBET, BMAN_EINJ
+ */
+
+/* Encapsulate "struct bman *" as a cast of the register space address. */
+
+static struct bman *bm_create(void *regs)
+{
+	return (struct bman *)regs;
+}
+
+static inline u32 __bm_in(struct bman *bm, u32 offset)
+{
+	return in_be32((void *)bm + offset);
+}
+static inline void __bm_out(struct bman *bm, u32 offset, u32 val)
+{
+	out_be32((void *)bm + offset, val);
+}
+#define bm_in(reg)		__bm_in(bm, REG_##reg)
+#define bm_out(reg, val)	__bm_out(bm, REG_##reg, val)
+
+static u32 __bm_err_isr_read(struct bman *bm, enum bm_isr_reg n)
+{
+	return __bm_in(bm, REG_ERR_ISR + (n << 2));
+}
+
+static void __bm_err_isr_write(struct bman *bm, enum bm_isr_reg n, u32 val)
+{
+	__bm_out(bm, REG_ERR_ISR + (n << 2), val);
+}
+
+static void bm_get_version(struct bman *bm, u16 *id, u8 *major, u8 *minor)
+{
+	u32 v = bm_in(IP_REV_1);
+	*id = (v >> 16);
+	*major = (v >> 8) & 0xff;
+	*minor = v & 0xff;
+}
+
+static u32 __generate_thresh(u32 val, int roundup)
+{
+	u32 e = 0;	/* co-efficient, exponent */
+	int oddbit = 0;
+	while (val > 0xff) {
+		oddbit = val & 1;
+		val >>= 1;
+		e++;
+		if (roundup && oddbit)
+			val++;
+	}
+	DPA_ASSERT(e < 0x10);
+	return val | (e << 8);
+}
+
+static void bm_set_pool(struct bman *bm, u8 pool, u32 swdet, u32 swdxt,
+			u32 hwdet, u32 hwdxt)
+{
+	DPA_ASSERT(pool < bman_pool_max);
+	bm_out(POOL_SWDET(pool), __generate_thresh(swdet, 0));
+	bm_out(POOL_SWDXT(pool), __generate_thresh(swdxt, 1));
+	bm_out(POOL_HWDET(pool), __generate_thresh(hwdet, 0));
+	bm_out(POOL_HWDXT(pool), __generate_thresh(hwdxt, 1));
+}
+
+static void bm_set_memory(struct bman *bm, u64 ba, int prio, u32 size)
+{
+	u32 exp = ilog2(size);
+	/* choke if size isn't within range */
+	DPA_ASSERT((size >= 4096) && (size <= 1073741824) &&
+			is_power_of_2(size));
+	/* choke if '[e]ba' has lower-alignment than 'size' */
+	DPA_ASSERT(!(ba & (size - 1)));
+	bm_out(FBPR_BARE, upper_32_bits(ba));
+	bm_out(FBPR_BAR, lower_32_bits(ba));
+	bm_out(FBPR_AR, (prio ? 0x40000000 : 0) | (exp - 1));
+}
+
+/*****************/
+/* Config driver */
+/*****************/
+
+/* TODO: Kconfig these? */
+#define DEFAULT_FBPR_SZ	(PAGE_SIZE << 12)
+
+/* We support only one of these. */
+static struct bman *bm;
+static struct device_node *bm_node;
+
+/* And this state belongs to 'bm'. It is set during fsl_bman_init(), but used
+ * during bman_init_ccsr(). */
+static dma_addr_t fbpr_a;
+static size_t fbpr_sz = DEFAULT_FBPR_SZ;
+
+static int bman_fbpr(struct reserved_mem *rmem)
+{
+	fbpr_a = rmem->base;
+	fbpr_sz = rmem->size;
+
+	WARN_ON(!(fbpr_a && fbpr_sz));
+
+	return 0;
+}
+RESERVEDMEM_OF_DECLARE(bman_fbpr, "fsl,bman-fbpr", bman_fbpr);
+
+static int __init fsl_bman_init(struct device_node *node)
+{
+	struct resource res;
+	u32 __iomem *regs;
+	const char *s;
+	int ret, standby = 0;
+	u16 id;
+	u8 major, minor;
+
+	ret = of_address_to_resource(node, 0, &res);
+	if (ret) {
+		pr_err("Can't get %s property 'reg'\n",
+				node->full_name);
+		return ret;
+	}
+	s = of_get_property(node, "fsl,hv-claimable", &ret);
+	if (s && !strcmp(s, "standby"))
+		standby = 1;
+	/* Global configuration */
+	regs = ioremap(res.start, res.end - res.start + 1);
+	bm = bm_create(regs);
+	BUG_ON(!bm);
+	bm_node = node;
+	bm_get_version(bm, &id, &major, &minor);
+	pr_info("Bman ver:%04x,%02x,%02x\n", id, major, minor);
+	if ((major == 1) && (minor == 0)) {
+		bman_ip_rev = BMAN_REV10;
+		bman_pool_max = 64;
+	} else if ((major == 2) && (minor == 0)) {
+		bman_ip_rev = BMAN_REV20;
+		bman_pool_max = 8;
+	} else if ((major == 2) && (minor == 1)) {
+		bman_ip_rev = BMAN_REV21;
+		bman_pool_max = 64;
+	} else {
+		pr_warn("unknown Bman version, default to rev1.0\n");
+	}
+
+	if (standby) {
+		pr_info("  -> in standby mode\n");
+		return 0;
+	}
+	return 0;
+}
+
+int bman_have_ccsr(void)
+{
+	return bm ? 1 : 0;
+}
+
+int bm_pool_set(u32 bpid, const u32 *thresholds)
+{
+	if (!bm)
+		return -ENODEV;
+	bm_set_pool(bm, bpid, thresholds[0],
+		    thresholds[1], thresholds[2],
+		    thresholds[3]);
+	return 0;
+}
+EXPORT_SYMBOL(bm_pool_set);
+
+__init int bman_init_early(void)
+{
+	struct device_node *dn;
+	int ret;
+
+	for_each_compatible_node(dn, NULL, "fsl,bman") {
+		if (bm)
+			pr_err("%s: only one 'fsl,bman' allowed\n",
+				dn->full_name);
+		else {
+			if (!of_device_is_available(dn))
+				continue;
+
+			ret = fsl_bman_init(dn);
+			BUG_ON(ret);
+		}
+	}
+	return 0;
+}
+postcore_initcall_sync(bman_init_early);
+
+
+static void log_edata_bits(u32 bit_count)
+{
+	u32 i, j, mask = 0xffffffff;
+
+	pr_warn("Bman ErrInt, EDATA:\n");
+	i = bit_count/32;
+	if (bit_count%32) {
+		i++;
+		mask = ~(mask << bit_count%32);
+	}
+	j = 16-i;
+	pr_warn("  0x%08x\n", bm_in(EDATA(j)) & mask);
+	j++;
+	for (; j < 16; j++)
+		pr_warn("  0x%08x\n", bm_in(EDATA(j)));
+}
+
+static void log_additional_error_info(u32 isr_val, u32 ecsr_val)
+{
+	union bman_ecir ecir_val;
+	union bman_eadr eadr_val;
+
+	ecir_val.ecir_raw = bm_in(ECIR);
+	/* Is portal info valid */
+	if (ecsr_val & PORTAL_ECSR_ERR) {
+		pr_warn("Bman ErrInt: SWP id %d, numb %d, pid %d\n",
+			ecir_val.info.portal_num, ecir_val.info.numb,
+			ecir_val.info.pid);
+	}
+	if (ecsr_val & (BM_EIRQ_SBEI|BM_EIRQ_MBEI)) {
+		eadr_val.eadr_raw = bm_in(EADR);
+		pr_warn("Bman ErrInt: EADR Memory: %s, 0x%x\n",
+			error_mdata[eadr_val.info.memid].txt,
+			error_mdata[eadr_val.info.memid].addr_mask
+				& eadr_val.info.eadr);
+		log_edata_bits(error_mdata[eadr_val.info.memid].bits);
+	}
+}
+
+/* Bman interrupt handler */
+static irqreturn_t bman_isr(int irq, void *ptr)
+{
+	u32 isr_val, ier_val, ecsr_val, isr_mask, i;
+
+	ier_val = bm_err_isr_enable_read(bm);
+	isr_val = bm_err_isr_status_read(bm);
+	ecsr_val = bm_in(ECSR);
+	isr_mask = isr_val & ier_val;
+
+	if (!isr_mask)
+		return IRQ_NONE;
+	for (i = 0; i < BMAN_HWE_COUNT; i++) {
+		if (bman_hwerr_txts[i].mask & isr_mask) {
+			pr_warn("Bman ErrInt: %s\n", bman_hwerr_txts[i].txt);
+			if (bman_hwerr_txts[i].mask & ecsr_val) {
+				log_additional_error_info(isr_mask, ecsr_val);
+				/* Re-arm error capture registers */
+				bm_out(ECSR, ecsr_val);
+			}
+			if (bman_hwerr_txts[i].mask & BMAN_ERRS_TO_UNENABLE) {
+				pr_devel("Bman un-enabling error 0x%x\n",
+					bman_hwerr_txts[i].mask);
+				ier_val &= ~bman_hwerr_txts[i].mask;
+				bm_err_isr_enable_write(bm, ier_val);
+			}
+		}
+	}
+	bm_err_isr_status_clear(bm, isr_val);
+	return IRQ_HANDLED;
+}
+
+static int __bind_irq(void)
+{
+	int ret, err_irq;
+
+	err_irq = of_irq_to_resource(bm_node, 0, NULL);
+	if (err_irq == 0) {
+		pr_info("Can't get %s property '%s'\n", bm_node->full_name,
+			"interrupts");
+		return -ENODEV;
+	}
+	ret = request_irq(err_irq, bman_isr, IRQF_SHARED, "bman-err", bm_node);
+	if (ret)  {
+		pr_err("request_irq() failed %d for '%s'\n", ret,
+			bm_node->full_name);
+		return -ENODEV;
+	}
+	/* Disable Buffer Pool State Change */
+	bm_err_isr_disable_write(bm, BM_EIRQ_BSCN);
+	/* Write-to-clear any stale bits, (eg. starvation being asserted prior
+	 * to resource allocation during driver init). */
+	bm_err_isr_status_clear(bm, 0xffffffff);
+	/* Enable Error Interrupts */
+	bm_err_isr_enable_write(bm, 0xffffffff);
+	return 0;
+}
+
+int bman_init_ccsr(struct device_node *node)
+{
+	int ret;
+	if (!bman_have_ccsr())
+		return 0;
+	if (node != bm_node)
+		return -EINVAL;
+	/* FBPR memory */
+	bm_set_memory(bm, fbpr_a, 0, fbpr_sz);
+	pr_info("bman-fbpr addr 0x%llx size 0x%zx\n",
+			(unsigned long long)fbpr_a, fbpr_sz);
+
+	ret = __bind_irq();
+	if (ret)
+		return ret;
+	return 0;
+}
+
+u32 bm_pool_free_buffers(u32 bpid)
+{
+	return bm_in(POOL_CONTENT(bpid));
+}
+
+#ifdef CONFIG_SYSFS
+
+#define DRV_NAME "fsl-bman"
+#define SBEC_MAX_ID 1
+#define SBEC_MIN_ID 0
+
+static ssize_t show_fbpr_fpc(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(FBPR_FPC));
+};
+
+static ssize_t show_pool_count(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	u32 data;
+	int i;
+
+	if (!sscanf(dev_attr->attr.name, "%d", &i) || (i >= bman_pool_max))
+		return -EINVAL;
+	data = bm_in(POOL_CONTENT(i));
+	return snprintf(buf, PAGE_SIZE, "%d\n", data);
+};
+
+static ssize_t show_err_isr(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "0x%08x\n", bm_in(ERR_ISR));
+};
+
+static ssize_t show_sbec(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	int i;
+
+	if (!sscanf(dev_attr->attr.name, "sbec_%d", &i))
+		return -EINVAL;
+	if (i < SBEC_MIN_ID || i > SBEC_MAX_ID)
+		return -EINVAL;
+	return snprintf(buf, PAGE_SIZE, "%u\n", bm_in(SBEC(i)));
+};
+
+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL);
+static DEVICE_ATTR(fbpr_fpc, S_IRUSR, show_fbpr_fpc, NULL);
+
+/* Didn't use DEVICE_ATTR as 64 of this would be required.
+ * Initialize them when needed. */
+static char *name_attrs_pool_count; /* "xx" + null-terminator */
+static struct device_attribute *dev_attr_buffer_pool_count;
+
+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL);
+
+static struct attribute *bman_dev_attributes[] = {
+	&dev_attr_fbpr_fpc.attr,
+	&dev_attr_err_isr.attr,
+	NULL
+};
+
+static struct attribute *bman_dev_ecr_attributes[] = {
+	&dev_attr_sbec_0.attr,
+	&dev_attr_sbec_1.attr,
+	NULL
+};
+
+static struct attribute **bman_dev_pool_count_attributes;
+
+
+/* root level */
+static const struct attribute_group bman_dev_attr_grp = {
+	.name = NULL,
+	.attrs = bman_dev_attributes
+};
+static const struct attribute_group bman_dev_ecr_grp = {
+	.name = "error_capture",
+	.attrs = bman_dev_ecr_attributes
+};
+static struct attribute_group bman_dev_pool_countent_grp = {
+	.name = "pool_count",
+};
+
+static int of_fsl_bman_remove(struct platform_device *ofdev)
+{
+	sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
+	return 0;
+};
+
+static int of_fsl_bman_probe(struct platform_device *ofdev)
+{
+	int ret, i;
+
+	ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
+	if (ret)
+		goto done;
+	ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_ecr_grp);
+	if (ret)
+		goto del_group_0;
+
+	name_attrs_pool_count = kmalloc(sizeof(char) * bman_pool_max * 3,
+								 GFP_KERNEL);
+	if (!name_attrs_pool_count) {
+		pr_err("Can't alloc name_attrs_pool_count\n");
+		goto del_group_1;
+	}
+
+	dev_attr_buffer_pool_count = kmalloc(sizeof(struct device_attribute) *
+					bman_pool_max, GFP_KERNEL);
+	if (!dev_attr_buffer_pool_count) {
+		pr_err("Can't alloc dev_attr-buffer_pool_count\n");
+		goto del_group_2;
+	}
+
+	bman_dev_pool_count_attributes = kmalloc(sizeof(struct attribute *) *
+			 (bman_pool_max + 1), GFP_KERNEL);
+	if (!bman_dev_pool_count_attributes) {
+			pr_err("can't alloc bman_dev_pool_count_attributes\n");
+			goto del_group_3;
+	}
+
+	for (i = 0; i < bman_pool_max; i++) {
+		ret = scnprintf((name_attrs_pool_count + i * 3), 3, "%d", i);
+		if (!ret)
+			goto del_group_4;
+		dev_attr_buffer_pool_count[i].attr.name =
+			(name_attrs_pool_count + i * 3);
+		dev_attr_buffer_pool_count[i].attr.mode = S_IRUSR;
+		dev_attr_buffer_pool_count[i].show = show_pool_count;
+		bman_dev_pool_count_attributes[i] =
+			&dev_attr_buffer_pool_count[i].attr;
+		sysfs_attr_init(bman_dev_pool_count_attributes[i]);
+	}
+	bman_dev_pool_count_attributes[bman_pool_max] = NULL;
+
+	bman_dev_pool_countent_grp.attrs = bman_dev_pool_count_attributes;
+
+	ret = sysfs_create_group(&ofdev->dev.kobj, &bman_dev_pool_countent_grp);
+	if (ret)
+		goto del_group_4;
+
+	goto done;
+
+del_group_4:
+	kfree(bman_dev_pool_count_attributes);
+del_group_3:
+	kfree(dev_attr_buffer_pool_count);
+del_group_2:
+	kfree(name_attrs_pool_count);
+del_group_1:
+	sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_ecr_grp);
+del_group_0:
+	sysfs_remove_group(&ofdev->dev.kobj, &bman_dev_attr_grp);
+done:
+	if (ret)
+		dev_err(&ofdev->dev,
+			"Cannot create dev attributes ret=%d\n", ret);
+	return ret;
+};
+
+static struct of_device_id of_fsl_bman_ids[] = {
+	{
+		.compatible = "fsl,bman",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, of_fsl_bman_ids);
+
+#ifdef CONFIG_SUSPEND
+static u32 saved_isdr;
+
+static int bman_pm_suspend_noirq(struct device *dev)
+{
+	uint32_t idle_state;
+
+	suspend_unused_bportal();
+	/* save isdr, disable all, clear isr */
+	saved_isdr = bm_err_isr_disable_read(bm);
+	bm_err_isr_disable_write(bm, 0xffffffff);
+	bm_err_isr_status_clear(bm, 0xffffffff);
+
+	if (bman_ip_rev < BMAN_REV21) {
+#ifdef CONFIG_PM_DEBUG
+		pr_info("Bman version doesn't have STATE_IDLE\n");
+#endif
+		return 0;
+	}
+	idle_state = bm_in(STATE_IDLE);
+	if (!(idle_state & 0x1)) {
+		pr_err("Bman not idle 0x%x aborting\n", idle_state);
+		bm_err_isr_disable_write(bm, saved_isdr);
+		resume_unused_bportal();
+		return -EBUSY;
+	}
+#ifdef CONFIG_PM_DEBUG
+	pr_info("Bman suspend code, IDLE_STAT = 0x%x\n", idle_state);
+#endif
+	return 0;
+}
+
+static int bman_pm_resume_noirq(struct device *dev)
+{
+	/* restore isdr */
+	bm_err_isr_disable_write(bm, saved_isdr);
+	resume_unused_bportal();
+	return 0;
+}
+#else
+#define bman_pm_suspend_noirq	NULL
+#define bman_pm_resume_noirq	NULL
+#endif
+
+static const struct dev_pm_ops bman_pm_ops = {
+	.suspend_noirq = bman_pm_suspend_noirq,
+	.resume_noirq = bman_pm_resume_noirq,
+};
+
+static struct platform_driver of_fsl_bman_driver = {
+	.driver = {
+		.owner = THIS_MODULE,
+		.name = DRV_NAME,
+		.of_match_table = of_fsl_bman_ids,
+		.pm = &bman_pm_ops,
+	},
+	.probe = of_fsl_bman_probe,
+	.remove = of_fsl_bman_remove,
+};
+
+static int bman_ctrl_init(void)
+{
+	return platform_driver_register(&of_fsl_bman_driver);
+}
+
+static void bman_ctrl_exit(void)
+{
+	platform_driver_unregister(&of_fsl_bman_driver);
+}
+
+module_init(bman_ctrl_init);
+module_exit(bman_ctrl_exit);
+
+#endif /* CONFIG_SYSFS */
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_debugfs.c
@@ -0,0 +1,119 @@
+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include <linux/module.h>
+#include <linux/fsl_bman.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/uaccess.h>
+
+static struct dentry *dfs_root; /* debugfs root directory */
+
+/*******************************************************************************
+ *  Query Buffer Pool State
+ ******************************************************************************/
+static int query_bp_state_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct bm_pool_state state;
+	int i, j;
+	u32 mask;
+
+	memset(&state, 0, sizeof(struct bm_pool_state));
+	ret = bman_query_pools(&state);
+	if (ret) {
+		seq_printf(file, "Error %d\n", ret);
+		return 0;
+	}
+	seq_puts(file, "bp_id  free_buffers_avail  bp_depleted\n");
+	for (i = 0; i < 2; i++) {
+		mask = 0x80000000;
+		for (j = 0; j < 32; j++) {
+			seq_printf(file,
+			 "  %-2u           %-3s             %-3s\n",
+			 (i*32)+j,
+			 (state.as.state.__state[i] & mask) ? "no" : "yes",
+			 (state.ds.state.__state[i] & mask) ? "yes" : "no");
+			 mask >>= 1;
+		}
+	}
+	return 0;
+}
+
+static int query_bp_state_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, query_bp_state_show, NULL);
+}
+
+static const struct file_operations query_bp_state_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_bp_state_open,
+	.read           = seq_read,
+	.release	= single_release,
+};
+
+static int __init bman_debugfs_module_init(void)
+{
+	int ret = 0;
+	struct dentry *d;
+
+	dfs_root = debugfs_create_dir("bman", NULL);
+
+	if (dfs_root == NULL) {
+		ret = -ENOMEM;
+		pr_err("Cannot create bman debugfs dir\n");
+		goto _return;
+	}
+	d = debugfs_create_file("query_bp_state",
+		S_IRUGO,
+		dfs_root,
+		NULL,
+		&query_bp_state_fops);
+	if (d == NULL) {
+		ret = -ENOMEM;
+		pr_err("Cannot create query_bp_state\n");
+		goto _return;
+	}
+	return 0;
+
+_return:
+	debugfs_remove_recursive(dfs_root);
+	return ret;
+}
+
+static void __exit bman_debugfs_module_exit(void)
+{
+	debugfs_remove_recursive(dfs_root);
+}
+
+
+module_init(bman_debugfs_module_init);
+module_exit(bman_debugfs_module_exit);
+MODULE_LICENSE("Dual BSD/GPL");
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_driver.c
@@ -0,0 +1,574 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *	 notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *	 notice, this list of conditions and the following disclaimer in the
+ *	 documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *	 names of its contributors may be used to endorse or promote products
+ *	 derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "bman_low.h"
+#ifdef CONFIG_HOTPLUG_CPU
+#include <linux/cpu.h>
+#endif
+/*
+ * Global variables of the max portal/pool number this bman version supported
+ */
+u16 bman_ip_rev;
+EXPORT_SYMBOL(bman_ip_rev);
+u16 bman_pool_max;
+EXPORT_SYMBOL(bman_pool_max);
+static u16 bman_portal_max;
+
+/* After initialising cpus that own shared portal configs, we cache the
+ * resulting portals (ie. not just the configs) in this array. Then we
+ * initialise slave cpus that don't have their own portals, redirecting them to
+ * portals from this cache in a round-robin assignment. */
+static struct bman_portal *shared_portals[NR_CPUS];
+static int num_shared_portals;
+static int shared_portals_idx;
+static LIST_HEAD(unused_pcfgs);
+static DEFINE_SPINLOCK(unused_pcfgs_lock);
+static void *affine_bportals[NR_CPUS];
+
+static int __init fsl_bpool_init(struct device_node *node)
+{
+	int ret;
+	u32 *thresh, *bpid = (u32 *)of_get_property(node, "fsl,bpid", &ret);
+	if (!bpid || (ret != 4)) {
+		pr_err("Can't get %s property 'fsl,bpid'\n", node->full_name);
+		return -ENODEV;
+	}
+	thresh = (u32 *)of_get_property(node, "fsl,bpool-thresholds", &ret);
+	if (thresh) {
+		if (ret != 16) {
+			pr_err("Invalid %s property '%s'\n",
+				node->full_name, "fsl,bpool-thresholds");
+			return -ENODEV;
+		}
+	}
+	if (thresh) {
+#ifdef CONFIG_FSL_BMAN_CONFIG
+		ret = bm_pool_set(be32_to_cpu(*bpid), thresh);
+		if (ret)
+			pr_err("No CCSR node for %s property '%s'\n",
+				node->full_name, "fsl,bpool-thresholds");
+		return ret;
+#else
+		pr_err("Ignoring %s property '%s', no CCSR support\n",
+			node->full_name, "fsl,bpool-thresholds");
+#endif
+	}
+	return 0;
+}
+
+static int __init fsl_bpid_range_init(struct device_node *node)
+{
+	int ret;
+	u32 *range = (u32 *)of_get_property(node, "fsl,bpid-range", &ret);
+	if (!range) {
+		pr_err("No 'fsl,bpid-range' property in node %s\n",
+			node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err("'fsl,bpid-range' is not a 2-cell range in node %s\n",
+			node->full_name);
+		return -EINVAL;
+	}
+	bman_seed_bpid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	pr_info("Bman: BPID allocator includes range %d:%d\n",
+		be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	return 0;
+}
+
+static struct bm_portal_config * __init parse_pcfg(struct device_node *node)
+{
+	struct bm_portal_config *pcfg;
+	const u32 *index;
+	int irq, ret;
+	resource_size_t len;
+
+	pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL);
+	if (!pcfg) {
+		pr_err("can't allocate portal config");
+		return NULL;
+	}
+
+	if (of_device_is_compatible(node, "fsl,bman-portal-1.0") ||
+		of_device_is_compatible(node, "fsl,bman-portal-1.0.0")) {
+		bman_ip_rev = BMAN_REV10;
+		bman_pool_max = 64;
+		bman_portal_max = 10;
+	} else if (of_device_is_compatible(node, "fsl,bman-portal-2.0") ||
+		of_device_is_compatible(node, "fsl,bman-portal-2.0.8")) {
+		bman_ip_rev = BMAN_REV20;
+		bman_pool_max = 8;
+		bman_portal_max = 3;
+	} else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.0")) {
+		bman_ip_rev = BMAN_REV21;
+		bman_pool_max = 64;
+		bman_portal_max = 50;
+	} else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.1")) {
+		bman_ip_rev = BMAN_REV21;
+		bman_pool_max = 64;
+		bman_portal_max = 25;
+	} else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.2")) {
+		bman_ip_rev = BMAN_REV21;
+		bman_pool_max = 64;
+		bman_portal_max = 18;
+	} else if (of_device_is_compatible(node, "fsl,bman-portal-2.1.3")) {
+		bman_ip_rev = BMAN_REV21;
+		bman_pool_max = 64;
+		bman_portal_max = 10;
+	} else {
+		pr_warn("unknown BMan version in portal node,"
+			"default to rev1.0\n");
+		bman_ip_rev = BMAN_REV10;
+		bman_pool_max = 64;
+		bman_portal_max = 10;
+	}
+
+	ret = of_address_to_resource(node, DPA_PORTAL_CE,
+				&pcfg->addr_phys[DPA_PORTAL_CE]);
+	if (ret) {
+		pr_err("Can't get %s property 'reg::CE'\n", node->full_name);
+		goto err;
+	}
+	ret = of_address_to_resource(node, DPA_PORTAL_CI,
+				&pcfg->addr_phys[DPA_PORTAL_CI]);
+	if (ret) {
+		pr_err("Can't get %s property 'reg::CI'\n", node->full_name);
+		goto err;
+	}
+
+	index = of_get_property(node, "cell-index", &ret);
+	if (!index || (ret != 4)) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"cell-index");
+		goto err;
+	}
+	if (be32_to_cpu(*index) >= bman_portal_max) {
+		pr_err("BMan portal cell index %d out of range, max %d\n",
+		       be32_to_cpu(*index), bman_portal_max);
+		goto err;
+	}
+
+	pcfg->public_cfg.cpu = -1;
+
+	irq = irq_of_parse_and_map(node, 0);
+	if (irq == 0) {
+		pr_err("Can't get %s property 'interrupts'\n", node->full_name);
+		goto err;
+	}
+	pcfg->public_cfg.irq = irq;
+	pcfg->public_cfg.index = be32_to_cpu(*index);
+	bman_depletion_fill(&pcfg->public_cfg.mask);
+
+	len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]);
+	if (len != (unsigned long)len)
+		goto err;
+
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns(
+                                pcfg->addr_phys[DPA_PORTAL_CE].start,
+                                resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]));
+        pcfg->addr_virt[DPA_PORTAL_CI] = ioremap(
+                                pcfg->addr_phys[DPA_PORTAL_CI].start,
+                                resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]));
+
+#else
+	pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot(
+				pcfg->addr_phys[DPA_PORTAL_CE].start,
+				(unsigned long)len,
+				0);
+	pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot(
+				pcfg->addr_phys[DPA_PORTAL_CI].start,
+				resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]),
+				_PAGE_GUARDED | _PAGE_NO_CACHE);
+#endif
+	/* disable bp depletion */
+	__raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(0));
+	__raw_writel(0x0, pcfg->addr_virt[DPA_PORTAL_CI] + BM_REG_SCN(1));
+	return pcfg;
+err:
+	kfree(pcfg);
+	return NULL;
+}
+
+static struct bm_portal_config *get_pcfg(struct list_head *list)
+{
+	struct bm_portal_config *pcfg;
+	if (list_empty(list))
+		return NULL;
+	pcfg = list_entry(list->prev, struct bm_portal_config, list);
+	list_del(&pcfg->list);
+	return pcfg;
+}
+
+static struct bm_portal_config *get_pcfg_idx(struct list_head *list,
+					     uint32_t idx)
+{
+	struct bm_portal_config *pcfg;
+	if (list_empty(list))
+		return NULL;
+	list_for_each_entry(pcfg, list, list) {
+		if (pcfg->public_cfg.index == idx) {
+			list_del(&pcfg->list);
+			return pcfg;
+		}
+	}
+	return NULL;
+}
+
+struct bm_portal_config *bm_get_unused_portal(void)
+{
+	return bm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX);
+}
+
+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx)
+{
+	struct bm_portal_config *ret;
+	spin_lock(&unused_pcfgs_lock);
+	if (idx == QBMAN_ANY_PORTAL_IDX)
+		ret = get_pcfg(&unused_pcfgs);
+	else
+		ret = get_pcfg_idx(&unused_pcfgs, idx);
+	spin_unlock(&unused_pcfgs_lock);
+	return ret;
+}
+
+void bm_put_unused_portal(struct bm_portal_config *pcfg)
+{
+	spin_lock(&unused_pcfgs_lock);
+	list_add(&pcfg->list, &unused_pcfgs);
+	spin_unlock(&unused_pcfgs_lock);
+}
+
+static struct bman_portal *init_pcfg(struct bm_portal_config *pcfg)
+{
+	struct bman_portal *p;
+	p = bman_create_affine_portal(pcfg);
+	if (p) {
+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW
+		bman_p_irqsource_add(p, BM_PIRQ_RCRI | BM_PIRQ_BSCN);
+#endif
+		pr_info("Bman portal %sinitialised, cpu %d\n",
+			pcfg->public_cfg.is_shared ? "(shared) " : "",
+			pcfg->public_cfg.cpu);
+		affine_bportals[pcfg->public_cfg.cpu] = p;
+	} else
+		pr_crit("Bman portal failure on cpu %d\n",
+			pcfg->public_cfg.cpu);
+	return p;
+}
+
+static void init_slave(int cpu)
+{
+	struct bman_portal *p;
+	p = bman_create_affine_slave(shared_portals[shared_portals_idx++], cpu);
+	if (!p)
+		pr_err("Bman slave portal failure on cpu %d\n", cpu);
+	else
+		pr_info("Bman portal %sinitialised, cpu %d\n", "(slave) ", cpu);
+	if (shared_portals_idx >= num_shared_portals)
+		shared_portals_idx = 0;
+	affine_bportals[cpu] = p;
+}
+
+/* Bootarg "bportals=[...]" has the same syntax as "qportals=", and so the
+ * parsing is in dpa_sys.h. The syntax is a comma-separated list of indexes
+ * and/or ranges of indexes, with each being optionally prefixed by "s" to
+ * explicitly mark it or them for sharing.
+ *    Eg;
+ *	  bportals=s0,1-3,s4
+ * means that cpus 1,2,3 get "unshared" portals, cpus 0 and 4 get "shared"
+ * portals, and any remaining cpus share the portals that are assigned to cpus 0
+ * or 4, selected in a round-robin fashion. (In this example, cpu 5 would share
+ * cpu 0's portal, cpu 6 would share cpu4's portal, and cpu 7 would share cpu
+ * 0's portal.) */
+static struct cpumask want_unshared __initdata; /* cpus requested without "s" */
+static struct cpumask want_shared __initdata; /* cpus requested with "s" */
+
+static int __init parse_bportals(char *str)
+{
+	return parse_portals_bootarg(str, &want_shared, &want_unshared,
+				     "bportals");
+}
+__setup("bportals=", parse_bportals);
+
+static void bman_offline_cpu(unsigned int cpu)
+{
+	struct bman_portal *p;
+	const struct bm_portal_config *pcfg;
+	p = (struct bman_portal *)affine_bportals[cpu];
+	if (p) {
+		pcfg = bman_get_bm_portal_config(p);
+		if (pcfg)
+			irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0));
+	}
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void bman_online_cpu(unsigned int cpu)
+{
+	struct bman_portal *p;
+	const struct bm_portal_config *pcfg;
+	p = (struct bman_portal *)affine_bportals[cpu];
+	if (p) {
+		pcfg = bman_get_bm_portal_config(p);
+		if (pcfg)
+			irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu));
+	}
+}
+
+static int bman_hotplug_cpu_callback(struct notifier_block *nfb,
+					unsigned long action, void *hcpu)
+{
+	unsigned int cpu = (unsigned long)hcpu;
+
+	switch (action) {
+	case CPU_ONLINE:
+	case CPU_ONLINE_FROZEN:
+		bman_online_cpu(cpu);
+		break;
+	case CPU_DOWN_PREPARE:
+	case CPU_DOWN_PREPARE_FROZEN:
+		bman_offline_cpu(cpu);
+	default:
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block bman_hotplug_cpu_notifier = {
+	.notifier_call = bman_hotplug_cpu_callback,
+};
+#endif /* CONFIG_HOTPLUG_CPU */
+
+/* Initialise the Bman driver. The meat of this function deals with portals. The
+ * following describes the flow of portal-handling, the code "steps" refer to
+ * this description;
+ * 1. Portal configs are parsed from the device-tree into 'unused_pcfgs', with
+ *    ::cpu==-1. Regions and interrupts are mapped (but interrupts are not
+ *    bound).
+ * 2. The "want_shared" and "want_unshared" lists (as filled by the
+ *    "bportals=[...]" bootarg) are processed, allocating portals and assigning
+ *    them to cpus, placing them in the relevant list and setting ::cpu as
+ *    appropriate. If no "bportals" bootarg was present, the defaut is to try to
+ *    assign portals to all online cpus at the time of driver initialisation.
+ *    Any failure to allocate portals (when parsing the "want" lists or when
+ *    using default behaviour) will be silently tolerated (the "fixup" logic in
+ *    step 3 will determine what happens in this case).
+ * 3. Do fixups relative to cpu_online_mask(). If no portals are marked for
+ *    sharing and sharing is required (because not all cpus have been assigned
+ *    portals), then one portal will marked for sharing. Conversely if no
+ *    sharing is required, any portals marked for sharing will not be shared. It
+ *    may be that sharing occurs when it wasn't expected, if portal allocation
+ *    failed to honour all the requested assignments (including the default
+ *    assignments if no bootarg is present).
+ * 4. Unshared portals are initialised on their respective cpus.
+ * 5. Shared portals are initialised on their respective cpus.
+ * 6. Each remaining cpu is initialised to slave to one of the shared portals,
+ *    which are selected in a round-robin fashion.
+ * Any portal configs left unused are available for USDPAA allocation.
+ */
+__init int bman_init(void)
+{
+	struct cpumask slave_cpus;
+	struct cpumask unshared_cpus = *cpu_none_mask;
+	struct cpumask shared_cpus = *cpu_none_mask;
+	LIST_HEAD(unshared_pcfgs);
+	LIST_HEAD(shared_pcfgs);
+	struct device_node *dn;
+	struct bm_portal_config *pcfg;
+	struct bman_portal *p;
+	int cpu, ret;
+	struct cpumask offline_cpus;
+
+	/* Initialise the Bman (CCSR) device */
+	for_each_compatible_node(dn, NULL, "fsl,bman") {
+		if (!bman_init_ccsr(dn))
+			pr_info("Bman err interrupt handler present\n");
+		else
+			pr_err("Bman CCSR setup failed\n");
+	}
+	/* Initialise any declared buffer pools */
+	for_each_compatible_node(dn, NULL, "fsl,bpool") {
+		ret = fsl_bpool_init(dn);
+		if (ret)
+			return ret;
+	}
+	/* Step 1. See comments at the beginning of the file. */
+	for_each_compatible_node(dn, NULL, "fsl,bman-portal") {
+		if (!of_device_is_available(dn))
+			continue;
+		pcfg = parse_pcfg(dn);
+		if (pcfg)
+			list_add_tail(&pcfg->list, &unused_pcfgs);
+	}
+	/* Step 2. */
+	for_each_possible_cpu(cpu) {
+		if (cpumask_test_cpu(cpu, &want_shared)) {
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &shared_pcfgs);
+			cpumask_set_cpu(cpu, &shared_cpus);
+		}
+		if (cpumask_test_cpu(cpu, &want_unshared)) {
+			if (cpumask_test_cpu(cpu, &shared_cpus))
+				continue;
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &unshared_pcfgs);
+			cpumask_set_cpu(cpu, &unshared_cpus);
+		}
+	}
+	if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) {
+		/* Default, give an unshared portal to each online cpu */
+		for_each_online_cpu(cpu) {
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &unshared_pcfgs);
+			cpumask_set_cpu(cpu, &unshared_cpus);
+		}
+	}
+	/* Step 3. */
+	cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus);
+	cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus);
+	if (cpumask_empty(&slave_cpus)) {
+		/* No sharing required */
+		if (!list_empty(&shared_pcfgs)) {
+			/* Migrate "shared" to "unshared" */
+			cpumask_or(&unshared_cpus, &unshared_cpus,
+				   &shared_cpus);
+			cpumask_clear(&shared_cpus);
+			list_splice_tail(&shared_pcfgs, &unshared_pcfgs);
+			INIT_LIST_HEAD(&shared_pcfgs);
+		}
+	} else {
+		/* Sharing required */
+		if (list_empty(&shared_pcfgs)) {
+			/* Migrate one "unshared" to "shared" */
+			pcfg = get_pcfg(&unshared_pcfgs);
+			if (!pcfg) {
+				pr_crit("No BMan portals available!\n");
+				return 0;
+			}
+			cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus);
+			cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus);
+			list_add_tail(&pcfg->list, &shared_pcfgs);
+		}
+	}
+	/* Step 4. */
+	list_for_each_entry(pcfg, &unshared_pcfgs, list) {
+		pcfg->public_cfg.is_shared = 0;
+		p = init_pcfg(pcfg);
+		if (!p) {
+			pr_crit("Unable to initialize bman portal\n");
+			return 0;
+		}
+	}
+	/* Step 5. */
+	list_for_each_entry(pcfg, &shared_pcfgs, list) {
+		pcfg->public_cfg.is_shared = 1;
+		p = init_pcfg(pcfg);
+		if (p)
+			shared_portals[num_shared_portals++] = p;
+	}
+	/* Step 6. */
+	if (!cpumask_empty(&slave_cpus))
+		for_each_cpu(cpu, &slave_cpus)
+			init_slave(cpu);
+	pr_info("Bman portals initialised\n");
+	cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask);
+	for_each_cpu(cpu, &offline_cpus)
+		bman_offline_cpu(cpu);
+#ifdef CONFIG_HOTPLUG_CPU
+	register_hotcpu_notifier(&bman_hotplug_cpu_notifier);
+#endif
+	return 0;
+}
+
+__init int bman_resource_init(void)
+{
+	struct device_node *dn;
+	int ret;
+
+	/* Initialise BPID allocation ranges */
+	for_each_compatible_node(dn, NULL, "fsl,bpid-range") {
+		ret = fsl_bpid_range_init(dn);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+#ifdef CONFIG_SUSPEND
+void suspend_unused_bportal(void)
+{
+	struct bm_portal_config *pcfg;
+
+	if (list_empty(&unused_pcfgs))
+		return;
+
+	list_for_each_entry(pcfg, &unused_pcfgs, list) {
+#ifdef CONFIG_PM_DEBUG
+		pr_info("Need to save bportal %d\n", pcfg->public_cfg.index);
+#endif
+		/* save isdr, disable all via isdr, clear isr */
+		pcfg->saved_isdr =
+			__raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
+		__raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
+					0xe08);
+		__raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
+					0xe00);
+	}
+	return;
+}
+
+void resume_unused_bportal(void)
+{
+	struct bm_portal_config *pcfg;
+
+	if (list_empty(&unused_pcfgs))
+		return;
+
+	list_for_each_entry(pcfg, &unused_pcfgs, list) {
+#ifdef CONFIG_PM_DEBUG
+		pr_info("Need to resume bportal %d\n", pcfg->public_cfg.index);
+#endif
+		/* restore isdr */
+		__raw_writel(pcfg->saved_isdr,
+				pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
+	}
+	return;
+}
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_high.c
@@ -0,0 +1,1141 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman_low.h"
+
+/* Compilation constants */
+#define RCR_THRESH	2	/* reread h/w CI when running out of space */
+#define IRQNAME		"BMan portal %d"
+#define MAX_IRQNAME	16	/* big enough for "BMan portal %d" */
+
+struct bman_portal {
+	struct bm_portal p;
+	/* 2-element array. pools[0] is mask, pools[1] is snapshot. */
+	struct bman_depletion *pools;
+	int thresh_set;
+	unsigned long irq_sources;
+	u32 slowpoll;	/* only used when interrupts are off */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	struct bman_pool *rcri_owned; /* only 1 release WAIT_SYNC at a time */
+#endif
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	raw_spinlock_t sharing_lock; /* only used if is_shared */
+	int is_shared;
+	struct bman_portal *sharing_redirect;
+#endif
+	/* When the cpu-affine portal is activated, this is non-NULL */
+	const struct bm_portal_config *config;
+	/* This is needed for power management */
+	struct platform_device *pdev;
+	/* 64-entry hash-table of pool objects that are tracking depletion
+	 * entry/exit (ie. BMAN_POOL_FLAG_DEPLETION). This isn't fast-path, so
+	 * we're not fussy about cache-misses and so forth - whereas the above
+	 * members should all fit in one cacheline.
+	 * BTW, with 64 entries in the hash table and 64 buffer pools to track,
+	 * you'll never guess the hash-function ... */
+	struct bman_pool *cb[64];
+	char irqname[MAX_IRQNAME];
+	/* Track if the portal was alloced by the driver */
+	u8 alloced;
+	/* power management data */
+	u32 save_isdr;
+};
+
+/* For an explanation of the locking, redirection, or affine-portal logic,
+ * please consult the Qman driver for details. This is the same, only simpler
+ * (no fiddly Qman-specific bits.) */
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+#define PORTAL_IRQ_LOCK(p, irqflags) \
+	do { \
+		if ((p)->is_shared) \
+			raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
+		else \
+			local_irq_save(irqflags); \
+	} while (0)
+#define PORTAL_IRQ_UNLOCK(p, irqflags) \
+	do { \
+		if ((p)->is_shared) \
+			raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
+						   irqflags); \
+		else \
+			local_irq_restore(irqflags); \
+	} while (0)
+#else
+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
+#endif
+
+static cpumask_t affine_mask;
+static DEFINE_SPINLOCK(affine_mask_lock);
+static DEFINE_PER_CPU(struct bman_portal, bman_affine_portal);
+static inline struct bman_portal *get_raw_affine_portal(void)
+{
+	return &get_cpu_var(bman_affine_portal);
+}
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+static inline struct bman_portal *get_affine_portal(void)
+{
+	struct bman_portal *p = get_raw_affine_portal();
+	if (p->sharing_redirect)
+		return p->sharing_redirect;
+	return p;
+}
+#else
+#define get_affine_portal() get_raw_affine_portal()
+#endif
+static inline void put_affine_portal(void)
+{
+	put_cpu_var(bman_affine_portal);
+}
+static inline struct bman_portal *get_poll_portal(void)
+{
+	return &get_cpu_var(bman_affine_portal);
+}
+#define put_poll_portal()
+
+/* GOTCHA: this object type refers to a pool, it isn't *the* pool. There may be
+ * more than one such object per Bman buffer pool, eg. if different users of the
+ * pool are operating via different portals. */
+struct bman_pool {
+	struct bman_pool_params params;
+	/* Used for hash-table admin when using depletion notifications. */
+	struct bman_portal *portal;
+	struct bman_pool *next;
+	/* stockpile state - NULL unless BMAN_POOL_FLAG_STOCKPILE is set */
+	struct bm_buffer *sp;
+	unsigned int sp_fill;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	atomic_t in_use;
+#endif
+};
+
+/* (De)Registration of depletion notification callbacks */
+static void depletion_link(struct bman_portal *portal, struct bman_pool *pool)
+{
+	__maybe_unused unsigned long irqflags;
+	pool->portal = portal;
+	PORTAL_IRQ_LOCK(portal, irqflags);
+	pool->next = portal->cb[pool->params.bpid];
+	portal->cb[pool->params.bpid] = pool;
+	if (!pool->next)
+		/* First object for that bpid on this portal, enable the BSCN
+		 * mask bit. */
+		bm_isr_bscn_mask(&portal->p, pool->params.bpid, 1);
+	PORTAL_IRQ_UNLOCK(portal, irqflags);
+}
+static void depletion_unlink(struct bman_pool *pool)
+{
+	struct bman_pool *it, *last = NULL;
+	struct bman_pool **base = &pool->portal->cb[pool->params.bpid];
+	__maybe_unused unsigned long irqflags;
+	PORTAL_IRQ_LOCK(pool->portal, irqflags);
+	it = *base;	/* <-- gotcha, don't do this prior to the irq_save */
+	while (it != pool) {
+		last = it;
+		it = it->next;
+	}
+	if (!last)
+		*base = pool->next;
+	else
+		last->next = pool->next;
+	if (!last && !pool->next) {
+		/* Last object for that bpid on this portal, disable the BSCN
+		 * mask bit. */
+		bm_isr_bscn_mask(&pool->portal->p, pool->params.bpid, 0);
+		/* And "forget" that we last saw this pool as depleted */
+		bman_depletion_unset(&pool->portal->pools[1],
+					pool->params.bpid);
+	}
+	PORTAL_IRQ_UNLOCK(pool->portal, irqflags);
+}
+
+/* In the case that the application's core loop calls qman_poll() and
+ * bman_poll(), we ought to balance how often we incur the overheads of the
+ * slow-path poll. We'll use two decrementer sources. The idle decrementer
+ * constant is used when the last slow-poll detected no work to do, and the busy
+ * decrementer constant when the last slow-poll had work to do. */
+#define SLOW_POLL_IDLE 1000
+#define SLOW_POLL_BUSY 10
+static u32 __poll_portal_slow(struct bman_portal *p, u32 is);
+
+/* Portal interrupt handler */
+static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
+{
+	struct bman_portal *p = ptr;
+	u32 clear = p->irq_sources;
+	u32 is = bm_isr_status_read(&p->p) & p->irq_sources;
+	clear |= __poll_portal_slow(p, is);
+	bm_isr_status_clear(&p->p, clear);
+	return IRQ_HANDLED;
+}
+
+#ifdef CONFIG_SUSPEND
+static int _bman_portal_suspend_noirq(struct device *dev)
+{
+	struct bman_portal *p = (struct bman_portal *)dev->platform_data;
+#ifdef CONFIG_PM_DEBUG
+	struct platform_device *pdev = to_platform_device(dev);
+#endif
+	p->save_isdr = bm_isr_disable_read(&p->p);
+	bm_isr_disable_write(&p->p, 0xffffffff);
+	bm_isr_status_clear(&p->p, 0xffffffff);
+#ifdef CONFIG_PM_DEBUG
+	pr_info("Suspend for %s\n", pdev->name);
+#endif
+	return 0;
+}
+
+static int _bman_portal_resume_noirq(struct device *dev)
+{
+	struct bman_portal *p = (struct bman_portal *)dev->platform_data;
+
+	/* restore isdr */
+	bm_isr_disable_write(&p->p, p->save_isdr);
+	return 0;
+}
+#else
+#define _bman_portal_suspend_noirq NULL
+#define _bman_portal_resume_noirq NULL
+#endif
+
+struct dev_pm_domain bman_portal_device_pm_domain = {
+	.ops = {
+		USE_PLATFORM_PM_SLEEP_OPS
+		.suspend_noirq = _bman_portal_suspend_noirq,
+		.resume_noirq = _bman_portal_resume_noirq,
+	}
+};
+
+struct bman_portal *bman_create_portal(
+				       struct bman_portal *portal,
+				       const struct bm_portal_config *config)
+{
+	struct bm_portal *__p;
+	const struct bman_depletion *pools = &config->public_cfg.mask;
+	int ret;
+	u8 bpid = 0;
+	char buf[16];
+
+	if (!portal) {
+		portal = kmalloc(sizeof(*portal), GFP_KERNEL);
+		if (!portal)
+			return portal;
+		portal->alloced = 1;
+	} else
+		portal->alloced = 0;
+
+	__p = &portal->p;
+
+	/* prep the low-level portal struct with the mapped addresses from the
+	 * config, everything that follows depends on it and "config" is more
+	 * for (de)reference... */
+	__p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
+	__p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
+	if (bm_rcr_init(__p, bm_rcr_pvb, bm_rcr_cce)) {
+		pr_err("Bman RCR initialisation failed\n");
+		goto fail_rcr;
+	}
+	if (bm_mc_init(__p)) {
+		pr_err("Bman MC initialisation failed\n");
+		goto fail_mc;
+	}
+	if (bm_isr_init(__p)) {
+		pr_err("Bman ISR initialisation failed\n");
+		goto fail_isr;
+	}
+	portal->pools = kmalloc(2 * sizeof(*pools), GFP_KERNEL);
+	if (!portal->pools)
+		goto fail_pools;
+	portal->pools[0] = *pools;
+	bman_depletion_init(portal->pools + 1);
+	while (bpid < bman_pool_max) {
+		/* Default to all BPIDs disabled, we enable as required at
+		 * run-time. */
+		bm_isr_bscn_mask(__p, bpid, 0);
+		bpid++;
+	}
+	portal->slowpoll = 0;
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	portal->rcri_owned = NULL;
+#endif
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	raw_spin_lock_init(&portal->sharing_lock);
+	portal->is_shared = config->public_cfg.is_shared;
+	portal->sharing_redirect = NULL;
+#endif
+	sprintf(buf, "bportal-%u", config->public_cfg.index);
+	portal->pdev = platform_device_alloc(buf, -1);
+	if (!portal->pdev)
+		goto fail_devalloc;
+	portal->pdev->dev.pm_domain = &bman_portal_device_pm_domain;
+	portal->pdev->dev.platform_data = portal;
+	ret = platform_device_add(portal->pdev);
+	if (ret)
+		goto fail_devadd;
+	memset(&portal->cb, 0, sizeof(portal->cb));
+	/* Write-to-clear any stale interrupt status bits */
+	bm_isr_disable_write(__p, 0xffffffff);
+	portal->irq_sources = 0;
+	bm_isr_enable_write(__p, portal->irq_sources);
+	bm_isr_status_clear(__p, 0xffffffff);
+	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
+	if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
+			portal)) {
+		pr_err("request_irq() failed\n");
+		goto fail_irq;
+	}
+	if ((config->public_cfg.cpu != -1) &&
+			irq_can_set_affinity(config->public_cfg.irq) &&
+			irq_set_affinity(config->public_cfg.irq,
+				cpumask_of(config->public_cfg.cpu))) {
+		pr_err("irq_set_affinity() failed %s\n", portal->irqname);
+		goto fail_affinity;
+	}
+
+	/* Need RCR to be empty before continuing */
+	ret = bm_rcr_get_fill(__p);
+	if (ret) {
+		pr_err("Bman RCR unclean\n");
+		goto fail_rcr_empty;
+	}
+	/* Success */
+	portal->config = config;
+
+	bm_isr_disable_write(__p, 0);
+	bm_isr_uninhibit(__p);
+	return portal;
+fail_rcr_empty:
+fail_affinity:
+	free_irq(config->public_cfg.irq, portal);
+fail_irq:
+	platform_device_del(portal->pdev);
+fail_devadd:
+	platform_device_put(portal->pdev);
+fail_devalloc:
+	kfree(portal->pools);
+fail_pools:
+	bm_isr_finish(__p);
+fail_isr:
+	bm_mc_finish(__p);
+fail_mc:
+	bm_rcr_finish(__p);
+fail_rcr:
+	if (portal->alloced)
+		kfree(portal);
+	return NULL;
+}
+
+struct bman_portal *bman_create_affine_portal(
+			const struct bm_portal_config *config)
+{
+	struct bman_portal *portal;
+
+	portal = &per_cpu(bman_affine_portal, config->public_cfg.cpu);
+	portal = bman_create_portal(portal, config);
+	if (portal) {
+		spin_lock(&affine_mask_lock);
+		cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
+		spin_unlock(&affine_mask_lock);
+	}
+	return portal;
+}
+
+
+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
+								int cpu)
+{
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	struct bman_portal *p;
+	p = &per_cpu(bman_affine_portal, cpu);
+	BUG_ON(p->config);
+	BUG_ON(p->is_shared);
+	BUG_ON(!redirect->config->public_cfg.is_shared);
+	p->irq_sources = 0;
+	p->sharing_redirect = redirect;
+	return p;
+#else
+	BUG();
+	return NULL;
+#endif
+}
+
+void bman_destroy_portal(struct bman_portal *bm)
+{
+	const struct bm_portal_config *pcfg;
+	pcfg = bm->config;
+	bm_rcr_cce_update(&bm->p);
+	bm_rcr_cce_update(&bm->p);
+
+	free_irq(pcfg->public_cfg.irq, bm);
+
+	kfree(bm->pools);
+	bm_isr_finish(&bm->p);
+	bm_mc_finish(&bm->p);
+	bm_rcr_finish(&bm->p);
+	bm->config = NULL;
+	if (bm->alloced)
+		kfree(bm);
+}
+
+const struct bm_portal_config *bman_destroy_affine_portal(void)
+{
+	struct bman_portal *bm = get_raw_affine_portal();
+	const struct bm_portal_config *pcfg;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (bm->sharing_redirect) {
+		bm->sharing_redirect = NULL;
+		put_affine_portal();
+		return NULL;
+	}
+	bm->is_shared = 0;
+#endif
+	pcfg = bm->config;
+	bman_destroy_portal(bm);
+	spin_lock(&affine_mask_lock);
+	cpumask_clear_cpu(pcfg->public_cfg.cpu, &affine_mask);
+	spin_unlock(&affine_mask_lock);
+	put_affine_portal();
+	return pcfg;
+}
+
+/* When release logic waits on available RCR space, we need a global waitqueue
+ * in the case of "affine" use (as the waits wake on different cpus which means
+ * different portals - so we can't wait on any per-portal waitqueue). */
+static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
+
+static u32 __poll_portal_slow(struct bman_portal *p, u32 is)
+{
+	struct bman_depletion tmp;
+	u32 ret = is;
+
+	/* There is a gotcha to be aware of. If we do the query before clearing
+	 * the status register, we may miss state changes that occur between the
+	 * two. If we write to clear the status register before the query, the
+	 * cache-enabled query command may overtake the status register write
+	 * unless we use a heavyweight sync (which we don't want). Instead, we
+	 * write-to-clear the status register then *read it back* before doing
+	 * the query, hence the odd while loop with the 'is' accumulation. */
+	if (is & BM_PIRQ_BSCN) {
+		struct bm_mc_result *mcr;
+		__maybe_unused unsigned long irqflags;
+		unsigned int i, j;
+		u32 __is;
+		bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
+		while ((__is = bm_isr_status_read(&p->p)) & BM_PIRQ_BSCN) {
+			is |= __is;
+			bm_isr_status_clear(&p->p, BM_PIRQ_BSCN);
+		}
+		is &= ~BM_PIRQ_BSCN;
+		PORTAL_IRQ_LOCK(p, irqflags);
+		bm_mc_start(&p->p);
+		bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
+		while (!(mcr = bm_mc_result(&p->p)))
+			cpu_relax();
+		tmp = mcr->query.ds.state;
+		tmp.__state[0] = be32_to_cpu(tmp.__state[0]);
+		tmp.__state[1] = be32_to_cpu(tmp.__state[1]);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		for (i = 0; i < 2; i++) {
+			int idx = i * 32;
+			/* tmp is a mask of currently-depleted pools.
+			 * pools[0] is mask of those we care about.
+			 * pools[1] is our previous view (we only want to
+			 * be told about changes). */
+			tmp.__state[i] &= p->pools[0].__state[i];
+			if (tmp.__state[i] == p->pools[1].__state[i])
+				/* fast-path, nothing to see, move along */
+				continue;
+			for (j = 0; j <= 31; j++, idx++) {
+				struct bman_pool *pool = p->cb[idx];
+				int b4 = bman_depletion_get(&p->pools[1], idx);
+				int af = bman_depletion_get(&tmp, idx);
+				if (b4 == af)
+					continue;
+				while (pool) {
+					pool->params.cb(p, pool,
+						pool->params.cb_ctx, af);
+					pool = pool->next;
+				}
+			}
+		}
+		p->pools[1] = tmp;
+	}
+
+	if (is & BM_PIRQ_RCRI) {
+		__maybe_unused unsigned long irqflags;
+		PORTAL_IRQ_LOCK(p, irqflags);
+		bm_rcr_cce_update(&p->p);
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+		/* If waiting for sync, we only cancel the interrupt threshold
+		 * when the ring utilisation hits zero. */
+		if (p->rcri_owned) {
+			if (!bm_rcr_get_fill(&p->p)) {
+				p->rcri_owned = NULL;
+				bm_rcr_set_ithresh(&p->p, 0);
+			}
+		} else
+#endif
+		bm_rcr_set_ithresh(&p->p, 0);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		wake_up(&affine_queue);
+		bm_isr_status_clear(&p->p, BM_PIRQ_RCRI);
+		is &= ~BM_PIRQ_RCRI;
+	}
+
+	/* There should be no status register bits left undefined */
+	DPA_ASSERT(!is);
+	return ret;
+}
+
+const struct bman_portal_config *bman_get_portal_config(void)
+{
+	struct bman_portal *p = get_affine_portal();
+	const struct bman_portal_config *ret = &p->config->public_cfg;
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(bman_get_portal_config);
+
+u32 bman_irqsource_get(void)
+{
+	struct bman_portal *p = get_raw_affine_portal();
+	u32 ret = p->irq_sources & BM_PIRQ_VISIBLE;
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(bman_irqsource_get);
+
+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits)
+{
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (p->sharing_redirect)
+		return -EINVAL;
+	else
+#endif
+	{
+		__maybe_unused unsigned long irqflags;
+		PORTAL_IRQ_LOCK(p, irqflags);
+		set_bits(bits & BM_PIRQ_VISIBLE, &p->irq_sources);
+		bm_isr_enable_write(&p->p, p->irq_sources);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(bman_p_irqsource_add);
+
+int bman_irqsource_add(__maybe_unused u32 bits)
+{
+	struct bman_portal *p = get_raw_affine_portal();
+	int ret = 0;
+	ret = bman_p_irqsource_add(p, bits);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(bman_irqsource_add);
+
+int bman_irqsource_remove(u32 bits)
+{
+	struct bman_portal *p = get_raw_affine_portal();
+	__maybe_unused unsigned long irqflags;
+	u32 ier;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (p->sharing_redirect) {
+		put_affine_portal();
+		return -EINVAL;
+	}
+#endif
+	/* Our interrupt handler only processes+clears status register bits that
+	 * are in p->irq_sources. As we're trimming that mask, if one of them
+	 * were to assert in the status register just before we remove it from
+	 * the enable register, there would be an interrupt-storm when we
+	 * release the IRQ lock. So we wait for the enable register update to
+	 * take effect in h/w (by reading it back) and then clear all other bits
+	 * in the status register. Ie. we clear them from ISR once it's certain
+	 * IER won't allow them to reassert. */
+	PORTAL_IRQ_LOCK(p, irqflags);
+	bits &= BM_PIRQ_VISIBLE;
+	clear_bits(bits, &p->irq_sources);
+	bm_isr_enable_write(&p->p, p->irq_sources);
+	ier = bm_isr_enable_read(&p->p);
+	/* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
+	 * data-dependency, ie. to protect against re-ordering. */
+	bm_isr_status_clear(&p->p, ~ier);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return 0;
+}
+EXPORT_SYMBOL(bman_irqsource_remove);
+
+const cpumask_t *bman_affine_cpus(void)
+{
+	return &affine_mask;
+}
+EXPORT_SYMBOL(bman_affine_cpus);
+
+u32 bman_poll_slow(void)
+{
+	struct bman_portal *p = get_poll_portal();
+	u32 ret;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (unlikely(p->sharing_redirect))
+		ret = (u32)-1;
+	else
+#endif
+	{
+		u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
+		ret = __poll_portal_slow(p, is);
+		bm_isr_status_clear(&p->p, ret);
+	}
+	put_poll_portal();
+	return ret;
+}
+EXPORT_SYMBOL(bman_poll_slow);
+
+/* Legacy wrapper */
+void bman_poll(void)
+{
+	struct bman_portal *p = get_poll_portal();
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (unlikely(p->sharing_redirect))
+		goto done;
+#endif
+	if (!(p->slowpoll--)) {
+		u32 is = bm_isr_status_read(&p->p) & ~p->irq_sources;
+		u32 active = __poll_portal_slow(p, is);
+		if (active)
+			p->slowpoll = SLOW_POLL_BUSY;
+		else
+			p->slowpoll = SLOW_POLL_IDLE;
+	}
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+done:
+#endif
+	put_poll_portal();
+}
+EXPORT_SYMBOL(bman_poll);
+
+static const u32 zero_thresholds[4] = {0, 0, 0, 0};
+
+struct bman_pool *bman_new_pool(const struct bman_pool_params *params)
+{
+	struct bman_pool *pool = NULL;
+	u32 bpid;
+
+	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID) {
+		int ret = bman_alloc_bpid(&bpid);
+		if (ret)
+			return NULL;
+	} else {
+		if (params->bpid >= bman_pool_max)
+			return NULL;
+		bpid = params->bpid;
+	}
+#ifdef CONFIG_FSL_BMAN_CONFIG
+	if (params->flags & BMAN_POOL_FLAG_THRESH) {
+		int ret = bm_pool_set(bpid, params->thresholds);
+		if (ret)
+			goto err;
+	}
+#else
+	if (params->flags & BMAN_POOL_FLAG_THRESH)
+		goto err;
+#endif
+	pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+	if (!pool)
+		goto err;
+	pool->sp = NULL;
+	pool->sp_fill = 0;
+	pool->params = *params;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	atomic_set(&pool->in_use, 1);
+#endif
+	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+		pool->params.bpid = bpid;
+	if (params->flags & BMAN_POOL_FLAG_STOCKPILE) {
+		pool->sp = kmalloc(sizeof(struct bm_buffer) * BMAN_STOCKPILE_SZ,
+					GFP_KERNEL);
+		if (!pool->sp)
+			goto err;
+	}
+	if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION) {
+		struct bman_portal *p = get_affine_portal();
+		if (!p->pools || !bman_depletion_get(&p->pools[0], bpid)) {
+			pr_err("Depletion events disabled for bpid %d\n", bpid);
+			goto err;
+		}
+		depletion_link(p, pool);
+		put_affine_portal();
+	}
+	return pool;
+err:
+#ifdef CONFIG_FSL_BMAN_CONFIG
+	if (params->flags & BMAN_POOL_FLAG_THRESH)
+		bm_pool_set(bpid, zero_thresholds);
+#endif
+	if (params->flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+		bman_release_bpid(bpid);
+	if (pool) {
+		kfree(pool->sp);
+		kfree(pool);
+	}
+	return NULL;
+}
+EXPORT_SYMBOL(bman_new_pool);
+
+void bman_free_pool(struct bman_pool *pool)
+{
+#ifdef CONFIG_FSL_BMAN_CONFIG
+	if (pool->params.flags & BMAN_POOL_FLAG_THRESH)
+		bm_pool_set(pool->params.bpid, zero_thresholds);
+#endif
+	if (pool->params.flags & BMAN_POOL_FLAG_DEPLETION)
+		depletion_unlink(pool);
+	if (pool->params.flags & BMAN_POOL_FLAG_STOCKPILE) {
+		if (pool->sp_fill)
+			pr_err("Stockpile not flushed, has %u in bpid %u.\n",
+				pool->sp_fill, pool->params.bpid);
+		kfree(pool->sp);
+		pool->sp = NULL;
+		pool->params.flags ^= BMAN_POOL_FLAG_STOCKPILE;
+	}
+	if (pool->params.flags & BMAN_POOL_FLAG_DYNAMIC_BPID)
+		bman_release_bpid(pool->params.bpid);
+	kfree(pool);
+}
+EXPORT_SYMBOL(bman_free_pool);
+
+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool)
+{
+	return &pool->params;
+}
+EXPORT_SYMBOL(bman_get_params);
+
+static noinline void update_rcr_ci(struct bman_portal *p, u8 avail)
+{
+	if (avail)
+		bm_rcr_cce_prefetch(&p->p);
+	else
+		bm_rcr_cce_update(&p->p);
+}
+
+int bman_rcr_is_empty(void)
+{
+	__maybe_unused unsigned long irqflags;
+	struct bman_portal *p = get_affine_portal();
+	u8 avail;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	update_rcr_ci(p, 0);
+	avail = bm_rcr_get_fill(&p->p);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return avail == 0;
+}
+EXPORT_SYMBOL(bman_rcr_is_empty);
+
+static inline struct bm_rcr_entry *try_rel_start(struct bman_portal **p,
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+					__maybe_unused struct bman_pool *pool,
+#endif
+					__maybe_unused unsigned long *irqflags,
+					__maybe_unused u32 flags)
+{
+	struct bm_rcr_entry *r;
+	u8 avail;
+
+	*p = get_affine_portal();
+	PORTAL_IRQ_LOCK(*p, (*irqflags));
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
+			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
+		if ((*p)->rcri_owned) {
+			PORTAL_IRQ_UNLOCK(*p, (*irqflags));
+			put_affine_portal();
+			return NULL;
+		}
+		(*p)->rcri_owned = pool;
+	}
+#endif
+	avail = bm_rcr_get_avail(&(*p)->p);
+	if (avail < 2)
+		update_rcr_ci(*p, avail);
+	r = bm_rcr_start(&(*p)->p);
+	if (unlikely(!r)) {
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+		if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
+				(flags & BMAN_RELEASE_FLAG_WAIT_SYNC)))
+			(*p)->rcri_owned = NULL;
+#endif
+		PORTAL_IRQ_UNLOCK(*p, (*irqflags));
+		put_affine_portal();
+	}
+	return r;
+}
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+static noinline struct bm_rcr_entry *__wait_rel_start(struct bman_portal **p,
+					struct bman_pool *pool,
+					__maybe_unused unsigned long *irqflags,
+					u32 flags)
+{
+	struct bm_rcr_entry *rcr = try_rel_start(p, pool, irqflags, flags);
+	if (!rcr)
+		bm_rcr_set_ithresh(&(*p)->p, 1);
+	return rcr;
+}
+
+static noinline struct bm_rcr_entry *wait_rel_start(struct bman_portal **p,
+					struct bman_pool *pool,
+					__maybe_unused unsigned long *irqflags,
+					u32 flags)
+{
+	struct bm_rcr_entry *rcr;
+#ifndef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	pool = NULL;
+#endif
+	if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
+		/* NB: return NULL if signal occurs before completion. Signal
+		 * can occur during return. Caller must check for signal */
+		wait_event_interruptible(affine_queue,
+			(rcr = __wait_rel_start(p, pool, irqflags, flags)));
+	else
+		wait_event(affine_queue,
+			(rcr = __wait_rel_start(p, pool, irqflags, flags)));
+	return rcr;
+}
+#endif
+
+static inline int __bman_release(struct bman_pool *pool,
+			const struct bm_buffer *bufs, u8 num, u32 flags)
+{
+	struct bman_portal *p;
+	struct bm_rcr_entry *r;
+	__maybe_unused unsigned long irqflags;
+	u32 i = num - 1;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & BMAN_RELEASE_FLAG_WAIT)
+		r = wait_rel_start(&p, pool, &irqflags, flags);
+	else
+		r = try_rel_start(&p, pool, &irqflags, flags);
+#else
+	r = try_rel_start(&p, &irqflags, flags);
+#endif
+	if (!r)
+		return -EBUSY;
+	/* We can copy all but the first entry, as this can trigger badness
+	 * with the valid-bit. Use the overlay to mask the verb byte. */
+	r->bufs[0].opaque =
+		((cpu_to_be64((bufs[0].opaque |
+			      ((u64)pool->params.bpid<<48))
+			      & 0x00ffffffffffffff)));
+	if (i) {
+		for (i = 1; i < num; i++)
+			r->bufs[i].opaque =
+				cpu_to_be64(bufs[i].opaque);
+	}
+
+	bm_rcr_pvb_commit(&p->p, BM_RCR_VERB_CMD_BPID_SINGLE |
+			(num & BM_RCR_VERB_BUFCOUNT_MASK));
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	/* if we wish to sync we need to set the threshold after h/w sees the
+	 * new ring entry. As we're mixing cache-enabled and cache-inhibited
+	 * accesses, this requires a heavy-weight sync. */
+	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
+			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
+		hwsync();
+		bm_rcr_set_ithresh(&p->p, 1);
+	}
+#endif
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & BMAN_RELEASE_FLAG_WAIT) &&
+			(flags & BMAN_RELEASE_FLAG_WAIT_SYNC))) {
+		if (flags & BMAN_RELEASE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->rcri_owned != pool));
+		else
+			wait_event(affine_queue, (p->rcri_owned != pool));
+	}
+#endif
+	return 0;
+}
+
+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
+			u32 flags)
+{
+	int ret;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (!num || (num > 8))
+		return -EINVAL;
+	if (pool->params.flags & BMAN_POOL_FLAG_NO_RELEASE)
+		return -EINVAL;
+#endif
+	/* Without stockpile, this API is a pass-through to the h/w operation */
+	if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
+		return __bman_release(pool, bufs, num, flags);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (!atomic_dec_and_test(&pool->in_use)) {
+		pr_crit("Parallel attempts to enter bman_released() detected.");
+		panic("only one instance of bman_released/acquired allowed");
+	}
+#endif
+	/* Two movements of buffers are possible, and can occur in either order.
+	 * A: moving buffers from the caller to the stockpile.
+	 * B: moving buffers from the stockpile to hardware.
+	 * Order 1: if there is already enough space in the stockpile for A
+	 * then we want to do A first, and only do B if we trigger the
+	 * stockpile-high threshold.
+	 * Order 2: if there is not enough space in the stockpile for A, then
+	 * we want to do B first, then do A if B had succeeded. However in this
+	 * case B is dependent on how many buffers the user needs to release,
+	 * not the stockpile-high threshold.
+	 * Due to the different handling of B between the two cases, putting A
+	 * and B in a while() loop would require quite obscure logic, so handle
+	 * the different sequences explicitly. */
+	if ((pool->sp_fill + num) <= BMAN_STOCKPILE_SZ) {
+		/* Order 1: do A */
+		copy_words(pool->sp + pool->sp_fill, bufs,
+			   sizeof(struct bm_buffer) * num);
+		pool->sp_fill += num;
+		/* do B relative to STOCKPILE_HIGH */
+		while (pool->sp_fill >= BMAN_STOCKPILE_HIGH) {
+			ret = __bman_release(pool,
+					     pool->sp + (pool->sp_fill - 8), 8,
+					     flags);
+			if (ret >= 0)
+				pool->sp_fill -= 8;
+		}
+	} else {
+		/* Order 2: do B relative to 'num' */
+		do {
+			ret = __bman_release(pool,
+					     pool->sp + (pool->sp_fill - 8), 8,
+					     flags);
+			if (ret < 0)
+				/* failure */
+				goto release_done;
+			pool->sp_fill -= 8;
+		} while ((pool->sp_fill + num) > BMAN_STOCKPILE_SZ);
+		/* do A */
+		copy_words(pool->sp + pool->sp_fill, bufs,
+			   sizeof(struct bm_buffer) * num);
+		pool->sp_fill += num;
+	}
+	/* success */
+	ret = 0;
+release_done:
+#ifdef CONFIG_FSL_DPA_CHECKING
+	atomic_inc(&pool->in_use);
+#endif
+	return ret;
+}
+EXPORT_SYMBOL(bman_release);
+
+static inline int __bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs,
+					u8 num)
+{
+	struct bman_portal *p = get_affine_portal();
+	struct bm_mc_command *mcc;
+	struct bm_mc_result *mcr;
+	__maybe_unused unsigned long irqflags;
+	int ret, i;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = bm_mc_start(&p->p);
+	mcc->acquire.bpid = pool->params.bpid;
+	bm_mc_commit(&p->p, BM_MCC_VERB_CMD_ACQUIRE |
+			(num & BM_MCC_VERB_ACQUIRE_BUFCOUNT));
+	while (!(mcr = bm_mc_result(&p->p)))
+		cpu_relax();
+	ret = mcr->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT;
+	if (bufs) {
+		for (i = 0; i < num; i++)
+			bufs[i].opaque =
+				be64_to_cpu(mcr->acquire.bufs[i].opaque);
+	}
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (ret != num)
+		ret = -ENOMEM;
+	return ret;
+}
+
+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
+			u32 flags)
+{
+	int ret;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (!num || (num > 8))
+		return -EINVAL;
+	if (pool->params.flags & BMAN_POOL_FLAG_ONLY_RELEASE)
+		return -EINVAL;
+#endif
+	/* Without stockpile, this API is a pass-through to the h/w operation */
+	if (!(pool->params.flags & BMAN_POOL_FLAG_STOCKPILE))
+		return __bman_acquire(pool, bufs, num);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (!atomic_dec_and_test(&pool->in_use)) {
+		pr_crit("Parallel attempts to enter bman_acquire() detected.");
+		panic("only one instance of bman_released/acquired allowed");
+	}
+#endif
+	/* Two movements of buffers are possible, and can occur in either order.
+	 * A: moving buffers from stockpile to the caller.
+	 * B: moving buffers from hardware to the stockpile.
+	 * Order 1: if there are already enough buffers in the stockpile for A
+	 * then we want to do A first, and only do B if we trigger the
+	 * stockpile-low threshold.
+	 * Order 2: if there are not enough buffers in the stockpile for A,
+	 * then we want to do B first, then do A if B had succeeded. However in
+	 * this case B is dependent on how many buffers the user needs, not the
+	 * stockpile-low threshold.
+	 * Due to the different handling of B between the two cases, putting A
+	 * and B in a while() loop would require quite obscure logic, so handle
+	 * the different sequences explicitly. */
+	if (num <= pool->sp_fill) {
+		/* Order 1: do A */
+		copy_words(bufs, pool->sp + (pool->sp_fill - num),
+			   sizeof(struct bm_buffer) * num);
+		pool->sp_fill -= num;
+		/* do B relative to STOCKPILE_LOW */
+		while (pool->sp_fill <= BMAN_STOCKPILE_LOW) {
+			ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8);
+			if (ret < 0)
+				ret = __bman_acquire(pool,
+						pool->sp + pool->sp_fill, 1);
+			if (ret < 0)
+				break;
+			pool->sp_fill += ret;
+		}
+	} else {
+		/* Order 2: do B relative to 'num' */
+		do {
+			ret = __bman_acquire(pool, pool->sp + pool->sp_fill, 8);
+			if (ret < 0)
+				ret = __bman_acquire(pool,
+						pool->sp + pool->sp_fill, 1);
+			if (ret < 0)
+				/* failure */
+				goto acquire_done;
+			pool->sp_fill += ret;
+		} while (pool->sp_fill < num);
+		/* do A */
+		copy_words(bufs, pool->sp + (pool->sp_fill - num),
+			   sizeof(struct bm_buffer) * num);
+		pool->sp_fill -= num;
+	}
+	/* success */
+	ret = num;
+acquire_done:
+#ifdef CONFIG_FSL_DPA_CHECKING
+	atomic_inc(&pool->in_use);
+#endif
+	return ret;
+}
+EXPORT_SYMBOL(bman_acquire);
+
+int bman_flush_stockpile(struct bman_pool *pool, u32 flags)
+{
+	u8 num;
+	int ret;
+
+	while (pool->sp_fill) {
+		num = ((pool->sp_fill > 8) ? 8 : pool->sp_fill);
+		ret = __bman_release(pool, pool->sp + (pool->sp_fill - num),
+				     num, flags);
+		if (ret)
+			return ret;
+		pool->sp_fill -= num;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(bman_flush_stockpile);
+
+int bman_query_pools(struct bm_pool_state *state)
+{
+	struct bman_portal *p = get_affine_portal();
+	struct bm_mc_result *mcr;
+	__maybe_unused unsigned long irqflags;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	bm_mc_start(&p->p);
+	bm_mc_commit(&p->p, BM_MCC_VERB_CMD_QUERY);
+	while (!(mcr = bm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & BM_MCR_VERB_CMD_MASK) == BM_MCR_VERB_CMD_QUERY);
+	*state = mcr->query;
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return 0;
+}
+EXPORT_SYMBOL(bman_query_pools);
+
+#ifdef CONFIG_FSL_BMAN_CONFIG
+u32 bman_query_free_buffers(struct bman_pool *pool)
+{
+	return bm_pool_free_buffers(pool->params.bpid);
+}
+EXPORT_SYMBOL(bman_query_free_buffers);
+
+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds)
+{
+	u32 bpid;
+
+	bpid = bman_get_params(pool)->bpid;
+
+	return bm_pool_set(bpid, thresholds);
+}
+EXPORT_SYMBOL(bman_update_pool_thresholds);
+#endif
+
+int bman_shutdown_pool(u32 bpid)
+{
+	struct bman_portal *p = get_affine_portal();
+	__maybe_unused unsigned long irqflags;
+	int ret;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	ret = bm_shutdown_pool(&p->p, bpid);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(bman_shutdown_pool);
+
+const struct bm_portal_config *bman_get_bm_portal_config(
+						struct bman_portal *portal)
+{
+	return portal->sharing_redirect ? NULL : portal->config;
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_low.h
@@ -0,0 +1,559 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman_private.h"
+
+/***************************/
+/* Portal register assists */
+/***************************/
+
+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+
+/* Cache-inhibited register offsets */
+#define BM_REG_RCR_PI_CINH	0x0000
+#define BM_REG_RCR_CI_CINH	0x0004
+#define BM_REG_RCR_ITR		0x0008
+#define BM_REG_CFG		0x0100
+#define BM_REG_SCN(n)		(0x0200 + ((n) << 2))
+#define BM_REG_ISR		0x0e00
+#define BM_REG_IIR              0x0e0c
+
+/* Cache-enabled register offsets */
+#define BM_CL_CR		0x0000
+#define BM_CL_RR0		0x0100
+#define BM_CL_RR1		0x0140
+#define BM_CL_RCR		0x1000
+#define BM_CL_RCR_PI_CENA	0x3000
+#define BM_CL_RCR_CI_CENA	0x3100
+
+#endif
+
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+
+/* Cache-inhibited register offsets */
+#define BM_REG_RCR_PI_CINH	0x3000
+#define BM_REG_RCR_CI_CINH	0x3100
+#define BM_REG_RCR_ITR		0x3200
+#define BM_REG_CFG		0x3300
+#define BM_REG_SCN(n)		(0x3400 + ((n) << 6))
+#define BM_REG_ISR		0x3e00
+#define BM_REG_IIR              0x3ec0
+
+/* Cache-enabled register offsets */
+#define BM_CL_CR		0x0000
+#define BM_CL_RR0		0x0100
+#define BM_CL_RR1		0x0140
+#define BM_CL_RCR		0x1000
+#define BM_CL_RCR_PI_CENA	0x3000
+#define BM_CL_RCR_CI_CENA	0x3100
+
+#endif
+
+/* BTW, the drivers (and h/w programming model) already obtain the required
+ * synchronisation for portal accesses via lwsync(), hwsync(), and
+ * data-dependencies. Use of barrier()s or other order-preserving primitives
+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which
+ * simply ensure that the compiler treats the portal registers as volatile (ie.
+ * non-coherent). */
+
+/* Cache-inhibited register access. */
+#define __bm_in(bm, o)		be32_to_cpu(__raw_readl((bm)->addr_ci + (o)))
+#define __bm_out(bm, o, val)    __raw_writel(cpu_to_be32(val), \
+					     (bm)->addr_ci + (o));
+#define bm_in(reg)		__bm_in(&portal->addr, BM_REG_##reg)
+#define bm_out(reg, val)	__bm_out(&portal->addr, BM_REG_##reg, val)
+
+/* Cache-enabled (index) register access */
+#define __bm_cl_touch_ro(bm, o) dcbt_ro((bm)->addr_ce + (o))
+#define __bm_cl_touch_rw(bm, o) dcbt_rw((bm)->addr_ce + (o))
+#define __bm_cl_in(bm, o)	be32_to_cpu(__raw_readl((bm)->addr_ce + (o)))
+#define __bm_cl_out(bm, o, val) \
+	do { \
+		u32 *__tmpclout = (bm)->addr_ce + (o); \
+		__raw_writel(cpu_to_be32(val), __tmpclout); \
+		dcbf(__tmpclout); \
+	} while (0)
+#define __bm_cl_invalidate(bm, o) dcbi((bm)->addr_ce + (o))
+#define bm_cl_touch_ro(reg) __bm_cl_touch_ro(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_touch_rw(reg) __bm_cl_touch_rw(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_in(reg)	    __bm_cl_in(&portal->addr, BM_CL_##reg##_CENA)
+#define bm_cl_out(reg, val) __bm_cl_out(&portal->addr, BM_CL_##reg##_CENA, val)
+#define bm_cl_invalidate(reg)\
+	__bm_cl_invalidate(&portal->addr, BM_CL_##reg##_CENA)
+
+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
+ * analysis, look at using the "extra" bit in the ring index registers to avoid
+ * cyclic issues. */
+static inline u8 bm_cyc_diff(u8 ringsize, u8 first, u8 last)
+{
+	/* 'first' is included, 'last' is excluded */
+	if (first <= last)
+		return last - first;
+	return ringsize + last - first;
+}
+
+/* Portal modes.
+ *   Enum types;
+ *     pmode == production mode
+ *     cmode == consumption mode,
+ *   Enum values use 3 letter codes. First letter matches the portal mode,
+ *   remaining two letters indicate;
+ *     ci == cache-inhibited portal register
+ *     ce == cache-enabled portal register
+ *     vb == in-band valid-bit (cache-enabled)
+ */
+enum bm_rcr_pmode {		/* matches BCSP_CFG::RPM */
+	bm_rcr_pci = 0,		/* PI index, cache-inhibited */
+	bm_rcr_pce = 1,		/* PI index, cache-enabled */
+	bm_rcr_pvb = 2		/* valid-bit */
+};
+enum bm_rcr_cmode {		/* s/w-only */
+	bm_rcr_cci,		/* CI index, cache-inhibited */
+	bm_rcr_cce		/* CI index, cache-enabled */
+};
+
+
+/* ------------------------- */
+/* --- Portal structures --- */
+
+#define BM_RCR_SIZE		8
+
+struct bm_rcr {
+	struct bm_rcr_entry *ring, *cursor;
+	u8 ci, available, ithresh, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	u32 busy;
+	enum bm_rcr_pmode pmode;
+	enum bm_rcr_cmode cmode;
+#endif
+};
+
+struct bm_mc {
+	struct bm_mc_command *cr;
+	struct bm_mc_result *rr;
+	u8 rridx, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	enum {
+		/* Can only be _mc_start()ed */
+		mc_idle,
+		/* Can only be _mc_commit()ed or _mc_abort()ed */
+		mc_user,
+		/* Can only be _mc_retry()ed */
+		mc_hw
+	} state;
+#endif
+};
+
+struct bm_addr {
+	void __iomem *addr_ce;	/* cache-enabled */
+	void __iomem *addr_ci;	/* cache-inhibited */
+};
+
+struct bm_portal {
+	struct bm_addr addr;
+	struct bm_rcr rcr;
+	struct bm_mc mc;
+	struct bm_portal_config config;
+} ____cacheline_aligned;
+
+
+/* --------------- */
+/* --- RCR API --- */
+
+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
+#define RCR_CARRYCLEAR(p) \
+	(void *)((unsigned long)(p) & (~(unsigned long)(BM_RCR_SIZE << 6)))
+
+/* Bit-wise logic to convert a ring pointer to a ring index */
+static inline u8 RCR_PTR2IDX(struct bm_rcr_entry *e)
+{
+	return ((uintptr_t)e >> 6) & (BM_RCR_SIZE - 1);
+}
+
+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
+static inline void RCR_INC(struct bm_rcr *rcr)
+{
+	/* NB: this is odd-looking, but experiments show that it generates
+	 * fast code with essentially no branching overheads. We increment to
+	 * the next RCR pointer and handle overflow and 'vbit'. */
+	struct bm_rcr_entry *partial = rcr->cursor + 1;
+	rcr->cursor = RCR_CARRYCLEAR(partial);
+	if (partial != rcr->cursor)
+		rcr->vbit ^= BM_RCR_VERB_VBIT;
+}
+
+static inline int bm_rcr_init(struct bm_portal *portal, enum bm_rcr_pmode pmode,
+		__maybe_unused enum bm_rcr_cmode cmode)
+{
+	/* This use of 'register', as well as all other occurrences, is because
+	 * it has been observed to generate much faster code with gcc than is
+	 * otherwise the case. */
+	register struct bm_rcr *rcr = &portal->rcr;
+	u32 cfg;
+	u8 pi;
+
+	rcr->ring = portal->addr.addr_ce + BM_CL_RCR;
+	rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+
+	pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
+	rcr->cursor = rcr->ring + pi;
+	rcr->vbit = (bm_in(RCR_PI_CINH) & BM_RCR_SIZE) ?  BM_RCR_VERB_VBIT : 0;
+	rcr->available = BM_RCR_SIZE - 1
+		- bm_cyc_diff(BM_RCR_SIZE, rcr->ci, pi);
+	rcr->ithresh = bm_in(RCR_ITR);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 0;
+	rcr->pmode = pmode;
+	rcr->cmode = cmode;
+#endif
+	cfg = (bm_in(CFG) & 0xffffffe0) | (pmode & 0x3); /* BCSP_CFG::RPM */
+	bm_out(CFG, cfg);
+	return 0;
+}
+
+static inline void bm_rcr_finish(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	u8 pi = bm_in(RCR_PI_CINH) & (BM_RCR_SIZE - 1);
+	u8 ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+	DPA_ASSERT(!rcr->busy);
+	if (pi != RCR_PTR2IDX(rcr->cursor))
+		pr_crit("losing uncommited RCR entries\n");
+	if (ci != rcr->ci)
+		pr_crit("missing existing RCR completions\n");
+	if (rcr->ci != RCR_PTR2IDX(rcr->cursor))
+		pr_crit("RCR destroyed unquiesced\n");
+}
+
+static inline struct bm_rcr_entry *bm_rcr_start(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(!rcr->busy);
+	if (!rcr->available)
+		return NULL;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 1;
+#endif
+	dcbz_64(rcr->cursor);
+	return rcr->cursor;
+}
+
+static inline void bm_rcr_abort(struct bm_portal *portal)
+{
+	__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->busy);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 0;
+#endif
+}
+
+static inline struct bm_rcr_entry *bm_rcr_pend_and_next(
+					struct bm_portal *portal, u8 myverb)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->busy);
+	DPA_ASSERT(rcr->pmode != bm_rcr_pvb);
+	if (rcr->available == 1)
+		return NULL;
+	rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+	dcbf_64(rcr->cursor);
+	RCR_INC(rcr);
+	rcr->available--;
+	dcbz_64(rcr->cursor);
+	return rcr->cursor;
+}
+
+static inline void bm_rcr_pci_commit(struct bm_portal *portal, u8 myverb)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->busy);
+	DPA_ASSERT(rcr->pmode == bm_rcr_pci);
+	rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+	RCR_INC(rcr);
+	rcr->available--;
+	hwsync();
+	bm_out(RCR_PI_CINH, RCR_PTR2IDX(rcr->cursor));
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 0;
+#endif
+}
+
+static inline void bm_rcr_pce_prefetch(struct bm_portal *portal)
+{
+	__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->pmode == bm_rcr_pce);
+	bm_cl_invalidate(RCR_PI);
+	bm_cl_touch_rw(RCR_PI);
+}
+
+static inline void bm_rcr_pce_commit(struct bm_portal *portal, u8 myverb)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->busy);
+	DPA_ASSERT(rcr->pmode == bm_rcr_pce);
+	rcr->cursor->__dont_write_directly__verb = myverb | rcr->vbit;
+	RCR_INC(rcr);
+	rcr->available--;
+	lwsync();
+	bm_cl_out(RCR_PI, RCR_PTR2IDX(rcr->cursor));
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 0;
+#endif
+}
+
+static inline void bm_rcr_pvb_commit(struct bm_portal *portal, u8 myverb)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	struct bm_rcr_entry *rcursor;
+	DPA_ASSERT(rcr->busy);
+	DPA_ASSERT(rcr->pmode == bm_rcr_pvb);
+	lwsync();
+	rcursor = rcr->cursor;
+	rcursor->__dont_write_directly__verb = myverb | rcr->vbit;
+	dcbf_64(rcursor);
+	RCR_INC(rcr);
+	rcr->available--;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	rcr->busy = 0;
+#endif
+}
+
+static inline u8 bm_rcr_cci_update(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	u8 diff, old_ci = rcr->ci;
+	DPA_ASSERT(rcr->cmode == bm_rcr_cci);
+	rcr->ci = bm_in(RCR_CI_CINH) & (BM_RCR_SIZE - 1);
+	diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
+	rcr->available += diff;
+	return diff;
+}
+
+static inline void bm_rcr_cce_prefetch(struct bm_portal *portal)
+{
+	__maybe_unused register struct bm_rcr *rcr = &portal->rcr;
+	DPA_ASSERT(rcr->cmode == bm_rcr_cce);
+	bm_cl_touch_ro(RCR_CI);
+}
+
+static inline u8 bm_rcr_cce_update(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	u8 diff, old_ci = rcr->ci;
+	DPA_ASSERT(rcr->cmode == bm_rcr_cce);
+	rcr->ci = bm_cl_in(RCR_CI) & (BM_RCR_SIZE - 1);
+	bm_cl_invalidate(RCR_CI);
+	diff = bm_cyc_diff(BM_RCR_SIZE, old_ci, rcr->ci);
+	rcr->available += diff;
+	return diff;
+}
+
+static inline u8 bm_rcr_get_ithresh(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	return rcr->ithresh;
+}
+
+static inline void bm_rcr_set_ithresh(struct bm_portal *portal, u8 ithresh)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	rcr->ithresh = ithresh;
+	bm_out(RCR_ITR, ithresh);
+}
+
+static inline u8 bm_rcr_get_avail(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	return rcr->available;
+}
+
+static inline u8 bm_rcr_get_fill(struct bm_portal *portal)
+{
+	register struct bm_rcr *rcr = &portal->rcr;
+	return BM_RCR_SIZE - 1 - rcr->available;
+}
+
+
+/* ------------------------------ */
+/* --- Management command API --- */
+
+static inline int bm_mc_init(struct bm_portal *portal)
+{
+	register struct bm_mc *mc = &portal->mc;
+	mc->cr = portal->addr.addr_ce + BM_CL_CR;
+	mc->rr = portal->addr.addr_ce + BM_CL_RR0;
+	mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
+			BM_MCC_VERB_VBIT) ?  0 : 1;
+	mc->vbit = mc->rridx ? BM_MCC_VERB_VBIT : 0;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = mc_idle;
+#endif
+	return 0;
+}
+
+static inline void bm_mc_finish(struct bm_portal *portal)
+{
+	__maybe_unused register struct bm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == mc_idle);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (mc->state != mc_idle)
+		pr_crit("Losing incomplete MC command\n");
+#endif
+}
+
+static inline struct bm_mc_command *bm_mc_start(struct bm_portal *portal)
+{
+	register struct bm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == mc_idle);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = mc_user;
+#endif
+	dcbz_64(mc->cr);
+	return mc->cr;
+}
+
+static inline void bm_mc_abort(struct bm_portal *portal)
+{
+	__maybe_unused register struct bm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == mc_user);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = mc_idle;
+#endif
+}
+
+static inline void bm_mc_commit(struct bm_portal *portal, u8 myverb)
+{
+	register struct bm_mc *mc = &portal->mc;
+	struct bm_mc_result *rr = mc->rr + mc->rridx;
+	DPA_ASSERT(mc->state == mc_user);
+	lwsync();
+	mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
+	dcbf(mc->cr);
+	dcbit_ro(rr);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = mc_hw;
+#endif
+}
+
+static inline struct bm_mc_result *bm_mc_result(struct bm_portal *portal)
+{
+	register struct bm_mc *mc = &portal->mc;
+	struct bm_mc_result *rr = mc->rr + mc->rridx;
+	DPA_ASSERT(mc->state == mc_hw);
+	/* The inactive response register's verb byte always returns zero until
+	 * its command is submitted and completed. This includes the valid-bit,
+	 * in case you were wondering... */
+	if (!__raw_readb(&rr->verb)) {
+		dcbit_ro(rr);
+		return NULL;
+	}
+	mc->rridx ^= 1;
+	mc->vbit ^= BM_MCC_VERB_VBIT;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = mc_idle;
+#endif
+	return rr;
+}
+
+
+/* ------------------------------------- */
+/* --- Portal interrupt register API --- */
+
+static inline int bm_isr_init(__always_unused struct bm_portal *portal)
+{
+	return 0;
+}
+
+static inline void bm_isr_finish(__always_unused struct bm_portal *portal)
+{
+}
+
+#define SCN_REG(bpid) BM_REG_SCN((bpid) / 32)
+#define SCN_BIT(bpid) (0x80000000 >> (bpid & 31))
+static inline void bm_isr_bscn_mask(struct bm_portal *portal, u8 bpid,
+					int enable)
+{
+	u32 val;
+	DPA_ASSERT(bpid < bman_pool_max);
+	/* REG_SCN for bpid=0..31, REG_SCN+4 for bpid=32..63 */
+	val = __bm_in(&portal->addr, SCN_REG(bpid));
+	if (enable)
+		val |= SCN_BIT(bpid);
+	else
+		val &= ~SCN_BIT(bpid);
+	__bm_out(&portal->addr, SCN_REG(bpid), val);
+}
+
+static inline u32 __bm_isr_read(struct bm_portal *portal, enum bm_isr_reg n)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	return __bm_in(&portal->addr, BM_REG_ISR + (n << 6));
+#else
+	return __bm_in(&portal->addr, BM_REG_ISR + (n << 2));
+#endif
+}
+
+static inline void __bm_isr_write(struct bm_portal *portal, enum bm_isr_reg n,
+					u32 val)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	__bm_out(&portal->addr, BM_REG_ISR + (n << 6), val);
+#else
+	__bm_out(&portal->addr, BM_REG_ISR + (n << 2), val);
+#endif
+}
+
+/* Buffer Pool Cleanup */
+static inline int bm_shutdown_pool(struct bm_portal *p, u32 bpid)
+{
+	struct bm_mc_command *bm_cmd;
+	struct bm_mc_result *bm_res;
+
+	int aq_count = 0;
+	bool stop = false;
+	while (!stop) {
+		/* Acquire buffers until empty */
+		bm_cmd = bm_mc_start(p);
+		bm_cmd->acquire.bpid = bpid;
+		bm_mc_commit(p, BM_MCC_VERB_CMD_ACQUIRE |  1);
+		while (!(bm_res = bm_mc_result(p)))
+			cpu_relax();
+		if (!(bm_res->verb & BM_MCR_VERB_ACQUIRE_BUFCOUNT)) {
+			/* Pool is empty */
+			/* TBD : Should we do a few extra iterations in
+			   case some other some blocks keep buffers 'on deck',
+			   which may also be problematic */
+			stop = true;
+		} else
+			++aq_count;
+	}
+	return 0;
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_private.h
@@ -0,0 +1,166 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "dpa_sys.h"
+#include <linux/fsl_bman.h>
+
+/* Revision info (for errata and feature handling) */
+#define BMAN_REV10 0x0100
+#define BMAN_REV20 0x0200
+#define BMAN_REV21 0x0201
+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
+extern u16 bman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
+
+/*
+ * Global variables of the max portal/pool number this bman version supported
+ */
+extern u16 bman_pool_max;
+
+/* used by CCSR and portal interrupt code */
+enum bm_isr_reg {
+	bm_isr_status = 0,
+	bm_isr_enable = 1,
+	bm_isr_disable = 2,
+	bm_isr_inhibit = 3
+};
+
+struct bm_portal_config {
+	/* Corenet portal addresses;
+	 * [0]==cache-enabled, [1]==cache-inhibited. */
+	__iomem void *addr_virt[2];
+	struct resource addr_phys[2];
+	/* Allow these to be joined in lists */
+	struct list_head list;
+	/* User-visible portal configuration settings */
+	struct bman_portal_config public_cfg;
+	/* power management saved data */
+	u32 saved_isdr;
+};
+
+#ifdef CONFIG_FSL_BMAN_CONFIG
+/* Hooks from bman_driver.c to bman_config.c */
+int bman_init_ccsr(struct device_node *node);
+#endif
+
+/* Hooks from bman_driver.c in to bman_high.c */
+struct bman_portal *bman_create_portal(
+				       struct bman_portal *portal,
+				       const struct bm_portal_config *config);
+struct bman_portal *bman_create_affine_portal(
+			const struct bm_portal_config *config);
+struct bman_portal *bman_create_affine_slave(struct bman_portal *redirect,
+								int cpu);
+void bman_destroy_portal(struct bman_portal *bm);
+
+const struct bm_portal_config *bman_destroy_affine_portal(void);
+
+/* Hooks from fsl_usdpaa.c to bman_driver.c */
+struct bm_portal_config *bm_get_unused_portal(void);
+struct bm_portal_config *bm_get_unused_portal_idx(uint32_t idx);
+void bm_put_unused_portal(struct bm_portal_config *pcfg);
+void bm_set_liodns(struct bm_portal_config *pcfg);
+
+/* Pool logic in the portal driver, during initialisation, needs to know if
+ * there's access to CCSR or not (if not, it'll cripple the pool allocator). */
+#ifdef CONFIG_FSL_BMAN_CONFIG
+int bman_have_ccsr(void);
+#else
+#define bman_have_ccsr() 0
+#endif
+
+/* Stockpile build constants. The _LOW value: when bman_acquire() is called and
+ * the stockpile fill-level is <= _LOW, an acquire is attempted from h/w but it
+ * might fail (if the buffer pool is depleted). So this value provides some
+ * "stagger" in that the bman_acquire() function will only fail if lots of bufs
+ * are requested at once or if h/w has been tested a couple of times without
+ * luck. The _HIGH value: when bman_release() is called and the stockpile
+ * fill-level is >= _HIGH, a release is attempted to h/w but it might fail (if
+ * the release ring is full). So this value provides some "stagger" so that
+ * ring-access is retried a couple of times prior to the API returning a
+ * failure. The following *must* be true;
+ *   BMAN_STOCKPILE_HIGH-BMAN_STOCKPILE_LOW > 8
+ *     (to avoid thrashing)
+ *   BMAN_STOCKPILE_SZ >= 16
+ *     (as the release logic expects to either send 8 buffers to hw prior to
+ *     adding the given buffers to the stockpile or add the buffers to the
+ *     stockpile before sending 8 to hw, as the API must be an all-or-nothing
+ *     success/fail.)
+ */
+#define BMAN_STOCKPILE_SZ   16u /* number of bufs in per-pool cache */
+#define BMAN_STOCKPILE_LOW  2u  /* when fill is <= this, acquire from hw */
+#define BMAN_STOCKPILE_HIGH 14u /* when fill is >= this, release to hw */
+
+/*************************************************/
+/*   BMan s/w corenet portal, low-level i/face   */
+/*************************************************/
+
+/* Used by all portal interrupt registers except 'inhibit'
+ * This mask contains all the "irqsource" bits visible to API users
+ */
+#define BM_PIRQ_VISIBLE	(BM_PIRQ_RCRI | BM_PIRQ_BSCN)
+
+/* These are bm_<reg>_<verb>(). So for example, bm_disable_write() means "write
+ * the disable register" rather than "disable the ability to write". */
+#define bm_isr_status_read(bm)		__bm_isr_read(bm, bm_isr_status)
+#define bm_isr_status_clear(bm, m)	__bm_isr_write(bm, bm_isr_status, m)
+#define bm_isr_enable_read(bm)		__bm_isr_read(bm, bm_isr_enable)
+#define bm_isr_enable_write(bm, v)	__bm_isr_write(bm, bm_isr_enable, v)
+#define bm_isr_disable_read(bm)		__bm_isr_read(bm, bm_isr_disable)
+#define bm_isr_disable_write(bm, v)	__bm_isr_write(bm, bm_isr_disable, v)
+#define bm_isr_inhibit(bm)		__bm_isr_write(bm, bm_isr_inhibit, 1)
+#define bm_isr_uninhibit(bm)		__bm_isr_write(bm, bm_isr_inhibit, 0)
+
+#ifdef CONFIG_FSL_BMAN_CONFIG
+/* Set depletion thresholds associated with a buffer pool. Requires that the
+ * operating system have access to Bman CCSR (ie. compiled in support and
+ * run-time access courtesy of the device-tree). */
+int bm_pool_set(u32 bpid, const u32 *thresholds);
+#define BM_POOL_THRESH_SW_ENTER 0
+#define BM_POOL_THRESH_SW_EXIT  1
+#define BM_POOL_THRESH_HW_ENTER 2
+#define BM_POOL_THRESH_HW_EXIT  3
+
+/* Read the free buffer count for a given buffer */
+u32 bm_pool_free_buffers(u32 bpid);
+
+__init int bman_init(void);
+__init int bman_resource_init(void);
+
+const struct bm_portal_config *bman_get_bm_portal_config(
+						struct bman_portal *portal);
+
+/* power management */
+#ifdef CONFIG_SUSPEND
+void suspend_unused_bportal(void);
+void resume_unused_bportal(void);
+#endif
+
+#endif /* CONFIG_FSL_BMAN_CONFIG */
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_test.c
@@ -0,0 +1,56 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman_test.h"
+
+MODULE_AUTHOR("Geoff Thorpe");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Bman testing");
+
+static int test_init(void)
+{
+#ifdef CONFIG_FSL_BMAN_TEST_HIGH
+	int loop = 1;
+	while (loop--)
+		bman_test_high();
+#endif
+#ifdef CONFIG_FSL_BMAN_TEST_THRESH
+	bman_test_thresh();
+#endif
+	return 0;
+}
+
+static void test_exit(void)
+{
+}
+
+module_init(test_init);
+module_exit(test_exit);
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_test.h
@@ -0,0 +1,44 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/kthread.h>
+
+#include <linux/fsl_bman.h>
+
+void bman_test_high(void);
+void bman_test_thresh(void);
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_test_high.c
@@ -0,0 +1,183 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman_test.h"
+#include "bman_private.h"
+
+/*************/
+/* constants */
+/*************/
+
+#define PORTAL_OPAQUE	((void *)0xf00dbeef)
+#define POOL_OPAQUE	((void *)0xdeadabba)
+#define NUM_BUFS	93
+#define LOOPS		3
+#define BMAN_TOKEN_MASK 0x00FFFFFFFFFFLLU
+
+/***************/
+/* global vars */
+/***************/
+
+static struct bman_pool *pool;
+static int depleted;
+static struct bm_buffer bufs_in[NUM_BUFS] ____cacheline_aligned;
+static struct bm_buffer bufs_out[NUM_BUFS] ____cacheline_aligned;
+static int bufs_received;
+
+/* Predeclare the callback so we can instantiate pool parameters */
+static void depletion_cb(struct bman_portal *, struct bman_pool *, void *, int);
+
+/**********************/
+/* internal functions */
+/**********************/
+
+static void bufs_init(void)
+{
+	int i;
+	for (i = 0; i < NUM_BUFS; i++)
+		bm_buffer_set64(&bufs_in[i], 0xfedc01234567LLU * i);
+	bufs_received = 0;
+}
+
+static inline int bufs_cmp(const struct bm_buffer *a, const struct bm_buffer *b)
+{
+	if ((bman_ip_rev == BMAN_REV20) || (bman_ip_rev == BMAN_REV21)) {
+
+		/* On SoCs with Bman revison 2.0, Bman only respects the 40
+		 * LS-bits of buffer addresses, masking off the upper 8-bits on
+		 * release commands. The API provides for 48-bit addresses
+		 * because some SoCs support all 48-bits. When generating
+		 * garbage addresses for testing, we either need to zero the
+		 * upper 8-bits when releasing to Bman (otherwise we'll be
+		 * disappointed when the buffers we acquire back from Bman
+		 * don't match), or we need to mask the upper 8-bits off when
+		 * comparing. We do the latter.
+		 */
+		if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK)
+				< (bm_buffer_get64(b) & BMAN_TOKEN_MASK))
+			return -1;
+		if ((bm_buffer_get64(a) & BMAN_TOKEN_MASK)
+				> (bm_buffer_get64(b) & BMAN_TOKEN_MASK))
+			return 1;
+	} else {
+		if (bm_buffer_get64(a) < bm_buffer_get64(b))
+			return -1;
+		if (bm_buffer_get64(a) > bm_buffer_get64(b))
+			return 1;
+	}
+
+	return 0;
+}
+
+static void bufs_confirm(void)
+{
+	int i, j;
+	for (i = 0; i < NUM_BUFS; i++) {
+		int matches = 0;
+		for (j = 0; j < NUM_BUFS; j++)
+			if (!bufs_cmp(&bufs_in[i], &bufs_out[j]))
+				matches++;
+		BUG_ON(matches != 1);
+	}
+}
+
+/********/
+/* test */
+/********/
+
+static void depletion_cb(struct bman_portal *__portal, struct bman_pool *__pool,
+			void *pool_ctx, int __depleted)
+{
+	BUG_ON(__pool != pool);
+	BUG_ON(pool_ctx != POOL_OPAQUE);
+	depleted = __depleted;
+}
+
+void bman_test_high(void)
+{
+	struct bman_pool_params pparams = {
+		.flags = BMAN_POOL_FLAG_DEPLETION | BMAN_POOL_FLAG_DYNAMIC_BPID,
+		.cb = depletion_cb,
+		.cb_ctx = POOL_OPAQUE,
+	};
+	int i, loops = LOOPS;
+	struct bm_buffer tmp_buf;
+
+	bufs_init();
+
+	pr_info("BMAN:  --- starting high-level test ---\n");
+
+	pool = bman_new_pool(&pparams);
+	BUG_ON(!pool);
+
+	/*******************/
+	/* Release buffers */
+	/*******************/
+do_loop:
+	i = 0;
+	while (i < NUM_BUFS) {
+		u32 flags = BMAN_RELEASE_FLAG_WAIT;
+		int num = 8;
+		if ((i + num) > NUM_BUFS)
+			num = NUM_BUFS - i;
+		if ((i + num) == NUM_BUFS)
+			flags |= BMAN_RELEASE_FLAG_WAIT_SYNC;
+		if (bman_release(pool, bufs_in + i, num, flags))
+			panic("bman_release() failed\n");
+		i += num;
+	}
+
+	/*******************/
+	/* Acquire buffers */
+	/*******************/
+	while (i > 0) {
+		int tmp, num = 8;
+		if (num > i)
+			num = i;
+		tmp = bman_acquire(pool, bufs_out + i - num, num, 0);
+		BUG_ON(tmp != num);
+		i -= num;
+	}
+
+	i = bman_acquire(pool, &tmp_buf, 1, 0);
+	BUG_ON(i > 0);
+
+	bufs_confirm();
+
+	if (--loops)
+		goto do_loop;
+
+	/************/
+	/* Clean up */
+	/************/
+	bman_free_pool(pool);
+	pr_info("BMAN:  --- finished high-level test ---\n");
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/bman_test_thresh.c
@@ -0,0 +1,196 @@
+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "bman_test.h"
+
+/* Test constants */
+#define TEST_NUMBUFS	129728
+#define TEST_EXIT	129536
+#define TEST_ENTRY	129024
+
+struct affine_test_data {
+	struct task_struct *t;
+	int cpu;
+	int expect_affinity;
+	int drain;
+	int num_enter;
+	int num_exit;
+	struct list_head node;
+	struct completion wakethread;
+	struct completion wakeparent;
+};
+
+static void cb_depletion(struct bman_portal *portal,
+			struct bman_pool *pool,
+			void *opaque,
+			int depleted)
+{
+	struct affine_test_data *data = opaque;
+	int c = smp_processor_id();
+	pr_info("cb_depletion: bpid=%d, depleted=%d, cpu=%d, original=%d\n",
+		bman_get_params(pool)->bpid, !!depleted, c, data->cpu);
+	/* We should be executing on the CPU of the thread that owns the pool if
+	 * and that CPU has an affine portal (ie. it isn't slaved). */
+	BUG_ON((c != data->cpu) && data->expect_affinity);
+	BUG_ON((c == data->cpu) && !data->expect_affinity);
+	if (depleted)
+		data->num_enter++;
+	else
+		data->num_exit++;
+}
+
+/* Params used to set up a pool, this also dynamically allocates a BPID */
+static const struct bman_pool_params params_nocb = {
+	.flags = BMAN_POOL_FLAG_DYNAMIC_BPID | BMAN_POOL_FLAG_THRESH,
+	.thresholds = { TEST_ENTRY, TEST_EXIT, 0, 0 }
+};
+
+/* Params used to set up each cpu's pool with callbacks enabled */
+static struct bman_pool_params params_cb = {
+	.bpid = 0, /* will be replaced to match pool_nocb */
+	.flags = BMAN_POOL_FLAG_DEPLETION,
+	.cb = cb_depletion
+};
+
+static struct bman_pool *pool_nocb;
+static LIST_HEAD(threads);
+
+static int affine_test(void *__data)
+{
+	struct bman_pool *pool;
+	struct affine_test_data *data = __data;
+	struct bman_pool_params my_params = params_cb;
+
+	pr_info("thread %d: starting\n", data->cpu);
+	/* create the pool */
+	my_params.cb_ctx = data;
+	pool = bman_new_pool(&my_params);
+	BUG_ON(!pool);
+	complete(&data->wakeparent);
+	wait_for_completion(&data->wakethread);
+	init_completion(&data->wakethread);
+
+	/* if we're the drainer, we get signalled for that */
+	if (data->drain) {
+		struct bm_buffer buf;
+		int ret;
+		pr_info("thread %d: draining...\n", data->cpu);
+		do {
+			ret = bman_acquire(pool, &buf, 1, 0);
+		} while (ret > 0);
+		pr_info("thread %d: draining done.\n", data->cpu);
+		complete(&data->wakeparent);
+		wait_for_completion(&data->wakethread);
+		init_completion(&data->wakethread);
+	}
+
+	/* cleanup */
+	bman_free_pool(pool);
+	while (!kthread_should_stop())
+		cpu_relax();
+	pr_info("thread %d: exiting\n", data->cpu);
+	return 0;
+}
+
+static struct affine_test_data *start_affine_test(int cpu, int drain)
+{
+	struct affine_test_data *data = kmalloc(sizeof(*data), GFP_KERNEL);
+
+	if (!data)
+		return NULL;
+	data->cpu = cpu;
+	data->expect_affinity = cpumask_test_cpu(cpu, bman_affine_cpus());
+	data->drain = drain;
+	data->num_enter = 0;
+	data->num_exit = 0;
+	init_completion(&data->wakethread);
+	init_completion(&data->wakeparent);
+	list_add_tail(&data->node, &threads);
+	data->t = kthread_create(affine_test, data, "threshtest%d", cpu);
+	BUG_ON(IS_ERR(data->t));
+	kthread_bind(data->t, cpu);
+	wake_up_process(data->t);
+	return data;
+}
+
+void bman_test_thresh(void)
+{
+	int loop = TEST_NUMBUFS;
+	int ret, num_cpus = 0;
+	struct affine_test_data *data, *drainer = NULL;
+
+	pr_info("bman_test_thresh: start\n");
+
+	/* allocate a BPID and seed it */
+	pool_nocb = bman_new_pool(&params_nocb);
+	BUG_ON(!pool_nocb);
+	while (loop--) {
+		struct bm_buffer buf;
+		bm_buffer_set64(&buf, 0x0badbeef + loop);
+		ret = bman_release(pool_nocb, &buf, 1,
+					BMAN_RELEASE_FLAG_WAIT);
+		BUG_ON(ret);
+	}
+	while (!bman_rcr_is_empty())
+		cpu_relax();
+	pr_info("bman_test_thresh: buffers are in\n");
+
+	/* create threads and wait for them to create pools */
+	params_cb.bpid = bman_get_params(pool_nocb)->bpid;
+	for_each_cpu(loop, cpu_online_mask) {
+		data = start_affine_test(loop, drainer ? 0 : 1);
+		BUG_ON(!data);
+		if (!drainer)
+			drainer = data;
+		num_cpus++;
+		wait_for_completion(&data->wakeparent);
+	}
+
+	/* signal the drainer to start draining */
+	complete(&drainer->wakethread);
+	wait_for_completion(&drainer->wakeparent);
+	init_completion(&drainer->wakeparent);
+
+	/* tear down */
+	list_for_each_entry_safe(data, drainer, &threads, node) {
+		complete(&data->wakethread);
+		ret = kthread_stop(data->t);
+		BUG_ON(ret);
+		list_del(&data->node);
+		/* check that we get the expected callbacks (and no others) */
+		BUG_ON(data->num_enter != 1);
+		BUG_ON(data->num_exit != 0);
+		kfree(data);
+	}
+	bman_free_pool(pool_nocb);
+
+	pr_info("bman_test_thresh: done\n");
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_alloc.c
@@ -0,0 +1,706 @@
+/* Copyright 2009-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "dpa_sys.h"
+#include <linux/fsl_qman.h>
+#include <linux/fsl_bman.h>
+
+/* Qman and Bman APIs are front-ends to the common code; */
+
+static DECLARE_DPA_ALLOC(bpalloc); /* BPID allocator */
+static DECLARE_DPA_ALLOC(fqalloc); /* FQID allocator */
+static DECLARE_DPA_ALLOC(qpalloc); /* pool-channel allocator */
+static DECLARE_DPA_ALLOC(cgralloc); /* CGR ID allocator */
+static DECLARE_DPA_ALLOC(ceetm0_challoc); /* CEETM Channel ID allocator */
+static DECLARE_DPA_ALLOC(ceetm0_lfqidalloc); /* CEETM LFQID allocator */
+static DECLARE_DPA_ALLOC(ceetm1_challoc); /* CEETM Channel ID allocator */
+static DECLARE_DPA_ALLOC(ceetm1_lfqidalloc); /* CEETM LFQID allocator */
+
+/* This is a sort-of-conditional dpa_alloc_free() routine. Eg. when releasing
+ * FQIDs (probably from user-space), it can filter out those that aren't in the
+ * OOS state (better to leak a h/w resource than to crash). This function
+ * returns the number of invalid IDs that were not released. */
+static u32 release_id_range(struct dpa_alloc *alloc, u32 id, u32 count,
+			     int (*is_valid)(u32 id))
+{
+	int valid_mode = 0;
+	u32 loop = id, total_invalid = 0;
+	while (loop < (id + count)) {
+		int isvalid = is_valid ? is_valid(loop) : 1;
+		if (!valid_mode) {
+			/* We're looking for a valid ID to terminate an invalid
+			 * range */
+			if (isvalid) {
+				/* We finished a range of invalid IDs, a valid
+				 * range is now underway */
+				valid_mode = 1;
+				count -= (loop - id);
+				id = loop;
+			} else
+				total_invalid++;
+		} else {
+			/* We're looking for an invalid ID to terminate a
+			 * valid range */
+			if (!isvalid) {
+				/* Release the range of valid IDs, an unvalid
+				 * range is now underway */
+				if (loop > id)
+					dpa_alloc_free(alloc, id, loop - id);
+				valid_mode = 0;
+			}
+		}
+		loop++;
+	}
+	/* Release any unterminated range of valid IDs */
+	if (valid_mode && count)
+		dpa_alloc_free(alloc, id, count);
+	return total_invalid;
+}
+
+/* BPID allocator front-end */
+
+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial)
+{
+	return dpa_alloc_new(&bpalloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(bman_alloc_bpid_range);
+
+static int bp_cleanup(u32 bpid)
+{
+	return bman_shutdown_pool(bpid) == 0;
+}
+void bman_release_bpid_range(u32 bpid, u32 count)
+{
+	u32 total_invalid = release_id_range(&bpalloc, bpid, count, bp_cleanup);
+	if (total_invalid)
+		pr_err("BPID range [%d..%d] (%d) had %d leaks\n",
+			bpid, bpid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(bman_release_bpid_range);
+
+void bman_seed_bpid_range(u32 bpid, u32 count)
+{
+	dpa_alloc_seed(&bpalloc, bpid, count);
+}
+EXPORT_SYMBOL(bman_seed_bpid_range);
+
+int bman_reserve_bpid_range(u32 bpid, u32 count)
+{
+	return dpa_alloc_reserve(&bpalloc, bpid, count);
+}
+EXPORT_SYMBOL(bman_reserve_bpid_range);
+
+
+/* FQID allocator front-end */
+
+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial)
+{
+	return dpa_alloc_new(&fqalloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_fqid_range);
+
+static int fq_cleanup(u32 fqid)
+{
+	return qman_shutdown_fq(fqid) == 0;
+}
+void qman_release_fqid_range(u32 fqid, u32 count)
+{
+	u32 total_invalid = release_id_range(&fqalloc, fqid, count, fq_cleanup);
+	if (total_invalid)
+		pr_err("FQID range [%d..%d] (%d) had %d leaks\n",
+			fqid, fqid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_fqid_range);
+
+int qman_reserve_fqid_range(u32 fqid, u32 count)
+{
+	return dpa_alloc_reserve(&fqalloc, fqid, count);
+}
+EXPORT_SYMBOL(qman_reserve_fqid_range);
+
+void qman_seed_fqid_range(u32 fqid, u32 count)
+{
+	dpa_alloc_seed(&fqalloc, fqid, count);
+}
+EXPORT_SYMBOL(qman_seed_fqid_range);
+
+/* Pool-channel allocator front-end */
+
+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial)
+{
+	return dpa_alloc_new(&qpalloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_pool_range);
+
+static int qpool_cleanup(u32 qp)
+{
+	/* We query all FQDs starting from
+	 * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
+	 * whose destination channel is the pool-channel being released.
+	 * When a non-OOS FQD is found we attempt to clean it up */
+	struct qman_fq fq = {
+		.fqid = 1
+	};
+	int err;
+	do {
+		struct qm_mcr_queryfq_np np;
+		err = qman_query_fq_np(&fq, &np);
+		if (err)
+			/* FQID range exceeded, found no problems */
+			return 1;
+		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
+			struct qm_fqd fqd;
+			err = qman_query_fq(&fq, &fqd);
+			BUG_ON(err);
+			if (fqd.dest.channel == qp) {
+				/* The channel is the FQ's target, clean it */
+				if (qman_shutdown_fq(fq.fqid) != 0)
+					/* Couldn't shut down the FQ
+					   so the pool must be leaked */
+					return 0;
+			}
+		}
+		/* Move to the next FQID */
+		fq.fqid++;
+	} while (1);
+}
+void qman_release_pool_range(u32 qp, u32 count)
+{
+	u32 total_invalid = release_id_range(&qpalloc, qp,
+					     count, qpool_cleanup);
+	if (total_invalid) {
+		/* Pool channels are almost always used individually */
+		if (count == 1)
+			pr_err("Pool channel 0x%x had %d leaks\n",
+				qp, total_invalid);
+		else
+			pr_err("Pool channels [%d..%d] (%d) had %d leaks\n",
+				qp, qp + count - 1, count, total_invalid);
+	}
+}
+EXPORT_SYMBOL(qman_release_pool_range);
+
+
+void qman_seed_pool_range(u32 poolid, u32 count)
+{
+	dpa_alloc_seed(&qpalloc, poolid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_pool_range);
+
+int qman_reserve_pool_range(u32 poolid, u32 count)
+{
+	return dpa_alloc_reserve(&qpalloc, poolid, count);
+}
+EXPORT_SYMBOL(qman_reserve_pool_range);
+
+
+/* CGR ID allocator front-end */
+
+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial)
+{
+	return dpa_alloc_new(&cgralloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_cgrid_range);
+
+static int cqr_cleanup(u32 cgrid)
+{
+	/* We query all FQDs starting from
+	 * FQID 1 until we get an "invalid FQID" error, looking for non-OOS FQDs
+	 * whose CGR is the CGR being released.
+	 */
+	struct qman_fq fq = {
+		.fqid = 1
+	};
+	int err;
+	do {
+		struct qm_mcr_queryfq_np np;
+		err = qman_query_fq_np(&fq, &np);
+		if (err)
+			/* FQID range exceeded, found no problems */
+			return 1;
+		if ((np.state & QM_MCR_NP_STATE_MASK) != QM_MCR_NP_STATE_OOS) {
+			struct qm_fqd fqd;
+			err = qman_query_fq(&fq, &fqd);
+			BUG_ON(err);
+			if ((fqd.fq_ctrl & QM_FQCTRL_CGE) &&
+			    (fqd.cgid == cgrid)) {
+				pr_err("CRGID 0x%x is being used by FQID 0x%x,"
+				       " CGR will be leaked\n",
+				       cgrid, fq.fqid);
+				return 1;
+			}
+		}
+		/* Move to the next FQID */
+		fq.fqid++;
+	} while (1);
+}
+
+void qman_release_cgrid_range(u32 cgrid, u32 count)
+{
+	u32 total_invalid = release_id_range(&cgralloc, cgrid,
+					     count, cqr_cleanup);
+	if (total_invalid)
+		pr_err("CGRID range [%d..%d] (%d) had %d leaks\n",
+			cgrid, cgrid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_cgrid_range);
+
+void qman_seed_cgrid_range(u32 cgrid, u32 count)
+{
+	dpa_alloc_seed(&cgralloc, cgrid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_cgrid_range);
+
+/* CEETM CHANNEL ID allocator front-end */
+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align,
+								 int partial)
+{
+	return dpa_alloc_new(&ceetm0_challoc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_ceetm0_channel_range);
+
+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align,
+								 int partial)
+{
+	return dpa_alloc_new(&ceetm1_challoc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_ceetm1_channel_range);
+
+void qman_release_ceetm0_channel_range(u32 channelid, u32 count)
+{
+	u32 total_invalid;
+
+	total_invalid = release_id_range(&ceetm0_challoc, channelid, count,
+									 NULL);
+	if (total_invalid)
+		pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n",
+			channelid, channelid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_ceetm0_channel_range);
+
+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count)
+{
+	dpa_alloc_seed(&ceetm0_challoc, channelid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_ceetm0_channel_range);
+
+void qman_release_ceetm1_channel_range(u32 channelid, u32 count)
+{
+	u32 total_invalid;
+	total_invalid = release_id_range(&ceetm1_challoc, channelid, count,
+									 NULL);
+	if (total_invalid)
+		pr_err("CEETM channel range [%d..%d] (%d) had %d leaks\n",
+			channelid, channelid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_ceetm1_channel_range);
+
+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count)
+{
+	dpa_alloc_seed(&ceetm1_challoc, channelid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_ceetm1_channel_range);
+
+/* CEETM LFQID allocator front-end */
+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align,
+								 int partial)
+{
+	return dpa_alloc_new(&ceetm0_lfqidalloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_ceetm0_lfqid_range);
+
+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align,
+								 int partial)
+{
+	return dpa_alloc_new(&ceetm1_lfqidalloc, result, count, align, partial);
+}
+EXPORT_SYMBOL(qman_alloc_ceetm1_lfqid_range);
+
+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count)
+{
+	u32 total_invalid;
+
+	total_invalid = release_id_range(&ceetm0_lfqidalloc, lfqid, count,
+									NULL);
+	if (total_invalid)
+		pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n",
+			lfqid, lfqid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_ceetm0_lfqid_range);
+
+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count)
+{
+	dpa_alloc_seed(&ceetm0_lfqidalloc, lfqid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_ceetm0_lfqid_range);
+
+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count)
+{
+	u32 total_invalid;
+
+	total_invalid = release_id_range(&ceetm1_lfqidalloc, lfqid, count,
+									NULL);
+	if (total_invalid)
+		pr_err("CEETM LFQID range [0x%x..0x%x] (%d) had %d leaks\n",
+			lfqid, lfqid + count - 1, count, total_invalid);
+}
+EXPORT_SYMBOL(qman_release_ceetm1_lfqid_range);
+
+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count)
+{
+	dpa_alloc_seed(&ceetm1_lfqidalloc, lfqid, count);
+
+}
+EXPORT_SYMBOL(qman_seed_ceetm1_lfqid_range);
+
+
+/* Everything else is the common backend to all the allocators */
+
+/* The allocator is a (possibly-empty) list of these; */
+struct alloc_node {
+	struct list_head list;
+	u32 base;
+	u32 num;
+	/* refcount and is_alloced are only set
+	   when the node is in the used list */
+	unsigned int refcount;
+	int is_alloced;
+};
+
+/* #define DPA_ALLOC_DEBUG */
+
+#ifdef DPA_ALLOC_DEBUG
+#define DPRINT pr_info
+static void DUMP(struct dpa_alloc *alloc)
+{
+	int off = 0;
+	char buf[256];
+	struct alloc_node *p;
+	pr_info("Free Nodes\n");
+	list_for_each_entry(p, &alloc->free, list) {
+		if (off < 255)
+			off += snprintf(buf + off, 255-off, "{%d,%d}",
+				p->base, p->base + p->num - 1);
+	}
+	pr_info("%s\n", buf);
+
+	off = 0;
+	pr_info("Used Nodes\n");
+	list_for_each_entry(p, &alloc->used, list) {
+		if (off < 255)
+			off += snprintf(buf + off, 255-off, "{%d,%d}",
+				p->base, p->base + p->num - 1);
+	}
+	pr_info("%s\n", buf);
+
+
+
+}
+#else
+#define DPRINT(x...)
+#define DUMP(a)
+#endif
+
+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align,
+		  int partial)
+{
+	struct alloc_node *i = NULL, *next_best = NULL, *used_node = NULL;
+	u32 base, next_best_base = 0, num = 0, next_best_num = 0;
+	struct alloc_node *margin_left, *margin_right;
+
+	*result = (u32)-1;
+	DPRINT("alloc_range(%d,%d,%d)\n", count, align, partial);
+	DUMP(alloc);
+	/* If 'align' is 0, it should behave as though it was 1 */
+	if (!align)
+		align = 1;
+	margin_left = kmalloc(sizeof(*margin_left), GFP_KERNEL);
+	if (!margin_left)
+		goto err;
+	margin_right = kmalloc(sizeof(*margin_right), GFP_KERNEL);
+	if (!margin_right) {
+		kfree(margin_left);
+		goto err;
+	}
+	spin_lock_irq(&alloc->lock);
+	list_for_each_entry(i, &alloc->free, list) {
+		base = (i->base + align - 1) / align;
+		base *= align;
+		if ((base - i->base) >= i->num)
+			/* alignment is impossible, regardless of count */
+			continue;
+		num = i->num - (base - i->base);
+		if (num >= count) {
+			/* this one will do nicely */
+			num = count;
+			goto done;
+		}
+		if (num > next_best_num) {
+			next_best = i;
+			next_best_base = base;
+			next_best_num = num;
+		}
+	}
+	if (partial && next_best) {
+		i = next_best;
+		base = next_best_base;
+		num = next_best_num;
+	} else
+		i = NULL;
+done:
+	if (i) {
+		if (base != i->base) {
+			margin_left->base = i->base;
+			margin_left->num = base - i->base;
+			list_add_tail(&margin_left->list, &i->list);
+		} else
+			kfree(margin_left);
+		if ((base + num) < (i->base + i->num)) {
+			margin_right->base = base + num;
+			margin_right->num = (i->base + i->num) -
+						(base + num);
+			list_add(&margin_right->list, &i->list);
+		} else
+			kfree(margin_right);
+		list_del(&i->list);
+		kfree(i);
+		*result = base;
+	} else {
+		spin_unlock_irq(&alloc->lock);
+		kfree(margin_left);
+		kfree(margin_right);
+	}
+
+err:
+	DPRINT("returning %d\n", i ? num : -ENOMEM);
+	DUMP(alloc);
+	if (!i)
+		return -ENOMEM;
+
+	/* Add the allocation to the used list with a refcount of 1 */
+	used_node = kmalloc(sizeof(*used_node), GFP_KERNEL);
+	if (!used_node) {
+		spin_unlock_irq(&alloc->lock);
+		return -ENOMEM;
+	}
+	used_node->base = *result;
+	used_node->num = num;
+	used_node->refcount = 1;
+	used_node->is_alloced = 1;
+	list_add_tail(&used_node->list, &alloc->used);
+	spin_unlock_irq(&alloc->lock);
+	return (int)num;
+}
+
+/* Allocate the list node using GFP_ATOMIC, because we *really* want to avoid
+ * forcing error-handling on to users in the deallocation path. */
+static void _dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count)
+{
+	struct alloc_node *i, *node = kmalloc(sizeof(*node), GFP_ATOMIC);
+	BUG_ON(!node);
+	DPRINT("release_range(%d,%d)\n", base_id, count);
+	DUMP(alloc);
+	BUG_ON(!count);
+	spin_lock_irq(&alloc->lock);
+
+
+	node->base = base_id;
+	node->num = count;
+	list_for_each_entry(i, &alloc->free, list) {
+		if (i->base >= node->base) {
+			/* BUG_ON(any overlapping) */
+			BUG_ON(i->base < (node->base + node->num));
+			list_add_tail(&node->list, &i->list);
+			goto done;
+		}
+	}
+	list_add_tail(&node->list, &alloc->free);
+done:
+	/* Merge to the left */
+	i = list_entry(node->list.prev, struct alloc_node, list);
+	if (node->list.prev != &alloc->free) {
+		BUG_ON((i->base + i->num) > node->base);
+		if ((i->base + i->num) == node->base) {
+			node->base = i->base;
+			node->num += i->num;
+			list_del(&i->list);
+			kfree(i);
+		}
+	}
+	/* Merge to the right */
+	i = list_entry(node->list.next, struct alloc_node, list);
+	if (node->list.next != &alloc->free) {
+		BUG_ON((node->base + node->num) > i->base);
+		if ((node->base + node->num) == i->base) {
+			node->num += i->num;
+			list_del(&i->list);
+			kfree(i);
+		}
+	}
+	spin_unlock_irq(&alloc->lock);
+	DUMP(alloc);
+}
+
+
+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count)
+{
+	struct alloc_node *i = NULL;
+	spin_lock_irq(&alloc->lock);
+
+	/* First find the node in the used list and decrement its ref count */
+	list_for_each_entry(i, &alloc->used, list) {
+		if (i->base == base_id && i->num == count) {
+			--i->refcount;
+			if (i->refcount == 0) {
+				list_del(&i->list);
+				spin_unlock_irq(&alloc->lock);
+				if (i->is_alloced)
+					_dpa_alloc_free(alloc, base_id, count);
+				kfree(i);
+				return;
+			}
+			spin_unlock_irq(&alloc->lock);
+			return;
+		}
+	}
+	/* Couldn't find the allocation */
+	pr_err("Attempt to free ID 0x%x COUNT %d that wasn't alloc'd or reserved\n",
+	       base_id, count);
+	spin_unlock_irq(&alloc->lock);
+}
+
+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 base_id, u32 count)
+{
+	/* Same as free but no previous allocation checking is needed */
+	_dpa_alloc_free(alloc, base_id, count);
+}
+
+
+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base, u32 num)
+{
+	struct alloc_node *i = NULL, *used_node;
+
+	DPRINT("alloc_reserve(%d,%d)\n", base, num);
+	DUMP(alloc);
+
+	spin_lock_irq(&alloc->lock);
+
+	/* Check for the node in the used list.
+	   If found, increase it's refcount */
+	list_for_each_entry(i, &alloc->used, list) {
+		if ((i->base == base) && (i->num == num)) {
+			++i->refcount;
+			spin_unlock_irq(&alloc->lock);
+			return 0;
+		}
+		if ((base >= i->base) && (base < (i->base + i->num))) {
+			/* This is an attempt to reserve a region that was
+			   already reserved or alloced with a different
+			   base or num */
+			pr_err("Cannot reserve %d - %d, it overlaps with"
+			       " existing reservation from %d - %d\n",
+			       base, base + num - 1, i->base,
+			       i->base + i->num - 1);
+			spin_unlock_irq(&alloc->lock);
+			return -1;
+		}
+	}
+	/* Check to make sure this ID isn't in the free list */
+	list_for_each_entry(i, &alloc->free, list) {
+		if ((base >= i->base) && (base < (i->base + i->num))) {
+			/* yep, the reservation is within this node */
+			pr_err("Cannot reserve %d - %d, it overlaps with"
+			       " free range %d - %d and must be alloced\n",
+			       base, base + num - 1,
+			       i->base, i->base + i->num - 1);
+			spin_unlock_irq(&alloc->lock);
+			return -1;
+		}
+	}
+	/* Add the allocation to the used list with a refcount of 1 */
+	used_node = kmalloc(sizeof(*used_node), GFP_KERNEL);
+	if (!used_node) {
+		spin_unlock_irq(&alloc->lock);
+		return -ENOMEM;
+
+	}
+	used_node->base = base;
+	used_node->num = num;
+	used_node->refcount = 1;
+	used_node->is_alloced = 0;
+	list_add_tail(&used_node->list, &alloc->used);
+	spin_unlock_irq(&alloc->lock);
+	return 0;
+}
+
+
+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count)
+{
+	struct alloc_node *i = NULL;
+	DPRINT("alloc_pop()\n");
+	DUMP(alloc);
+	spin_lock_irq(&alloc->lock);
+	if (!list_empty(&alloc->free)) {
+		i = list_entry(alloc->free.next, struct alloc_node, list);
+		list_del(&i->list);
+	}
+	spin_unlock_irq(&alloc->lock);
+	DPRINT("returning %d\n", i ? 0 : -ENOMEM);
+	DUMP(alloc);
+	if (!i)
+		return -ENOMEM;
+	*result = i->base;
+	*count = i->num;
+	kfree(i);
+	return 0;
+}
+
+int dpa_alloc_check(struct dpa_alloc *list_head, u32 item)
+{
+	struct alloc_node *i = NULL;
+	int res = 0;
+	DPRINT("alloc_check()\n");
+	spin_lock_irq(&list_head->lock);
+
+	list_for_each_entry(i, &list_head->free, list) {
+		if ((item >= i->base) && (item < (i->base + i->num))) {
+			res = 1;
+			break;
+		}
+	}
+	spin_unlock_irq(&list_head->lock);
+	return res;
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_sys.h
@@ -0,0 +1,259 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DPA_SYS_H
+#define DPA_SYS_H
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/dma-mapping.h>
+#include <linux/bootmem.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/of_platform.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/kthread.h>
+#include <linux/memblock.h>
+#include <linux/completion.h>
+#include <linux/log2.h>
+#include <linux/types.h>
+#include <linux/ioctl.h>
+#include <linux/miscdevice.h>
+#include <linux/uaccess.h>
+#include <linux/debugfs.h>
+#include <linux/seq_file.h>
+#include <linux/device.h>
+#include <linux/uio_driver.h>
+#include <linux/smp.h>
+#include <linux/fsl_hypervisor.h>
+#include <linux/vmalloc.h>
+#include <linux/ctype.h>
+#include <linux/math64.h>
+#include <linux/bitops.h>
+
+#include <linux/fsl_usdpaa.h>
+
+/* When copying aligned words or shorts, try to avoid memcpy() */
+#define CONFIG_TRY_BETTER_MEMCPY
+
+/* For 2-element tables related to cache-inhibited and cache-enabled mappings */
+#define DPA_PORTAL_CE 0
+#define DPA_PORTAL_CI 1
+
+/***********************/
+/* Misc inline assists */
+/***********************/
+
+#if defined CONFIG_PPC32
+#include "dpa_sys_ppc32.h"
+#elif defined CONFIG_PPC64
+#include "dpa_sys_ppc64.h"
+#elif defined CONFIG_ARM
+#include "dpa_sys_arm.h"
+#elif defined CONFIG_ARM64
+#include "dpa_sys_arm64.h"
+#endif
+
+
+#ifdef CONFIG_FSL_DPA_CHECKING
+#define DPA_ASSERT(x) \
+	do { \
+		if (!(x)) { \
+			pr_crit("ASSERT: (%s:%d) %s\n", __FILE__, __LINE__, \
+				__stringify_1(x)); \
+			dump_stack(); \
+			panic("assertion failure"); \
+		} \
+	} while (0)
+#else
+#define DPA_ASSERT(x)
+#endif
+
+/* memcpy() stuff - when you know alignments in advance */
+#ifdef CONFIG_TRY_BETTER_MEMCPY
+static inline void copy_words(void *dest, const void *src, size_t sz)
+{
+	u32 *__dest = dest;
+	const u32 *__src = src;
+	size_t __sz = sz >> 2;
+	BUG_ON((unsigned long)dest & 0x3);
+	BUG_ON((unsigned long)src & 0x3);
+	BUG_ON(sz & 0x3);
+	while (__sz--)
+		*(__dest++) = *(__src++);
+}
+static inline void copy_shorts(void *dest, const void *src, size_t sz)
+{
+	u16 *__dest = dest;
+	const u16 *__src = src;
+	size_t __sz = sz >> 1;
+	BUG_ON((unsigned long)dest & 0x1);
+	BUG_ON((unsigned long)src & 0x1);
+	BUG_ON(sz & 0x1);
+	while (__sz--)
+		*(__dest++) = *(__src++);
+}
+static inline void copy_bytes(void *dest, const void *src, size_t sz)
+{
+	u8 *__dest = dest;
+	const u8 *__src = src;
+	while (sz--)
+		*(__dest++) = *(__src++);
+}
+#else
+#define copy_words memcpy
+#define copy_shorts memcpy
+#define copy_bytes memcpy
+#endif
+
+/************/
+/* RB-trees */
+/************/
+
+/* We encapsulate RB-trees so that its easier to use non-linux forms in
+ * non-linux systems. This also encapsulates the extra plumbing that linux code
+ * usually provides when using RB-trees. This encapsulation assumes that the
+ * data type held by the tree is u32. */
+
+struct dpa_rbtree {
+	struct rb_root root;
+};
+#define DPA_RBTREE { .root = RB_ROOT }
+
+static inline void dpa_rbtree_init(struct dpa_rbtree *tree)
+{
+	tree->root = RB_ROOT;
+}
+
+#define IMPLEMENT_DPA_RBTREE(name, type, node_field, val_field) \
+static inline int name##_push(struct dpa_rbtree *tree, type *obj) \
+{ \
+	struct rb_node *parent = NULL, **p = &tree->root.rb_node; \
+	while (*p) { \
+		u32 item; \
+		parent = *p; \
+		item = rb_entry(parent, type, node_field)->val_field; \
+		if (obj->val_field < item) \
+			p = &parent->rb_left; \
+		else if (obj->val_field > item) \
+			p = &parent->rb_right; \
+		else \
+			return -EBUSY; \
+	} \
+	rb_link_node(&obj->node_field, parent, p); \
+	rb_insert_color(&obj->node_field, &tree->root); \
+	return 0; \
+} \
+static inline void name##_del(struct dpa_rbtree *tree, type *obj) \
+{ \
+	rb_erase(&obj->node_field, &tree->root); \
+} \
+static inline type *name##_find(struct dpa_rbtree *tree, u32 val) \
+{ \
+	type *ret; \
+	struct rb_node *p = tree->root.rb_node; \
+	while (p) { \
+		ret = rb_entry(p, type, node_field); \
+		if (val < ret->val_field) \
+			p = p->rb_left; \
+		else if (val > ret->val_field) \
+			p = p->rb_right; \
+		else \
+			return ret; \
+	} \
+	return NULL; \
+}
+
+/************/
+/* Bootargs */
+/************/
+
+/* Qman has "qportals=" and Bman has "bportals=", they use the same syntax
+ * though; a comma-separated list of items, each item being a cpu index and/or a
+ * range of cpu indices, and each item optionally be prefixed by "s" to indicate
+ * that the portal associated with that cpu should be shared. See bman_driver.c
+ * for more specifics. */
+static int __parse_portals_cpu(const char **s, unsigned int *cpu)
+{
+	*cpu = 0;
+	if (!isdigit(**s))
+		return -EINVAL;
+	while (isdigit(**s))
+		*cpu = *cpu * 10 + (*((*s)++) - '0');
+	return 0;
+}
+static inline int parse_portals_bootarg(char *str, struct cpumask *want_shared,
+					struct cpumask *want_unshared,
+					const char *argname)
+{
+	const char *s = str;
+	unsigned int shared, cpu1, cpu2, loop;
+
+keep_going:
+	if (*s == 's') {
+		shared = 1;
+		s++;
+	} else
+		shared = 0;
+	if (__parse_portals_cpu(&s, &cpu1))
+		goto err;
+	if (*s == '-') {
+		s++;
+		if (__parse_portals_cpu(&s, &cpu2))
+			goto err;
+		if (cpu2 < cpu1)
+			goto err;
+	} else
+		cpu2 = cpu1;
+	for (loop = cpu1; loop <= cpu2; loop++)
+		cpumask_set_cpu(loop, shared ? want_shared : want_unshared);
+	if (*s == ',') {
+		s++;
+		goto keep_going;
+	} else if ((*s == '\0') || isspace(*s))
+		return 0;
+err:
+	pr_crit("Malformed %s argument: %s, offset: %lu\n", argname, str,
+		(unsigned long)s - (unsigned long)str);
+	return -EINVAL;
+}
+
+/* Hooks from fsl_usdpaa_irq.c to fsl_usdpaa.c */
+int usdpaa_get_portal_config(struct file *filp, void *cinh,
+			     enum usdpaa_portal_type ptype, unsigned int *irq,
+			     void **iir_reg);
+
+#endif /* DPA_SYS_H */
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_sys_arm.h
@@ -0,0 +1,95 @@
+/* Copyright 2016 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DPA_SYS_ARM_H
+#define DPA_SYS_ARM_H
+
+#include <asm/cacheflush.h>
+#include <asm/barrier.h>
+
+/* Implementation of ARM specific routines */
+
+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
+ * barriers and that dcb*() won't fall victim to compiler or execution
+ * reordering with respect to other code/instructions that manipulate the same
+ * cacheline. */
+#define hwsync() { asm volatile("dmb st" : : : "memory"); }
+#define lwsync() { asm volatile("dmb st" : : : "memory"); }
+#define dcbf(p) { asm volatile("mcr p15, 0, %0, c7, c10, 1" : : "r" (p) : "memory"); }
+#define dcbt_ro(p) { asm volatile("pld [%0, #64];": : "r" (p)); }
+#define dcbt_rw(p) { asm volatile("pldw [%0, #64];": : "r" (p)); }
+#define dcbi(p) { asm volatile("mcr p15, 0, %0, c7, c6, 1" : : "r" (p) : "memory"); }
+
+#define dcbz_64(p) { memset(p, 0, sizeof(*p)); }
+
+#define dcbf_64(p) \
+	do { \
+		dcbf((u32)p); \
+	} while (0)
+/* Commonly used combo */
+#define dcbit_ro(p) \
+	do { \
+		dcbi((u32)p); \
+		dcbt_ro((u32)p); \
+	} while (0)
+
+static inline u64 mfatb(void)
+{
+	return get_cycles();
+}
+
+static inline u32 in_be32(volatile void *addr)
+{
+	return be32_to_cpu(*((volatile u32 *) addr));
+}
+
+static inline void out_be32(void *addr, u32 val)
+{
+	*((u32 *) addr) = cpu_to_be32(val);
+}
+
+
+static inline void set_bits(unsigned long mask, volatile unsigned long *p)
+{
+	*p |= mask;
+}
+static inline void clear_bits(unsigned long mask, volatile unsigned long *p)
+{
+	*p &= ~mask;
+}
+
+static inline void flush_dcache_range(unsigned long start, unsigned long stop)
+{
+	__cpuc_flush_dcache_area((void *) start, stop - start);
+}
+
+#define hard_smp_processor_id() raw_smp_processor_id()
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_sys_arm64.h
@@ -0,0 +1,102 @@
+/* Copyright 2014 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DPA_SYS_ARM64_H
+#define DPA_SYS_ARM64_H
+
+#include <asm/cacheflush.h>
+#include <asm/barrier.h>
+
+/* Implementation of ARM 64 bit specific routines */
+
+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
+ * barriers and that dcb*() won't fall victim to compiler or execution
+ * reordering with respect to other code/instructions that manipulate the same
+ * cacheline. */
+#define hwsync() { asm volatile("dmb st" : : : "memory"); }
+#define lwsync() { asm volatile("dmb st" : : : "memory"); }
+#define dcbf(p) { asm volatile("dc cvac, %0;" : : "r" (p) : "memory"); }
+#define dcbt_ro(p) { asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); }
+#define dcbt_rw(p) { asm volatile("prfm pldl1keep, [%0, #64]" : : "r" (p)); }
+#define dcbi(p) { asm volatile("dc ivac, %0" : : "r"(p) : "memory"); }
+#define dcbz(p) { asm volatile("dc zva, %0" : : "r" (p) : "memory"); }
+
+#define dcbz_64(p) \
+	do { \
+		dcbz(p);	\
+	} while (0)
+
+#define dcbf_64(p) \
+	do { \
+		dcbf(p); \
+	} while (0)
+/* Commonly used combo */
+#define dcbit_ro(p) \
+	do { \
+		dcbi(p); \
+		dcbt_ro(p); \
+	} while (0)
+
+static inline u64 mfatb(void)
+{
+	return get_cycles();
+}
+
+static inline u32 in_be32(volatile void *addr)
+{
+	return be32_to_cpu(*((volatile u32 *) addr));
+}
+
+static inline void out_be32(void *addr, u32 val)
+{
+	*((u32 *) addr) = cpu_to_be32(val);
+}
+
+
+static inline void set_bits(unsigned long mask, volatile unsigned long *p)
+{
+	*p |= mask;
+}
+static inline void clear_bits(unsigned long mask, volatile unsigned long *p)
+{
+	*p &= ~mask;
+}
+
+static inline void flush_dcache_range(unsigned long start, unsigned long stop)
+{
+	__flush_dcache_area((void *) start, stop - start);
+}
+
+#define hard_smp_processor_id() raw_smp_processor_id()
+
+
+
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc32.h
@@ -0,0 +1,70 @@
+/* Copyright 2014 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DPA_SYS_PPC32_H
+#define DPA_SYS_PPC32_H
+
+/* Implementation of PowerPC 32 bit specific routines */
+
+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
+ * barriers and that dcb*() won't fall victim to compiler or execution
+ * reordering with respect to other code/instructions that manipulate the same
+ * cacheline. */
+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory")
+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory")
+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory")
+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p))
+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p))
+#define dcbi(p) dcbf(p)
+
+#define dcbzl(p) __asm__ __volatile__ ("dcbzl 0,%0" : : "r" (p))
+#define dcbz_64(p) dcbzl(p)
+#define dcbf_64(p) dcbf(p)
+
+/* Commonly used combo */
+#define dcbit_ro(p) \
+	do { \
+		dcbi(p); \
+		dcbt_ro(p); \
+	} while (0)
+
+static inline u64 mfatb(void)
+{
+	u32 hi, lo, chk;
+	do {
+		hi = mfspr(SPRN_ATBU);
+		lo = mfspr(SPRN_ATBL);
+		chk = mfspr(SPRN_ATBU);
+	} while (unlikely(hi != chk));
+	return ((u64)hi << 32) | (u64)lo;
+}
+
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/dpa_sys_ppc64.h
@@ -0,0 +1,79 @@
+/* Copyright 2014 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef DPA_SYS_PPC64_H
+#define DPA_SYS_PPC64_H
+
+/* Implementation of PowerPC 64 bit specific routines */
+
+/* TODO: NB, we currently assume that hwsync() and lwsync() imply compiler
+ * barriers and that dcb*() won't fall victim to compiler or execution
+ * reordering with respect to other code/instructions that manipulate the same
+ * cacheline. */
+#define hwsync() __asm__ __volatile__ ("sync" : : : "memory")
+#define lwsync() __asm__ __volatile__ (stringify_in_c(LWSYNC) : : : "memory")
+#define dcbf(p) __asm__ __volatile__ ("dcbf 0,%0" : : "r" (p) : "memory")
+#define dcbt_ro(p) __asm__ __volatile__ ("dcbt 0,%0" : : "r" (p))
+#define dcbt_rw(p) __asm__ __volatile__ ("dcbtst 0,%0" : : "r" (p))
+#define dcbi(p) dcbf(p)
+
+#define dcbz(p) __asm__ __volatile__ ("dcbz 0,%0" : : "r" (p))
+#define dcbz_64(p) \
+	do { \
+		dcbz((void*)p + 32);	\
+		dcbz(p);	\
+	} while (0)
+#define dcbf_64(p) \
+	do { \
+		dcbf((void*)p + 32); \
+		dcbf(p); \
+	} while (0)
+/* Commonly used combo */
+#define dcbit_ro(p) \
+	do { \
+		dcbi(p); \
+		dcbi((void*)p + 32); \
+		dcbt_ro(p); \
+		dcbt_ro((void*)p + 32); \
+	} while (0)
+
+static inline u64 mfatb(void)
+{
+	u32 hi, lo, chk;
+	do {
+		hi = mfspr(SPRN_ATBU);
+		lo = mfspr(SPRN_ATBL);
+		chk = mfspr(SPRN_ATBU);
+	} while (unlikely(hi != chk));
+	return ((u64)hi << 32) | (u64)lo;
+}
+
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/fsl_usdpaa.c
@@ -0,0 +1,1982 @@
+/* Copyright (C) 2008-2012 Freescale Semiconductor, Inc.
+ * Authors: Andy Fleming <afleming@freescale.com>
+ *	    Timur Tabi <timur@freescale.com>
+ *	    Geoff Thorpe <Geoff.Thorpe@freescale.com>
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/mm.h>
+#include <linux/of.h>
+#include <linux/memblock.h>
+#include <linux/slab.h>
+#include <linux/mman.h>
+#include <linux/of_reserved_mem.h>
+
+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64))
+#include <mm/mmu_decl.h>
+#endif
+
+#include "dpa_sys.h"
+#include <linux/fsl_usdpaa.h>
+#include "bman_low.h"
+#include "qman_low.h"
+
+/* Physical address range of the memory reservation, exported for mm/mem.c */
+static u64 phys_start;
+static u64 phys_size;
+static u64 arg_phys_size;
+
+/* PFN versions of the above */
+static unsigned long pfn_start;
+static unsigned long pfn_size;
+
+/* Memory reservations are manipulated under this spinlock (which is why 'refs'
+ * isn't atomic_t). */
+static DEFINE_SPINLOCK(mem_lock);
+
+/* The range of TLB1 indices */
+static unsigned int first_tlb;
+static unsigned int num_tlb = 1;
+static unsigned int current_tlb; /* loops around for fault handling */
+
+/* Memory reservation is represented as a list of 'mem_fragment's, some of which
+ * may be mapped. Unmapped fragments are always merged where possible. */
+static LIST_HEAD(mem_list);
+
+struct mem_mapping;
+
+/* Memory fragments are in 'mem_list'. */
+struct mem_fragment {
+	u64 base;
+	u64 len;
+	unsigned long pfn_base; /* PFN version of 'base' */
+	unsigned long pfn_len; /* PFN version of 'len' */
+	unsigned int refs; /* zero if unmapped */
+	u64 root_len; /* Size of the orignal fragment */
+	unsigned long root_pfn; /* PFN of the orignal fragment */
+	struct list_head list;
+	/* if mapped, flags+name captured at creation time */
+	u32 flags;
+	char name[USDPAA_DMA_NAME_MAX];
+	u64 map_len;
+	/* support multi-process locks per-memory-fragment. */
+	int has_locking;
+	wait_queue_head_t wq;
+	struct mem_mapping *owner;
+};
+
+/* Mappings of memory fragments in 'struct ctx'. These are created from
+ * ioctl(USDPAA_IOCTL_DMA_MAP), though the actual mapping then happens via a
+ * mmap(). */
+struct mem_mapping {
+	struct mem_fragment *root_frag;
+	u32 frag_count;
+	u64 total_size;
+	struct list_head list;
+	int refs;
+	void *virt_addr;
+};
+
+struct portal_mapping {
+	struct usdpaa_ioctl_portal_map user;
+	union {
+		struct qm_portal_config *qportal;
+		struct bm_portal_config *bportal;
+	};
+	/* Declare space for the portals in case the process
+	   exits unexpectedly and needs to be cleaned by the kernel */
+	union {
+		struct qm_portal qman_portal_low;
+		struct bm_portal bman_portal_low;
+	};
+	struct list_head list;
+	struct resource *phys;
+	struct iommu_domain *iommu_domain;
+};
+
+/* Track the DPAA resources the process is using */
+struct active_resource {
+	struct list_head list;
+	u32 id;
+	u32 num;
+	unsigned int refcount;
+};
+
+/* Per-FD state (which should also be per-process but we don't enforce that) */
+struct ctx {
+	/* Lock to protect the context */
+	spinlock_t lock;
+	/* Allocated resources get put here for accounting */
+	struct list_head resources[usdpaa_id_max];
+	/* list of DMA maps */
+	struct list_head maps;
+	/* list of portal maps */
+	struct list_head portals;
+};
+
+/* Different resource classes */
+static const struct alloc_backend {
+	enum usdpaa_id_type id_type;
+	int (*alloc)(u32 *, u32, u32, int);
+	void (*release)(u32 base, unsigned int count);
+	int (*reserve)(u32 base, unsigned int count);
+	const char *acronym;
+} alloc_backends[] = {
+	{
+		.id_type = usdpaa_id_fqid,
+		.alloc = qman_alloc_fqid_range,
+		.release = qman_release_fqid_range,
+		.reserve = qman_reserve_fqid_range,
+		.acronym = "FQID"
+	},
+	{
+		.id_type = usdpaa_id_bpid,
+		.alloc = bman_alloc_bpid_range,
+		.release = bman_release_bpid_range,
+		.reserve = bman_reserve_bpid_range,
+		.acronym = "BPID"
+	},
+	{
+		.id_type = usdpaa_id_qpool,
+		.alloc = qman_alloc_pool_range,
+		.release = qman_release_pool_range,
+		.reserve = qman_reserve_pool_range,
+		.acronym = "QPOOL"
+	},
+	{
+		.id_type = usdpaa_id_cgrid,
+		.alloc = qman_alloc_cgrid_range,
+		.release = qman_release_cgrid_range,
+		.acronym = "CGRID"
+	},
+	{
+		.id_type = usdpaa_id_ceetm0_lfqid,
+		.alloc = qman_alloc_ceetm0_lfqid_range,
+		.release = qman_release_ceetm0_lfqid_range,
+		.acronym = "CEETM0_LFQID"
+	},
+	{
+		.id_type = usdpaa_id_ceetm0_channelid,
+		.alloc = qman_alloc_ceetm0_channel_range,
+		.release = qman_release_ceetm0_channel_range,
+		.acronym = "CEETM0_LFQID"
+	},
+	{
+		.id_type = usdpaa_id_ceetm1_lfqid,
+		.alloc = qman_alloc_ceetm1_lfqid_range,
+		.release = qman_release_ceetm1_lfqid_range,
+		.acronym = "CEETM1_LFQID"
+	},
+	{
+		.id_type = usdpaa_id_ceetm1_channelid,
+		.alloc = qman_alloc_ceetm1_channel_range,
+		.release = qman_release_ceetm1_channel_range,
+		.acronym = "CEETM1_LFQID"
+	},
+	{
+		/* This terminates the array */
+		.id_type = usdpaa_id_max
+	}
+};
+
+/* Determines the largest acceptable page size for a given size
+   The sizes are determined by what the TLB1 acceptable page sizes are */
+static u32 largest_page_size(u32 size)
+{
+	int shift = 30; /* Start at 1G size */
+	if (size < 4096)
+		return 0;
+	do {
+		if (size >= (1<<shift))
+			return 1<<shift;
+		shift -= 2;
+	} while (shift >= 12); /* Up to 4k */
+	return 0;
+}
+
+/* Determine if value is power of 4 */
+static inline bool is_power_of_4(u64 x)
+{
+	if (x == 0 || ((x & (x - 1)) != 0))
+		return false;
+	return !!(x & 0x5555555555555555ull);
+}
+
+/* Helper for ioctl_dma_map() when we have a larger fragment than we need. This
+ * splits the fragment into 4 and returns the upper-most. (The caller can loop
+ * until it has a suitable fragment size.) */
+static struct mem_fragment *split_frag(struct mem_fragment *frag)
+{
+	struct mem_fragment *x[3];
+
+	x[0] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
+	x[1] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
+	x[2] = kmalloc(sizeof(struct mem_fragment), GFP_ATOMIC);
+	if (!x[0] || !x[1] || !x[2]) {
+		kfree(x[0]);
+		kfree(x[1]);
+		kfree(x[2]);
+		return NULL;
+	}
+	BUG_ON(frag->refs);
+	frag->len >>= 2;
+	frag->pfn_len >>= 2;
+	x[0]->base = frag->base + frag->len;
+	x[1]->base = x[0]->base + frag->len;
+	x[2]->base = x[1]->base + frag->len;
+	x[0]->len = x[1]->len = x[2]->len = frag->len;
+	x[0]->pfn_base = frag->pfn_base + frag->pfn_len;
+	x[1]->pfn_base = x[0]->pfn_base + frag->pfn_len;
+	x[2]->pfn_base = x[1]->pfn_base + frag->pfn_len;
+	x[0]->pfn_len = x[1]->pfn_len = x[2]->pfn_len = frag->pfn_len;
+	x[0]->refs = x[1]->refs = x[2]->refs = 0;
+	x[0]->root_len = x[1]->root_len = x[2]->root_len = frag->root_len;
+	x[0]->root_pfn = x[1]->root_pfn = x[2]->root_pfn = frag->root_pfn;
+	x[0]->name[0] = x[1]->name[0] = x[2]->name[0] = 0;
+	list_add_tail(&x[0]->list, &frag->list);
+	list_add_tail(&x[1]->list, &x[0]->list);
+	list_add_tail(&x[2]->list, &x[1]->list);
+	return x[2];
+}
+
+static __maybe_unused void dump_frags(void)
+{
+	struct mem_fragment *frag;
+	int i = 0;
+	list_for_each_entry(frag, &mem_list, list) {
+		pr_info("FRAG %d: base 0x%llx pfn_base 0x%lx len 0x%llx root_len 0x%llx root_pfn 0x%lx refs %d name %s\n",
+			i, frag->base, frag->pfn_base,
+			frag->len, frag->root_len, frag->root_pfn,
+			frag->refs, frag->name);
+		++i;
+	}
+}
+
+/* Walk the list of fragments and adjoin neighbouring segments if possible */
+static void compress_frags(void)
+{
+	/* Walk the fragment list and combine fragments */
+	struct mem_fragment *frag, *nxtfrag;
+	u64 len = 0;
+
+	int i, numfrags;
+
+
+	frag = list_entry(mem_list.next, struct mem_fragment, list);
+
+	while (&frag->list != &mem_list) {
+		/* Must combine consecutive fragemenst with
+		   same root_pfn such that they are power of 4 */
+		if (frag->refs != 0) {
+			frag = list_entry(frag->list.next,
+					  struct mem_fragment, list);
+			continue; /* Not this window */
+		}
+		len = frag->len;
+		numfrags = 0;
+		nxtfrag =  list_entry(frag->list.next,
+				      struct mem_fragment, list);
+		while (true) {
+			if (&nxtfrag->list == &mem_list) {
+				numfrags = 0;
+				break; /* End of list */
+			}
+			if (nxtfrag->refs) {
+				numfrags = 0;
+				break; /* In use still */
+			}
+			if (nxtfrag->root_pfn != frag->root_pfn) {
+				numfrags = 0;
+				break; /* Crosses root fragment boundary */
+			}
+			len += nxtfrag->len;
+			numfrags++;
+			if (is_power_of_4(len)) {
+				/* These fragments can be combined */
+				break;
+			}
+			nxtfrag =  list_entry(nxtfrag->list.next,
+					      struct mem_fragment, list);
+		}
+		if (numfrags == 0) {
+			frag = list_entry(frag->list.next,
+					  struct mem_fragment, list);
+			continue; /* try the next window */
+		}
+		for (i = 0; i < numfrags; i++) {
+			struct mem_fragment *todel =
+				list_entry(nxtfrag->list.prev,
+					   struct mem_fragment, list);
+			nxtfrag->len += todel->len;
+			nxtfrag->pfn_len += todel->pfn_len;
+			list_del(&todel->list);
+		}
+		/* Re evaluate the list, things may merge now */
+		frag = list_entry(mem_list.next, struct mem_fragment, list);
+	}
+}
+
+/* Hook from arch/powerpc/mm/mem.c */
+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size)
+{
+	struct mem_fragment *frag;
+	int idx = -1;
+	if ((pfn < pfn_start) || (pfn >= (pfn_start + pfn_size)))
+		return -1;
+	/* It's in-range, we need to find the fragment */
+	spin_lock(&mem_lock);
+	list_for_each_entry(frag, &mem_list, list) {
+		if ((pfn >= frag->pfn_base) && (pfn < (frag->pfn_base +
+						       frag->pfn_len))) {
+			*phys_addr = frag->base;
+			*size = frag->len;
+			idx = current_tlb++;
+			if (current_tlb >= (first_tlb + num_tlb))
+				current_tlb = first_tlb;
+			break;
+		}
+	}
+	spin_unlock(&mem_lock);
+	return idx;
+}
+
+static int usdpaa_open(struct inode *inode, struct file *filp)
+{
+	const struct alloc_backend *backend = &alloc_backends[0];
+	struct ctx *ctx = kmalloc(sizeof(struct ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+	filp->private_data = ctx;
+
+	while (backend->id_type != usdpaa_id_max) {
+		INIT_LIST_HEAD(&ctx->resources[backend->id_type]);
+		backend++;
+	}
+
+	INIT_LIST_HEAD(&ctx->maps);
+	INIT_LIST_HEAD(&ctx->portals);
+	spin_lock_init(&ctx->lock);
+
+	//filp->f_mapping->backing_dev_info = &directly_mappable_cdev_bdi;
+
+	return 0;
+}
+
+#define DQRR_MAXFILL 15
+
+/* Reset a QMan portal to its default state */
+static int init_qm_portal(struct qm_portal_config *config,
+			  struct qm_portal *portal)
+{
+	const struct qm_dqrr_entry *dqrr = NULL;
+	int i;
+
+	portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
+	portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
+
+	/* Make sure interrupts are inhibited */
+	qm_out(IIR, 1);
+
+	/* Initialize the DQRR.  This will stop any dequeue
+	   commands that are in progress */
+	if (qm_dqrr_init(portal, config, qm_dqrr_dpush, qm_dqrr_pvb,
+			 qm_dqrr_cdc, DQRR_MAXFILL)) {
+		pr_err("qm_dqrr_init() failed when trying to"
+		       " recover portal, portal will be leaked\n");
+		return 1;
+	}
+
+	/* Discard any entries on the DQRR */
+	/* If we consume the ring twice something is wrong */
+	for (i = 0; i < DQRR_MAXFILL * 2; i++) {
+		qm_dqrr_pvb_update(portal);
+		dqrr = qm_dqrr_current(portal);
+		if (!dqrr)
+			break;
+		qm_dqrr_cdc_consume_1ptr(portal, dqrr, 0);
+		qm_dqrr_pvb_update(portal);
+		qm_dqrr_next(portal);
+	}
+	/* Initialize the EQCR */
+	if (qm_eqcr_init(portal, qm_eqcr_pvb,
+			qm_eqcr_get_ci_stashing(portal), 1)) {
+		pr_err("Qman EQCR initialisation failed\n");
+		return 1;
+	}
+	/* initialize the MR */
+	if (qm_mr_init(portal, qm_mr_pvb, qm_mr_cci)) {
+		pr_err("Qman MR initialisation failed\n");
+		return 1;
+	}
+	qm_mr_pvb_update(portal);
+	while (qm_mr_current(portal)) {
+		qm_mr_next(portal);
+		qm_mr_cci_consume_to_current(portal);
+		qm_mr_pvb_update(portal);
+	}
+
+	if (qm_mc_init(portal)) {
+		pr_err("Qman MC initialisation failed\n");
+		return 1;
+	}
+	return 0;
+}
+
+static int init_bm_portal(struct bm_portal_config *config,
+			  struct bm_portal *portal)
+{
+	portal->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
+	portal->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
+
+	if (bm_rcr_init(portal, bm_rcr_pvb, bm_rcr_cce)) {
+		pr_err("Bman RCR initialisation failed\n");
+	return 1;
+	}
+	if (bm_mc_init(portal)) {
+		pr_err("Bman MC initialisation failed\n");
+		return 1;
+	}
+	return 0;
+}
+
+/* Function that will scan all FQ's in the system.  For each FQ that is not
+   OOS it will call the check_channel helper to determine if the FQ should
+   be torn down.  If the check_channel helper returns true the FQ will be
+   transitioned to the OOS state */
+static int qm_check_and_destroy_fqs(struct qm_portal *portal, void *ctx,
+				    bool (*check_channel)(void*, u32))
+{
+	u32 fq_id = 0;
+	while (1) {
+		struct qm_mc_command *mcc;
+		struct qm_mc_result *mcr;
+		u8 state;
+		u32 channel;
+
+		/* Determine the channel for the FQID */
+		mcc = qm_mc_start(portal);
+		mcc->queryfq.fqid = fq_id;
+		qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ);
+		while (!(mcr = qm_mc_result(portal)))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK)
+			   == QM_MCR_VERB_QUERYFQ);
+		if (mcr->result != QM_MCR_RESULT_OK)
+			break; /* End of valid FQIDs */
+
+		channel = mcr->queryfq.fqd.dest.channel;
+		/* Determine the state of the FQID */
+		mcc = qm_mc_start(portal);
+		mcc->queryfq_np.fqid = fq_id;
+		qm_mc_commit(portal, QM_MCC_VERB_QUERYFQ_NP);
+		while (!(mcr = qm_mc_result(portal)))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK)
+			   == QM_MCR_VERB_QUERYFQ_NP);
+		state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
+		if (state == QM_MCR_NP_STATE_OOS)
+			/* Already OOS, no need to do anymore checks */
+			goto next;
+
+		if (check_channel(ctx, channel))
+			qm_shutdown_fq(&portal, 1, fq_id);
+ next:
+		++fq_id;
+	}
+	return 0;
+}
+
+static bool check_channel_device(void *_ctx, u32 channel)
+{
+	struct ctx *ctx = _ctx;
+	struct portal_mapping *portal, *tmpportal;
+	struct active_resource *res;
+
+	/* See if the FQ is destined for one of the portals we're cleaning up */
+	list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
+		if (portal->user.type == usdpaa_portal_qman) {
+			if (portal->qportal->public_cfg.channel == channel) {
+				/* This FQs destination is a portal
+				   we're cleaning, send a retire */
+				return true;
+			}
+		}
+	}
+
+	/* Check the pool channels that will be released as well */
+	list_for_each_entry(res, &ctx->resources[usdpaa_id_qpool], list) {
+		if ((res->id >= channel) &&
+		    ((res->id + res->num - 1) <= channel))
+			return true;
+	}
+	return false;
+}
+
+static bool check_portal_channel(void *ctx, u32 channel)
+{
+	u32 portal_channel = *(u32 *)ctx;
+	if (portal_channel == channel) {
+		/* This FQs destination is a portal
+		   we're cleaning, send a retire */
+		return true;
+	}
+	return false;
+}
+
+
+
+
+static int usdpaa_release(struct inode *inode, struct file *filp)
+{
+	struct ctx *ctx = filp->private_data;
+	struct mem_mapping *map, *tmpmap;
+	struct portal_mapping *portal, *tmpportal;
+	const struct alloc_backend *backend = &alloc_backends[0];
+	struct active_resource *res;
+	struct qm_portal *qm_cleanup_portal = NULL;
+	struct bm_portal *bm_cleanup_portal = NULL;
+	struct qm_portal_config *qm_alloced_portal = NULL;
+	struct bm_portal_config *bm_alloced_portal = NULL;
+
+	struct qm_portal *portal_array[qman_portal_max];
+	int portal_count = 0;
+
+	/* Ensure the release operation cannot be migrated to another
+	   CPU as CPU specific variables may be needed during cleanup */
+#ifdef CONFIG_PREEMPT_RT_FULL
+	migrate_disable();
+#endif
+	/* The following logic is used to recover resources that were not
+	   correctly released by the process that is closing the FD.
+	   Step 1: syncronize the HW with the qm_portal/bm_portal structures
+	   in the kernel
+	*/
+
+	list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
+		/* Try to recover any portals that weren't shut down */
+		if (portal->user.type == usdpaa_portal_qman) {
+			portal_array[portal_count] = &portal->qman_portal_low;
+			++portal_count;
+			init_qm_portal(portal->qportal,
+				       &portal->qman_portal_low);
+			if (!qm_cleanup_portal) {
+				qm_cleanup_portal = &portal->qman_portal_low;
+			} else {
+				/* Clean FQs on the dedicated channel */
+				u32 chan = portal->qportal->public_cfg.channel;
+				qm_check_and_destroy_fqs(
+					&portal->qman_portal_low, &chan,
+					check_portal_channel);
+			}
+		} else {
+			/* BMAN */
+			init_bm_portal(portal->bportal,
+				       &portal->bman_portal_low);
+			if (!bm_cleanup_portal)
+				bm_cleanup_portal = &portal->bman_portal_low;
+		}
+	}
+	/* If no portal was found, allocate one for cleanup */
+	if (!qm_cleanup_portal) {
+		qm_alloced_portal = qm_get_unused_portal();
+		if (!qm_alloced_portal) {
+			pr_crit("No QMan portal avalaible for cleanup\n");
+#ifdef CONFIG_PREEMPT_RT_FULL
+			migrate_enable();
+#endif
+			return -1;
+		}
+		qm_cleanup_portal = kmalloc(sizeof(struct qm_portal),
+					    GFP_KERNEL);
+		if (!qm_cleanup_portal) {
+#ifdef CONFIG_PREEMPT_RT_FULL
+			migrate_enable();
+#endif
+			return -ENOMEM;
+		}
+		init_qm_portal(qm_alloced_portal, qm_cleanup_portal);
+		portal_array[portal_count] = qm_cleanup_portal;
+		++portal_count;
+	}
+	if (!bm_cleanup_portal) {
+		bm_alloced_portal = bm_get_unused_portal();
+		if (!bm_alloced_portal) {
+			pr_crit("No BMan portal avalaible for cleanup\n");
+#ifdef CONFIG_PREEMPT_RT_FULL
+			migrate_enable();
+#endif
+			return -1;
+		}
+		bm_cleanup_portal = kmalloc(sizeof(struct bm_portal),
+					    GFP_KERNEL);
+		if (!bm_cleanup_portal) {
+#ifdef CONFIG_PREEMPT_RT_FULL
+			migrate_enable();
+#endif
+			return -ENOMEM;
+		}
+		init_bm_portal(bm_alloced_portal, bm_cleanup_portal);
+	}
+
+	/* OOS the FQs associated with this process */
+	qm_check_and_destroy_fqs(qm_cleanup_portal, ctx, check_channel_device);
+
+	while (backend->id_type != usdpaa_id_max) {
+		int leaks = 0;
+		list_for_each_entry(res, &ctx->resources[backend->id_type],
+				    list) {
+			if (backend->id_type == usdpaa_id_fqid) {
+				int i = 0;
+				for (; i < res->num; i++) {
+					/* Clean FQs with the cleanup portal */
+					qm_shutdown_fq(portal_array,
+						       portal_count,
+						       res->id + i);
+				}
+			}
+			leaks += res->num;
+			backend->release(res->id, res->num);
+		}
+		if (leaks)
+			pr_crit("USDPAA process leaking %d %s%s\n", leaks,
+				backend->acronym, (leaks > 1) ? "s" : "");
+		backend++;
+	}
+	/* Release any DMA regions */
+	spin_lock(&mem_lock);
+	list_for_each_entry_safe(map, tmpmap, &ctx->maps, list) {
+		struct mem_fragment *current_frag = map->root_frag;
+		int i;
+		if (map->root_frag->has_locking &&
+		    (map->root_frag->owner == map)) {
+			map->root_frag->owner = NULL;
+			wake_up(&map->root_frag->wq);
+		}
+		/* Check each fragment and merge if the ref count is 0 */
+		for (i = 0; i < map->frag_count; i++) {
+			--current_frag->refs;
+			current_frag = list_entry(current_frag->list.prev,
+						  struct mem_fragment, list);
+		}
+
+		compress_frags();
+		list_del(&map->list);
+		kfree(map);
+	}
+	spin_unlock(&mem_lock);
+
+	/* Return portals */
+	list_for_each_entry_safe(portal, tmpportal, &ctx->portals, list) {
+		if (portal->user.type == usdpaa_portal_qman) {
+			/* Give the portal back to the allocator */
+			init_qm_portal(portal->qportal,
+				       &portal->qman_portal_low);
+			qm_put_unused_portal(portal->qportal);
+		} else {
+			init_bm_portal(portal->bportal,
+				       &portal->bman_portal_low);
+			bm_put_unused_portal(portal->bportal);
+		}
+		list_del(&portal->list);
+		kfree(portal);
+	}
+	if (qm_alloced_portal) {
+		qm_put_unused_portal(qm_alloced_portal);
+		kfree(qm_cleanup_portal);
+	}
+	if (bm_alloced_portal) {
+		bm_put_unused_portal(bm_alloced_portal);
+		kfree(bm_cleanup_portal);
+	}
+
+	kfree(ctx);
+#ifdef CONFIG_PREEMPT_RT_FULL
+	migrate_enable();
+#endif
+	return 0;
+}
+
+static int check_mmap_dma(struct ctx *ctx, struct vm_area_struct *vma,
+			  int *match, unsigned long *pfn)
+{
+	struct mem_mapping *map;
+
+	list_for_each_entry(map, &ctx->maps, list) {
+		int i;
+		struct mem_fragment *frag = map->root_frag;
+
+		for (i = 0; i < map->frag_count; i++) {
+			if (frag->pfn_base == vma->vm_pgoff) {
+				*match = 1;
+				*pfn = frag->pfn_base;
+				return 0;
+			}
+			frag = list_entry(frag->list.next, struct mem_fragment,
+					  list);
+		}
+	}
+	*match = 0;
+	return 0;
+}
+
+static int check_mmap_resource(struct resource *res, struct vm_area_struct *vma,
+			       int *match, unsigned long *pfn)
+{
+	*pfn = res->start >> PAGE_SHIFT;
+	if (*pfn == vma->vm_pgoff) {
+		*match = 1;
+		if ((vma->vm_end - vma->vm_start) != resource_size(res))
+			return -EINVAL;
+	} else
+		*match = 0;
+	return 0;
+}
+
+static int check_mmap_portal(struct ctx *ctx, struct vm_area_struct *vma,
+			      int *match, unsigned long *pfn)
+{
+	struct portal_mapping *portal;
+	int ret;
+
+	list_for_each_entry(portal, &ctx->portals, list) {
+		ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CE], vma,
+					  match, pfn);
+		if (*match) {
+			vma->vm_page_prot =
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+				pgprot_cached_ns(vma->vm_page_prot);
+#else
+				pgprot_cached_noncoherent(vma->vm_page_prot);
+#endif
+			return ret;
+		}
+		ret = check_mmap_resource(&portal->phys[DPA_PORTAL_CI], vma,
+					  match, pfn);
+		if (*match) {
+			vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+			return ret;
+		}
+	}
+	*match = 0;
+	return 0;
+}
+
+static int usdpaa_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct ctx *ctx = filp->private_data;
+	unsigned long pfn = 0;
+	int match, ret;
+
+	spin_lock(&mem_lock);
+	ret = check_mmap_dma(ctx, vma, &match, &pfn);
+	if (!match)
+		ret = check_mmap_portal(ctx, vma, &match, &pfn);
+	spin_unlock(&mem_lock);
+	if (!match)
+		return -EINVAL;
+	if (!ret)
+		ret = remap_pfn_range(vma, vma->vm_start, pfn,
+				      vma->vm_end - vma->vm_start,
+				      vma->vm_page_prot);
+	return ret;
+}
+
+/* Return the nearest rounded-up address >= 'addr' that is 'sz'-aligned. 'sz'
+ * must be a power of 2, but both 'addr' and 'sz' can be expressions. */
+#define USDPAA_MEM_ROUNDUP(addr, sz) \
+	({ \
+		unsigned long foo_align = (sz) - 1; \
+		((addr) + foo_align) & ~foo_align; \
+	})
+/* Searching for a size-aligned virtual address range starting from 'addr' */
+static unsigned long usdpaa_get_unmapped_area(struct file *file,
+					      unsigned long addr,
+					      unsigned long len,
+					      unsigned long pgoff,
+					      unsigned long flags)
+{
+	struct vm_area_struct *vma;
+
+	if (len % PAGE_SIZE)
+		return -EINVAL;
+	if (!len)
+		return -EINVAL;
+
+	/* Need to align the address to the largest pagesize of the mapping
+	 * because the MMU requires the virtual address to have the same
+	 * alignment as the physical address */
+	addr = USDPAA_MEM_ROUNDUP(addr, largest_page_size(len));
+	vma = find_vma(current->mm, addr);
+	/* Keep searching until we reach the end of currently-used virtual
+	 * address-space or we find a big enough gap. */
+	while (vma) {
+		if ((addr + len) < vma->vm_start)
+			return addr;
+
+		addr = USDPAA_MEM_ROUNDUP(vma->vm_end,  largest_page_size(len));
+		vma = vma->vm_next;
+	}
+	if ((TASK_SIZE - len) < addr)
+		return -ENOMEM;
+	return addr;
+}
+
+static long ioctl_id_alloc(struct ctx *ctx, void __user *arg)
+{
+	struct usdpaa_ioctl_id_alloc i;
+	const struct alloc_backend *backend;
+	struct active_resource *res;
+	int ret = copy_from_user(&i, arg, sizeof(i));
+	if (ret)
+		return ret;
+	if ((i.id_type >= usdpaa_id_max) || !i.num)
+		return -EINVAL;
+	backend = &alloc_backends[i.id_type];
+	/* Allocate the required resource type */
+	ret = backend->alloc(&i.base, i.num, i.align, i.partial);
+	if (ret < 0)
+		return ret;
+	i.num = ret;
+	/* Copy the result to user-space */
+	ret = copy_to_user(arg, &i, sizeof(i));
+	if (ret) {
+		backend->release(i.base, i.num);
+		return ret;
+	}
+	/* Assign the allocated range to the FD accounting */
+	res = kmalloc(sizeof(*res), GFP_KERNEL);
+	if (!res) {
+		backend->release(i.base, i.num);
+		return -ENOMEM;
+	}
+	spin_lock(&ctx->lock);
+	res->id = i.base;
+	res->num = i.num;
+	res->refcount = 1;
+	list_add(&res->list, &ctx->resources[i.id_type]);
+	spin_unlock(&ctx->lock);
+	return 0;
+}
+
+static long ioctl_id_release(struct ctx *ctx, void __user *arg)
+{
+	struct usdpaa_ioctl_id_release i;
+	const struct alloc_backend *backend;
+	struct active_resource *tmp, *pos;
+
+	int ret = copy_from_user(&i, arg, sizeof(i));
+	if (ret)
+		return ret;
+	if ((i.id_type >= usdpaa_id_max) || !i.num)
+		return -EINVAL;
+	backend = &alloc_backends[i.id_type];
+	/* Pull the range out of the FD accounting - the range is valid iff this
+	 * succeeds. */
+	spin_lock(&ctx->lock);
+	list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) {
+		if (pos->id == i.base && pos->num == i.num) {
+			pos->refcount--;
+			if (pos->refcount) {
+				spin_unlock(&ctx->lock);
+				return 0; /* Still being used */
+			}
+			list_del(&pos->list);
+			kfree(pos);
+			spin_unlock(&ctx->lock);
+			goto found;
+		}
+	}
+	/* Failed to find the resource */
+	spin_unlock(&ctx->lock);
+	pr_err("Couldn't find resource type %d base 0x%x num %d\n",
+	       i.id_type, i.base, i.num);
+	return -EINVAL;
+found:
+	/* Release the resource to the backend */
+	backend->release(i.base, i.num);
+	return 0;
+}
+
+static long ioctl_id_reserve(struct ctx *ctx, void __user *arg)
+{
+	struct usdpaa_ioctl_id_reserve i;
+	const struct alloc_backend *backend;
+	struct active_resource *tmp, *pos;
+
+	int ret = copy_from_user(&i, arg, sizeof(i));
+	if (ret)
+		return ret;
+	if ((i.id_type >= usdpaa_id_max) || !i.num)
+		return -EINVAL;
+	backend = &alloc_backends[i.id_type];
+	if (!backend->reserve)
+		return -EINVAL;
+	/* Pull the range out of the FD accounting - the range is valid iff this
+	 * succeeds. */
+	spin_lock(&ctx->lock);
+	list_for_each_entry_safe(pos, tmp, &ctx->resources[i.id_type], list) {
+		if (pos->id == i.base && pos->num == i.num) {
+			pos->refcount++;
+			spin_unlock(&ctx->lock);
+			return 0;
+		}
+	}
+
+	/* Failed to find the resource */
+	spin_unlock(&ctx->lock);
+
+	/* Reserve the resource in the backend */
+	ret = backend->reserve(i.base, i.num);
+	if (ret)
+		return ret;
+	/* Assign the reserved range to the FD accounting */
+	pos = kmalloc(sizeof(*pos), GFP_KERNEL);
+	if (!pos) {
+		backend->release(i.base, i.num);
+		return -ENOMEM;
+	}
+	spin_lock(&ctx->lock);
+	pos->id = i.base;
+	pos->num = i.num;
+	pos->refcount = 1;
+	list_add(&pos->list, &ctx->resources[i.id_type]);
+	spin_unlock(&ctx->lock);
+	return 0;
+}
+
+static long ioctl_dma_map(struct file *fp, struct ctx *ctx,
+			  struct usdpaa_ioctl_dma_map *i)
+{
+	struct mem_fragment *frag, *start_frag, *next_frag;
+	struct mem_mapping *map, *tmp;
+	int ret = 0;
+	u32 largest_page, so_far = 0;
+	int frag_count = 0;
+	unsigned long next_addr = PAGE_SIZE, populate;
+
+	/* error checking to ensure values copied from user space are valid */
+	if (i->len % PAGE_SIZE)
+		return -EINVAL;
+
+	map = kmalloc(sizeof(*map), GFP_KERNEL);
+	if (!map)
+		return -ENOMEM;
+
+	spin_lock(&mem_lock);
+	if (i->flags & USDPAA_DMA_FLAG_SHARE) {
+		list_for_each_entry(frag, &mem_list, list) {
+			if (frag->refs && (frag->flags &
+					   USDPAA_DMA_FLAG_SHARE) &&
+					!strncmp(i->name, frag->name,
+						 USDPAA_DMA_NAME_MAX)) {
+				/* Matching entry */
+				if ((i->flags & USDPAA_DMA_FLAG_CREATE) &&
+				    !(i->flags & USDPAA_DMA_FLAG_LAZY)) {
+					ret = -EBUSY;
+					goto out;
+				}
+
+				/* Check to ensure size matches record */
+				if (i->len != frag->map_len && i->len) {
+					pr_err("ioctl_dma_map() Size requested does not match %s and is none zero. This usage will be disallowed in future release\n",
+					frag->name);
+				}
+
+				/* Check if this has already been mapped
+				   to this process */
+				list_for_each_entry(tmp, &ctx->maps, list)
+					if (tmp->root_frag == frag) {
+						/* Already mapped, just need to
+						   inc ref count */
+						tmp->refs++;
+						kfree(map);
+						i->did_create = 0;
+						i->len = tmp->total_size;
+						i->phys_addr = frag->base;
+						i->ptr = tmp->virt_addr;
+						spin_unlock(&mem_lock);
+						return 0;
+					}
+				/* Matching entry - just need to map */
+				i->has_locking = frag->has_locking;
+				i->did_create = 0;
+				i->len = frag->map_len;
+				start_frag = frag;
+				goto do_map;
+			}
+		}
+		/* No matching entry */
+		if (!(i->flags & USDPAA_DMA_FLAG_CREATE)) {
+			pr_err("ioctl_dma_map() No matching entry\n");
+			ret = -ENOMEM;
+			goto out;
+		}
+	}
+	/* New fragment required, size must be provided. */
+	if (!i->len) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	/* Find one of more contiguous fragments that satisfy the total length
+	   trying to minimize the number of fragments
+	   compute the largest page size that the allocation could use */
+	largest_page = largest_page_size(i->len);
+	start_frag = NULL;
+	while (largest_page &&
+	       largest_page <= largest_page_size(phys_size) &&
+	       start_frag == NULL) {
+		/* Search the list for a frag of that size */
+		list_for_each_entry(frag, &mem_list, list) {
+			if (!frag->refs && (frag->len == largest_page)) {
+				/* See if the next x fragments are free
+				   and can accomidate the size */
+				u32 found_size = largest_page;
+				next_frag = list_entry(frag->list.prev,
+						       struct mem_fragment,
+						       list);
+				/* If the fragement is too small check
+				   if the neighbours cab support it */
+				while (found_size < i->len) {
+					if (&mem_list == &next_frag->list)
+						break; /* End of list */
+					if (next_frag->refs != 0 ||
+					    next_frag->len == 0)
+						break; /* not enough space */
+					found_size += next_frag->len;
+					next_frag = list_entry(
+						next_frag->list.prev,
+						struct mem_fragment,
+						list);
+				}
+				if (found_size >= i->len) {
+					/* Success! there is enough contigous
+					   free space */
+					start_frag = frag;
+					break;
+				}
+			}
+		} /* next frag loop */
+		/* Couldn't statisfy the request with this
+		   largest page size, try a smaller one */
+		largest_page <<= 2;
+	}
+	if (start_frag == NULL) {
+		/* Couldn't find proper amount of space */
+		ret = -ENOMEM;
+		goto out;
+	}
+	i->did_create = 1;
+do_map:
+	/* Verify there is sufficient space to do the mapping */
+	down_write(&current->mm->mmap_sem);
+	next_addr = usdpaa_get_unmapped_area(fp, next_addr, i->len, 0, 0);
+	up_write(&current->mm->mmap_sem);
+
+	if (next_addr & ~PAGE_MASK) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	/* We may need to divide the final fragment to accomidate the mapping */
+	next_frag = start_frag;
+	while (so_far != i->len) {
+		BUG_ON(next_frag->len == 0);
+		while ((next_frag->len + so_far) > i->len) {
+			/* Split frag until they match */
+			split_frag(next_frag);
+		}
+		so_far += next_frag->len;
+		next_frag->refs++;
+		++frag_count;
+		next_frag = list_entry(next_frag->list.prev,
+				       struct mem_fragment, list);
+	}
+	if (i->did_create) {
+		size_t name_len = 0;
+		start_frag->flags = i->flags;
+		strncpy(start_frag->name, i->name, USDPAA_DMA_NAME_MAX);
+		name_len = strnlen(start_frag->name, USDPAA_DMA_NAME_MAX);
+		if (name_len >= USDPAA_DMA_NAME_MAX) {
+			ret = -EFAULT;
+			goto out;
+		}
+		start_frag->map_len = i->len;
+		start_frag->has_locking = i->has_locking;
+		init_waitqueue_head(&start_frag->wq);
+		start_frag->owner = NULL;
+	}
+
+	/* Setup the map entry */
+	map->root_frag = start_frag;
+	map->total_size = i->len;
+	map->frag_count = frag_count;
+	map->refs = 1;
+	list_add(&map->list, &ctx->maps);
+	i->phys_addr = start_frag->base;
+out:
+	spin_unlock(&mem_lock);
+
+	if (!ret) {
+		unsigned long longret;
+		down_write(&current->mm->mmap_sem);
+		longret = do_mmap_pgoff(fp, next_addr, map->total_size,
+					PROT_READ |
+					(i->flags &
+					 USDPAA_DMA_FLAG_RDONLY ? 0
+					 : PROT_WRITE),
+					MAP_SHARED,
+					start_frag->pfn_base,
+					&populate);
+		up_write(&current->mm->mmap_sem);
+		if (longret & ~PAGE_MASK) {
+			ret = (int)longret;
+		} else {
+			i->ptr = (void *)longret;
+			map->virt_addr = i->ptr;
+		}
+	} else
+		kfree(map);
+	return ret;
+}
+
+static long ioctl_dma_unmap(struct ctx *ctx, void __user *arg)
+{
+	struct mem_mapping *map;
+	struct vm_area_struct *vma;
+	int ret, i;
+	struct mem_fragment *current_frag;
+	size_t sz;
+	unsigned long base;
+	unsigned long vaddr;
+
+	down_write(&current->mm->mmap_sem);
+	vma = find_vma(current->mm, (unsigned long)arg);
+	if (!vma || (vma->vm_start > (unsigned long)arg)) {
+		up_write(&current->mm->mmap_sem);
+		return -EFAULT;
+	}
+	spin_lock(&mem_lock);
+	list_for_each_entry(map, &ctx->maps, list) {
+		if (map->root_frag->pfn_base == vma->vm_pgoff) {
+			/* Drop the map lock if we hold it */
+			if (map->root_frag->has_locking &&
+					(map->root_frag->owner == map)) {
+				map->root_frag->owner = NULL;
+				wake_up(&map->root_frag->wq);
+			}
+			goto map_match;
+		}
+	}
+	/* Failed to find a matching mapping for this process */
+	ret = -EFAULT;
+	spin_unlock(&mem_lock);
+	goto out;
+map_match:
+	map->refs--;
+	if (map->refs != 0) {
+		/* Another call the dma_map is referencing this */
+		ret = 0;
+		spin_unlock(&mem_lock);
+		goto out;
+	}
+
+	current_frag = map->root_frag;
+	vaddr = (unsigned long) map->virt_addr;
+	for (i = 0; i < map->frag_count; i++) {
+		DPA_ASSERT(current_frag->refs > 0);
+		--current_frag->refs;
+#if !(defined(CONFIG_ARM) || defined(CONFIG_ARM64))
+		/*
+		 * Make sure we invalidate the TLB entry for
+		 * this fragment, otherwise a remap of a different
+		 * page to this vaddr would give acces to an
+		 * incorrect piece of memory
+		 */
+		cleartlbcam(vaddr, mfspr(SPRN_PID));
+#endif
+		vaddr += current_frag->len;
+		current_frag = list_entry(current_frag->list.prev,
+					  struct mem_fragment, list);
+	}
+	map->root_frag->name[0] = 0;
+	list_del(&map->list);
+	compress_frags();
+	spin_unlock(&mem_lock);
+
+	base = vma->vm_start;
+	sz = vma->vm_end - vma->vm_start;
+	do_munmap(current->mm, base, sz);
+	ret = 0;
+ out:
+	up_write(&current->mm->mmap_sem);
+	return ret;
+}
+
+static long ioctl_dma_stats(struct ctx *ctx, void __user *arg)
+{
+	struct mem_fragment *frag;
+	struct usdpaa_ioctl_dma_used result;
+
+	result.free_bytes = 0;
+	result.total_bytes = phys_size;
+
+	list_for_each_entry(frag, &mem_list, list) {
+		if (frag->refs == 0)
+			result.free_bytes += frag->len;
+	}
+
+	return copy_to_user(arg, &result, sizeof(result)); }
+
+static int test_lock(struct mem_mapping *map)
+{
+	int ret = 0;
+	spin_lock(&mem_lock);
+	if (!map->root_frag->owner) {
+		map->root_frag->owner = map;
+		ret = 1;
+	}
+	spin_unlock(&mem_lock);
+	return ret;
+}
+
+static long ioctl_dma_lock(struct ctx *ctx, void __user *arg)
+{
+	struct mem_mapping *map;
+	struct vm_area_struct *vma;
+
+	down_read(&current->mm->mmap_sem);
+	vma = find_vma(current->mm, (unsigned long)arg);
+	if (!vma || (vma->vm_start > (unsigned long)arg)) {
+		up_read(&current->mm->mmap_sem);
+		return -EFAULT;
+	}
+	spin_lock(&mem_lock);
+	list_for_each_entry(map, &ctx->maps, list) {
+		if (map->root_frag->pfn_base == vma->vm_pgoff)
+			goto map_match;
+	}
+	map = NULL;
+map_match:
+	spin_unlock(&mem_lock);
+	up_read(&current->mm->mmap_sem);
+
+	if (!map)
+		return -EFAULT;
+	if (!map->root_frag->has_locking)
+		return -ENODEV;
+	return wait_event_interruptible(map->root_frag->wq, test_lock(map));
+}
+
+static long ioctl_dma_unlock(struct ctx *ctx, void __user *arg)
+{
+	struct mem_mapping *map;
+	struct vm_area_struct *vma;
+	int ret;
+
+	down_read(&current->mm->mmap_sem);
+	vma = find_vma(current->mm, (unsigned long)arg);
+	if (!vma || (vma->vm_start > (unsigned long)arg))
+		ret = -EFAULT;
+	else {
+		spin_lock(&mem_lock);
+		list_for_each_entry(map, &ctx->maps, list) {
+			if (map->root_frag->pfn_base == vma->vm_pgoff) {
+				if (!map->root_frag->has_locking)
+					ret = -ENODEV;
+				else if (map->root_frag->owner == map) {
+					map->root_frag->owner = NULL;
+					wake_up(&map->root_frag->wq);
+					ret = 0;
+				} else
+					ret = -EBUSY;
+				goto map_match;
+			}
+		}
+		ret = -EINVAL;
+map_match:
+		spin_unlock(&mem_lock);
+	}
+	up_read(&current->mm->mmap_sem);
+	return ret;
+}
+
+static int portal_mmap(struct file *fp, struct resource *res, void **ptr)
+{
+	unsigned long longret = 0, populate;
+	resource_size_t len;
+
+	down_write(&current->mm->mmap_sem);
+	len = resource_size(res);
+	if (len != (unsigned long)len)
+		return -EINVAL;
+	longret = do_mmap_pgoff(fp, PAGE_SIZE, (unsigned long)len,
+				PROT_READ | PROT_WRITE, MAP_SHARED,
+				res->start >> PAGE_SHIFT, &populate);
+	up_write(&current->mm->mmap_sem);
+
+	if (longret & ~PAGE_MASK)
+		return (int)longret;
+
+	*ptr = (void *) longret;
+	return 0;
+}
+
+static void portal_munmap(struct resource *res, void  *ptr)
+{
+	down_write(&current->mm->mmap_sem);
+	do_munmap(current->mm, (unsigned long)ptr, resource_size(res));
+	up_write(&current->mm->mmap_sem);
+}
+
+static long ioctl_portal_map(struct file *fp, struct ctx *ctx,
+			     struct usdpaa_ioctl_portal_map  *arg)
+{
+	struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
+	int ret;
+
+	if (!mapping)
+		return -ENOMEM;
+
+	mapping->user = *arg;
+	mapping->iommu_domain = NULL;
+
+	if (mapping->user.type == usdpaa_portal_qman) {
+		mapping->qportal =
+			qm_get_unused_portal_idx(mapping->user.index);
+		if (!mapping->qportal) {
+			ret = -ENODEV;
+			goto err_get_portal;
+		}
+		mapping->phys = &mapping->qportal->addr_phys[0];
+		mapping->user.channel = mapping->qportal->public_cfg.channel;
+		mapping->user.pools = mapping->qportal->public_cfg.pools;
+		mapping->user.index = mapping->qportal->public_cfg.index;
+	} else if (mapping->user.type == usdpaa_portal_bman) {
+		mapping->bportal =
+			bm_get_unused_portal_idx(mapping->user.index);
+		if (!mapping->bportal) {
+			ret = -ENODEV;
+			goto err_get_portal;
+		}
+		mapping->phys = &mapping->bportal->addr_phys[0];
+		mapping->user.index = mapping->bportal->public_cfg.index;
+	} else {
+		ret = -EINVAL;
+		goto err_copy_from_user;
+	}
+	/* Need to put pcfg in ctx's list before the mmaps because the mmap
+	 * handlers look it up. */
+	spin_lock(&mem_lock);
+	list_add(&mapping->list, &ctx->portals);
+	spin_unlock(&mem_lock);
+	ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CE],
+			  &mapping->user.addr.cena);
+	if (ret)
+		goto err_mmap_cena;
+	ret = portal_mmap(fp, &mapping->phys[DPA_PORTAL_CI],
+			  &mapping->user.addr.cinh);
+	if (ret)
+		goto err_mmap_cinh;
+	*arg = mapping->user;
+	return ret;
+
+err_mmap_cinh:
+	portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena);
+err_mmap_cena:
+	if ((mapping->user.type == usdpaa_portal_qman) && mapping->qportal)
+		qm_put_unused_portal(mapping->qportal);
+	else if ((mapping->user.type == usdpaa_portal_bman) && mapping->bportal)
+		bm_put_unused_portal(mapping->bportal);
+	spin_lock(&mem_lock);
+	list_del(&mapping->list);
+	spin_unlock(&mem_lock);
+err_get_portal:
+err_copy_from_user:
+	kfree(mapping);
+	return ret;
+}
+
+static long ioctl_portal_unmap(struct ctx *ctx, struct usdpaa_portal_map *i)
+{
+	struct portal_mapping *mapping;
+	struct vm_area_struct *vma;
+	unsigned long pfn;
+	u32 channel;
+
+	/* Get the PFN corresponding to one of the virt addresses */
+	down_read(&current->mm->mmap_sem);
+	vma = find_vma(current->mm, (unsigned long)i->cinh);
+	if (!vma || (vma->vm_start > (unsigned long)i->cinh)) {
+		up_read(&current->mm->mmap_sem);
+		return -EFAULT;
+	}
+	pfn = vma->vm_pgoff;
+	up_read(&current->mm->mmap_sem);
+
+	/* Find the corresponding portal */
+	spin_lock(&mem_lock);
+	list_for_each_entry(mapping, &ctx->portals, list) {
+		if (pfn == (mapping->phys[DPA_PORTAL_CI].start >> PAGE_SHIFT))
+			goto found;
+	}
+	mapping = NULL;
+found:
+	if (mapping)
+		list_del(&mapping->list);
+	spin_unlock(&mem_lock);
+	if (!mapping)
+		return -ENODEV;
+	portal_munmap(&mapping->phys[DPA_PORTAL_CI], mapping->user.addr.cinh);
+	portal_munmap(&mapping->phys[DPA_PORTAL_CE], mapping->user.addr.cena);
+	if (mapping->user.type == usdpaa_portal_qman) {
+		init_qm_portal(mapping->qportal,
+				       &mapping->qman_portal_low);
+
+		/* Tear down any FQs this portal is referencing */
+		channel = mapping->qportal->public_cfg.channel;
+		qm_check_and_destroy_fqs(&mapping->qman_portal_low,
+					 &channel,
+					 check_portal_channel);
+		qm_put_unused_portal(mapping->qportal);
+	} else if (mapping->user.type == usdpaa_portal_bman) {
+		init_bm_portal(mapping->bportal,
+			       &mapping->bman_portal_low);
+		bm_put_unused_portal(mapping->bportal);
+	}
+	kfree(mapping);
+	return 0;
+}
+
+static void portal_config_pamu(struct qm_portal_config *pcfg, uint8_t sdest,
+			       uint32_t cpu, uint32_t cache, uint32_t window)
+{
+#ifdef CONFIG_FSL_PAMU
+	int ret;
+	int window_count = 1;
+	struct iommu_domain_geometry geom_attr;
+	struct pamu_stash_attribute stash_attr;
+
+	pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type);
+	if (!pcfg->iommu_domain) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed",
+			   __func__);
+		goto _no_iommu;
+	}
+	geom_attr.aperture_start = 0;
+	geom_attr.aperture_end =
+		((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1;
+	geom_attr.force_aperture = true;
+	ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY,
+				    &geom_attr);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS,
+				    &window_count);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	stash_attr.cpu = cpu;
+	stash_attr.cache = cache;
+	/* set stash information for the window */
+	stash_attr.window = 0;
+
+	ret = iommu_domain_set_attr(pcfg->iommu_domain,
+				    DOMAIN_ATTR_FSL_PAMU_STASH,
+				    &stash_attr);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36,
+					 IOMMU_READ | IOMMU_WRITE);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_set_attr(pcfg->iommu_domain,
+				    DOMAIN_ATTR_FSL_PAMU_ENABLE,
+				    &window_count);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_detach_device;
+	}
+_no_iommu:
+#endif
+
+#ifdef CONFIG_FSL_QMAN_CONFIG
+	if (qman_set_sdest(pcfg->public_cfg.channel, sdest))
+#endif
+		pr_warn("Failed to set QMan portal's stash request queue\n");
+
+	return;
+
+#ifdef CONFIG_FSL_PAMU
+_iommu_detach_device:
+	iommu_detach_device(pcfg->iommu_domain, NULL);
+_iommu_domain_free:
+	iommu_domain_free(pcfg->iommu_domain);
+#endif
+}
+
+static long ioctl_allocate_raw_portal(struct file *fp, struct ctx *ctx,
+				      struct usdpaa_ioctl_raw_portal *arg)
+{
+	struct portal_mapping *mapping = kmalloc(sizeof(*mapping), GFP_KERNEL);
+	int ret;
+
+	if (!mapping)
+		return -ENOMEM;
+
+	mapping->user.type = arg->type;
+	mapping->iommu_domain = NULL;
+	if (arg->type == usdpaa_portal_qman) {
+		mapping->qportal = qm_get_unused_portal_idx(arg->index);
+		if (!mapping->qportal) {
+			ret = -ENODEV;
+			goto err;
+		}
+		mapping->phys = &mapping->qportal->addr_phys[0];
+		arg->index = mapping->qportal->public_cfg.index;
+		arg->cinh = mapping->qportal->addr_phys[DPA_PORTAL_CI].start;
+		arg->cena = mapping->qportal->addr_phys[DPA_PORTAL_CE].start;
+		if (arg->enable_stash) {
+			/* Setup the PAMU with the supplied parameters */
+			portal_config_pamu(mapping->qportal, arg->sdest,
+					   arg->cpu, arg->cache, arg->window);
+		}
+	} else if (mapping->user.type == usdpaa_portal_bman) {
+		mapping->bportal =
+			bm_get_unused_portal_idx(arg->index);
+		if (!mapping->bportal) {
+			ret = -ENODEV;
+			goto err;
+		}
+		mapping->phys = &mapping->bportal->addr_phys[0];
+		arg->index = mapping->bportal->public_cfg.index;
+		arg->cinh = mapping->bportal->addr_phys[DPA_PORTAL_CI].start;
+		arg->cena = mapping->bportal->addr_phys[DPA_PORTAL_CE].start;
+	} else {
+		ret = -EINVAL;
+		goto err;
+	}
+	/* Need to put pcfg in ctx's list before the mmaps because the mmap
+	 * handlers look it up. */
+	spin_lock(&mem_lock);
+	list_add(&mapping->list, &ctx->portals);
+	spin_unlock(&mem_lock);
+	return 0;
+err:
+	kfree(mapping);
+	return ret;
+}
+
+static long ioctl_free_raw_portal(struct file *fp, struct ctx *ctx,
+				      struct usdpaa_ioctl_raw_portal *arg)
+{
+	struct portal_mapping *mapping;
+	u32 channel;
+
+	/* Find the corresponding portal */
+	spin_lock(&mem_lock);
+	list_for_each_entry(mapping, &ctx->portals, list) {
+		if (mapping->phys[DPA_PORTAL_CI].start == arg->cinh)
+			goto found;
+	}
+	mapping = NULL;
+found:
+	if (mapping)
+		list_del(&mapping->list);
+	spin_unlock(&mem_lock);
+	if (!mapping)
+		return -ENODEV;
+	if (mapping->user.type == usdpaa_portal_qman) {
+		init_qm_portal(mapping->qportal,
+				       &mapping->qman_portal_low);
+
+		/* Tear down any FQs this portal is referencing */
+		channel = mapping->qportal->public_cfg.channel;
+		qm_check_and_destroy_fqs(&mapping->qman_portal_low,
+					 &channel,
+					 check_portal_channel);
+		qm_put_unused_portal(mapping->qportal);
+	} else if (mapping->user.type == usdpaa_portal_bman) {
+		init_bm_portal(mapping->bportal,
+			       &mapping->bman_portal_low);
+		bm_put_unused_portal(mapping->bportal);
+	}
+	kfree(mapping);
+	return 0;
+}
+
+static long usdpaa_ioctl(struct file *fp, unsigned int cmd, unsigned long arg)
+{
+	struct ctx *ctx = fp->private_data;
+	void __user *a = (void __user *)arg;
+	switch (cmd) {
+	case USDPAA_IOCTL_ID_ALLOC:
+		return ioctl_id_alloc(ctx, a);
+	case USDPAA_IOCTL_ID_RELEASE:
+		return ioctl_id_release(ctx, a);
+	case USDPAA_IOCTL_ID_RESERVE:
+		return ioctl_id_reserve(ctx, a);
+	case USDPAA_IOCTL_DMA_MAP:
+	{
+		struct usdpaa_ioctl_dma_map input;
+		int ret;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		ret = ioctl_dma_map(fp, ctx, &input);
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_DMA_UNMAP:
+		return ioctl_dma_unmap(ctx, a);
+	case USDPAA_IOCTL_DMA_LOCK:
+		return ioctl_dma_lock(ctx, a);
+	case USDPAA_IOCTL_DMA_UNLOCK:
+		return ioctl_dma_unlock(ctx, a);
+	case USDPAA_IOCTL_PORTAL_MAP:
+	{
+		struct usdpaa_ioctl_portal_map input;
+		int ret;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		ret =  ioctl_portal_map(fp, ctx, &input);
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_PORTAL_UNMAP:
+	{
+		struct usdpaa_portal_map input;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		return ioctl_portal_unmap(ctx, &input);
+	}
+	case USDPAA_IOCTL_DMA_USED:
+		return ioctl_dma_stats(ctx, a);
+	case USDPAA_IOCTL_ALLOC_RAW_PORTAL:
+	{
+		struct usdpaa_ioctl_raw_portal input;
+		int ret;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		ret = ioctl_allocate_raw_portal(fp, ctx, &input);
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_FREE_RAW_PORTAL:
+	{
+		struct usdpaa_ioctl_raw_portal input;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		return ioctl_free_raw_portal(fp, ctx, &input);
+	}
+	}
+	return -EINVAL;
+}
+
+static long usdpaa_ioctl_compat(struct file *fp, unsigned int cmd,
+				unsigned long arg)
+{
+#ifdef CONFIG_COMPAT
+	struct ctx *ctx = fp->private_data;
+	void __user *a = (void __user *)arg;
+#endif
+	switch (cmd) {
+#ifdef CONFIG_COMPAT
+	case USDPAA_IOCTL_DMA_MAP_COMPAT:
+	{
+		int ret;
+		struct usdpaa_ioctl_dma_map_compat input;
+		struct usdpaa_ioctl_dma_map converted;
+
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+
+		converted.ptr = compat_ptr(input.ptr);
+		converted.phys_addr = input.phys_addr;
+		converted.len = input.len;
+		converted.flags = input.flags;
+		strncpy(converted.name, input.name, USDPAA_DMA_NAME_MAX);
+		converted.has_locking = input.has_locking;
+		converted.did_create = input.did_create;
+
+		ret = ioctl_dma_map(fp, ctx, &converted);
+		input.ptr = ptr_to_compat(converted.ptr);
+		input.phys_addr = converted.phys_addr;
+		input.len = converted.len;
+		input.flags = converted.flags;
+		strncpy(input.name, converted.name, USDPAA_DMA_NAME_MAX);
+		input.has_locking = converted.has_locking;
+		input.did_create = converted.did_create;
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_PORTAL_MAP_COMPAT:
+	{
+		int ret;
+		struct compat_usdpaa_ioctl_portal_map input;
+		struct usdpaa_ioctl_portal_map converted;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		converted.type = input.type;
+		converted.index = input.index;
+		ret = ioctl_portal_map(fp, ctx, &converted);
+		input.addr.cinh = ptr_to_compat(converted.addr.cinh);
+		input.addr.cena = ptr_to_compat(converted.addr.cena);
+		input.channel = converted.channel;
+		input.pools = converted.pools;
+		input.index = converted.index;
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_PORTAL_UNMAP_COMPAT:
+	{
+		struct usdpaa_portal_map_compat input;
+		struct usdpaa_portal_map converted;
+
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		converted.cinh = compat_ptr(input.cinh);
+		converted.cena = compat_ptr(input.cena);
+		return ioctl_portal_unmap(ctx, &converted);
+	}
+	case USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT:
+	{
+		int ret;
+		struct usdpaa_ioctl_raw_portal converted;
+		struct compat_ioctl_raw_portal input;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		converted.type = input.type;
+		converted.index = input.index;
+		converted.enable_stash = input.enable_stash;
+		converted.cpu = input.cpu;
+		converted.cache = input.cache;
+		converted.window = input.window;
+		converted.sdest = input.sdest;
+		ret = ioctl_allocate_raw_portal(fp, ctx, &converted);
+
+		input.cinh = converted.cinh;
+		input.cena = converted.cena;
+		input.index = converted.index;
+
+		if (copy_to_user(a, &input, sizeof(input)))
+			return -EFAULT;
+		return ret;
+	}
+	case USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT:
+	{
+		struct usdpaa_ioctl_raw_portal converted;
+		struct compat_ioctl_raw_portal input;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		converted.type = input.type;
+		converted.index = input.index;
+		converted.cinh = input.cinh;
+		converted.cena = input.cena;
+		return ioctl_free_raw_portal(fp, ctx, &converted);
+	}
+#endif
+	default:
+		return usdpaa_ioctl(fp, cmd, arg);
+	}
+	return -EINVAL;
+}
+
+int usdpaa_get_portal_config(struct file *filp, void *cinh,
+			     enum usdpaa_portal_type ptype, unsigned int *irq,
+			     void **iir_reg)
+{
+	/* Walk the list of portals for filp and return the config
+	   for the portal that matches the hint */
+	struct ctx *context;
+	struct portal_mapping *portal;
+
+	/* First sanitize the filp */
+	if (filp->f_op->open != usdpaa_open)
+		return -ENODEV;
+	context = filp->private_data;
+	spin_lock(&context->lock);
+	list_for_each_entry(portal, &context->portals, list) {
+		if (portal->user.type == ptype &&
+		    portal->user.addr.cinh == cinh) {
+			if (ptype == usdpaa_portal_qman) {
+				*irq = portal->qportal->public_cfg.irq;
+				*iir_reg = portal->qportal->addr_virt[1] +
+					QM_REG_IIR;
+			} else {
+				*irq = portal->bportal->public_cfg.irq;
+				*iir_reg = portal->bportal->addr_virt[1] +
+					BM_REG_IIR;
+			}
+			spin_unlock(&context->lock);
+			return 0;
+		}
+	}
+	spin_unlock(&context->lock);
+	return -EINVAL;
+}
+
+static const struct file_operations usdpaa_fops = {
+	.open		   = usdpaa_open,
+	.release	   = usdpaa_release,
+	.mmap		   = usdpaa_mmap,
+	.get_unmapped_area = usdpaa_get_unmapped_area,
+	.unlocked_ioctl	   = usdpaa_ioctl,
+	.compat_ioctl	   = usdpaa_ioctl_compat
+};
+
+static struct miscdevice usdpaa_miscdev = {
+	.name = "fsl-usdpaa",
+	.fops = &usdpaa_fops,
+	.minor = MISC_DYNAMIC_MINOR,
+};
+
+/* Early-boot memory allocation. The boot-arg "usdpaa_mem=<x>" is used to
+ * indicate how much memory (if any) to allocate during early boot. If the
+ * format "usdpaa_mem=<x>,<y>" is used, then <y> will be interpreted as the
+ * number of TLB1 entries to reserve (default is 1). If there are more mappings
+ * than there are TLB1 entries, fault-handling will occur. */
+
+static __init int usdpaa_mem(char *arg)
+{
+	pr_warn("uspdaa_mem argument is depracated\n");
+	arg_phys_size = memparse(arg, &arg);
+	num_tlb = 1;
+	if (*arg == ',') {
+		unsigned long ul;
+		int err = kstrtoul(arg + 1, 0, &ul);
+		if (err < 0) {
+			num_tlb = 1;
+			pr_warn("ERROR, usdpaa_mem arg is invalid\n");
+		} else
+			num_tlb = (unsigned int)ul;
+	}
+	return 0;
+}
+early_param("usdpaa_mem", usdpaa_mem);
+
+static int usdpaa_mem_init(struct reserved_mem *rmem)
+{
+	phys_start = rmem->base;
+	phys_size = rmem->size;
+
+	WARN_ON(!(phys_start && phys_size));
+
+	return 0;
+}
+RESERVEDMEM_OF_DECLARE(usdpaa_mem_init, "fsl,usdpaa-mem", usdpaa_mem_init);
+
+__init int fsl_usdpaa_init_early(void)
+{
+	if (!phys_size || !phys_start) {
+		pr_info("No USDPAA memory, no 'fsl,usdpaa-mem' in device-tree\n");
+		return 0;
+	}
+	if (phys_size % PAGE_SIZE) {
+		pr_err("'fsl,usdpaa-mem' size must be a multiple of page size\n");
+		phys_size = 0;
+		return 0;
+	}
+	if (arg_phys_size && phys_size != arg_phys_size) {
+		pr_err("'usdpaa_mem argument size (0x%llx) does not match device tree size (0x%llx)\n",
+		       arg_phys_size, phys_size);
+		phys_size = 0;
+		return 0;
+	}
+	pfn_start = phys_start >> PAGE_SHIFT;
+	pfn_size = phys_size >> PAGE_SHIFT;
+#ifdef CONFIG_PPC
+	first_tlb = current_tlb = tlbcam_index;
+	tlbcam_index += num_tlb;
+#endif
+	pr_info("USDPAA region at %llx:%llx(%lx:%lx), %d TLB1 entries)\n",
+		phys_start, phys_size, pfn_start, pfn_size, num_tlb);
+	return 0;
+}
+subsys_initcall(fsl_usdpaa_init_early);
+
+
+static int __init usdpaa_init(void)
+{
+	struct mem_fragment *frag;
+	int ret;
+	u64 tmp_size = phys_size;
+	u64 tmp_start = phys_start;
+	u64 tmp_pfn_size = pfn_size;
+	u64 tmp_pfn_start = pfn_start;
+
+	pr_info("Freescale USDPAA process driver\n");
+	if (!phys_start) {
+		pr_warn("fsl-usdpaa: no region found\n");
+		return 0;
+	}
+
+	while (tmp_size != 0) {
+		u32 frag_size = largest_page_size(tmp_size);
+		frag = kmalloc(sizeof(*frag), GFP_KERNEL);
+		if (!frag) {
+			pr_err("Failed to setup USDPAA memory accounting\n");
+			return -ENOMEM;
+		}
+		frag->base = tmp_start;
+		frag->len = frag->root_len = frag_size;
+		frag->root_pfn = tmp_pfn_start;
+		frag->pfn_base = tmp_pfn_start;
+		frag->pfn_len = frag_size / PAGE_SIZE;
+		frag->refs = 0;
+		init_waitqueue_head(&frag->wq);
+		frag->owner = NULL;
+		list_add(&frag->list, &mem_list);
+
+		/* Adjust for this frag */
+		tmp_start += frag_size;
+		tmp_size -= frag_size;
+		tmp_pfn_start += frag_size / PAGE_SIZE;
+		tmp_pfn_size -= frag_size / PAGE_SIZE;
+	}
+	ret = misc_register(&usdpaa_miscdev);
+	if (ret)
+		pr_err("fsl-usdpaa: failed to register misc device\n");
+	return ret;
+}
+
+static void __exit usdpaa_exit(void)
+{
+	misc_deregister(&usdpaa_miscdev);
+}
+
+module_init(usdpaa_init);
+module_exit(usdpaa_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Freescale Semiconductor");
+MODULE_DESCRIPTION("Freescale USDPAA process driver");
--- /dev/null
+++ b/drivers/staging/fsl_qbman/fsl_usdpaa_irq.c
@@ -0,0 +1,289 @@
+/* Copyright (c) 2013 Freescale Semiconductor, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *	 notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *	 notice, this list of conditions and the following disclaimer in the
+ *	 documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *	 names of its contributors may be used to endorse or promote products
+ *	 derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/* define a device that allows USPDAA processes to open a file
+   descriptor and specify which IRQ it wants to montior using an ioctl()
+   When an IRQ is received, the device becomes readable so that a process
+   can use read() or select() type calls to monitor for IRQs */
+
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/cdev.h>
+#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/poll.h>
+#include <linux/uaccess.h>
+#include <linux/fsl_usdpaa.h>
+#include <linux/module.h>
+#include <linux/fdtable.h>
+#include <linux/file.h>
+
+#include "qman_low.h"
+#include "bman_low.h"
+
+struct usdpaa_irq_ctx {
+	int irq_set; /* Set to true once the irq is set via ioctl */
+	unsigned int irq_num;
+	u32 last_irq_count; /* Last value returned from read */
+	u32 irq_count; /* Number of irqs since last read */
+	wait_queue_head_t wait_queue; /* Waiting processes */
+	spinlock_t lock;
+	void *inhibit_addr; /* inhibit register address */
+	struct file *usdpaa_filp;
+	char irq_name[128];
+};
+
+static int usdpaa_irq_open(struct inode *inode, struct file *filp)
+{
+	struct usdpaa_irq_ctx *ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
+	if (!ctx)
+		return -ENOMEM;
+	ctx->irq_set = 0;
+	ctx->irq_count = 0;
+	ctx->last_irq_count = 0;
+	init_waitqueue_head(&ctx->wait_queue);
+	spin_lock_init(&ctx->lock);
+	filp->private_data = ctx;
+	return 0;
+}
+
+static int usdpaa_irq_release(struct inode *inode, struct file *filp)
+{
+	struct usdpaa_irq_ctx *ctx = filp->private_data;
+	if (ctx->irq_set) {
+		/* Inhibit the IRQ */
+		out_be32(ctx->inhibit_addr, 0x1);
+		irq_set_affinity_hint(ctx->irq_num, NULL);
+		free_irq(ctx->irq_num, ctx);
+		ctx->irq_set = 0;
+		fput(ctx->usdpaa_filp);
+	}
+	kfree(filp->private_data);
+	return 0;
+}
+
+static irqreturn_t usdpaa_irq_handler(int irq, void *_ctx)
+{
+	unsigned long flags;
+	struct usdpaa_irq_ctx *ctx = _ctx;
+	spin_lock_irqsave(&ctx->lock, flags);
+	++ctx->irq_count;
+	spin_unlock_irqrestore(&ctx->lock, flags);
+	wake_up_all(&ctx->wait_queue);
+	/* Set the inhibit register.  This will be reenabled
+	   once the USDPAA code handles the IRQ */
+	out_be32(ctx->inhibit_addr, 0x1);
+	pr_info("Inhibit at %p count %d", ctx->inhibit_addr, ctx->irq_count);
+	return IRQ_HANDLED;
+}
+
+static int map_irq(struct file *fp, struct usdpaa_ioctl_irq_map *irq_map)
+{
+	struct usdpaa_irq_ctx *ctx = fp->private_data;
+	int ret;
+
+	if (ctx->irq_set) {
+		pr_debug("Setting USDPAA IRQ when it was already set!\n");
+		return -EBUSY;
+	}
+
+	ctx->usdpaa_filp = fget(irq_map->fd);
+	if (!ctx->usdpaa_filp) {
+		pr_debug("USDPAA fget(%d) returned NULL\n", irq_map->fd);
+		return -EINVAL;
+	}
+
+	ret = usdpaa_get_portal_config(ctx->usdpaa_filp, irq_map->portal_cinh,
+				       irq_map->type, &ctx->irq_num,
+				       &ctx->inhibit_addr);
+	if (ret) {
+		pr_debug("USDPAA IRQ couldn't identify portal\n");
+		fput(ctx->usdpaa_filp);
+		return ret;
+	}
+
+	ctx->irq_set = 1;
+
+	snprintf(ctx->irq_name, sizeof(ctx->irq_name),
+		 "usdpaa_irq %d", ctx->irq_num);
+
+	ret = request_irq(ctx->irq_num, usdpaa_irq_handler, 0,
+			  ctx->irq_name, ctx);
+	if (ret) {
+		pr_err("USDPAA request_irq(%d) failed, ret= %d\n",
+		       ctx->irq_num, ret);
+		ctx->irq_set = 0;
+		fput(ctx->usdpaa_filp);
+		return ret;
+	}
+	ret = irq_set_affinity(ctx->irq_num, tsk_cpus_allowed(current));
+	if (ret)
+		pr_err("USDPAA irq_set_affinity() failed, ret= %d\n", ret);
+
+	ret = irq_set_affinity_hint(ctx->irq_num, tsk_cpus_allowed(current));
+	if (ret)
+		pr_err("USDPAA irq_set_affinity_hint() failed, ret= %d\n", ret);
+
+	return 0;
+}
+
+static long usdpaa_irq_ioctl(struct file *fp, unsigned int cmd,
+			     unsigned long arg)
+{
+	int ret;
+	struct usdpaa_ioctl_irq_map irq_map;
+
+	if (cmd != USDPAA_IOCTL_PORTAL_IRQ_MAP) {
+		pr_debug("USDPAA IRQ unknown command 0x%x\n", cmd);
+		return -EINVAL;
+	}
+
+	ret = copy_from_user(&irq_map, (void __user *)arg,
+			     sizeof(irq_map));
+	if (ret)
+		return ret;
+	return map_irq(fp, &irq_map);
+}
+
+static ssize_t usdpaa_irq_read(struct file *filp, char __user *buff,
+			       size_t count, loff_t *offp)
+{
+	struct usdpaa_irq_ctx *ctx = filp->private_data;
+	int ret;
+
+	if (!ctx->irq_set) {
+		pr_debug("Reading USDPAA IRQ before it was set\n");
+		return -EINVAL;
+	}
+
+	if (count < sizeof(ctx->irq_count)) {
+		pr_debug("USDPAA IRQ Read too small\n");
+		return -EINVAL;
+	}
+	if (ctx->irq_count == ctx->last_irq_count) {
+		if (filp->f_flags & O_NONBLOCK)
+			return -EAGAIN;
+
+		ret = wait_event_interruptible(ctx->wait_queue,
+				       ctx->irq_count != ctx->last_irq_count);
+		if (ret == -ERESTARTSYS)
+			return ret;
+	}
+
+	ctx->last_irq_count = ctx->irq_count;
+
+	if (copy_to_user(buff, &ctx->last_irq_count,
+			 sizeof(ctx->last_irq_count)))
+		return -EFAULT;
+	return sizeof(ctx->irq_count);
+}
+
+static unsigned int usdpaa_irq_poll(struct file *filp, poll_table *wait)
+{
+	struct usdpaa_irq_ctx *ctx = filp->private_data;
+	unsigned int ret = 0;
+	unsigned long flags;
+
+	if (!ctx->irq_set)
+		return POLLHUP;
+
+	poll_wait(filp, &ctx->wait_queue, wait);
+
+	spin_lock_irqsave(&ctx->lock, flags);
+	if (ctx->irq_count != ctx->last_irq_count)
+		ret |= POLLIN | POLLRDNORM;
+	spin_unlock_irqrestore(&ctx->lock, flags);
+	return ret;
+}
+
+static long usdpaa_irq_ioctl_compat(struct file *fp, unsigned int cmd,
+				unsigned long arg)
+{
+#ifdef CONFIG_COMPAT
+	void __user *a = (void __user *)arg;
+#endif
+	switch (cmd) {
+#ifdef CONFIG_COMPAT
+	case  USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT:
+	{
+		struct compat_ioctl_irq_map input;
+		struct usdpaa_ioctl_irq_map converted;
+		if (copy_from_user(&input, a, sizeof(input)))
+			return -EFAULT;
+		converted.type = input.type;
+		converted.fd = input.fd;
+		converted.portal_cinh = compat_ptr(input.portal_cinh);
+		return map_irq(fp, &converted);
+	}
+#endif
+	default:
+		return usdpaa_irq_ioctl(fp, cmd, arg);
+	}
+}
+
+static const struct file_operations usdpaa_irq_fops = {
+	.open		   = usdpaa_irq_open,
+	.release	   = usdpaa_irq_release,
+	.unlocked_ioctl	   = usdpaa_irq_ioctl,
+	.compat_ioctl	   = usdpaa_irq_ioctl_compat,
+	.read              = usdpaa_irq_read,
+	.poll              = usdpaa_irq_poll
+};
+
+static struct miscdevice usdpaa_miscdev = {
+	.name = "fsl-usdpaa-irq",
+	.fops = &usdpaa_irq_fops,
+	.minor = MISC_DYNAMIC_MINOR,
+};
+
+static int __init usdpaa_irq_init(void)
+{
+	int ret;
+
+	pr_info("Freescale USDPAA process IRQ driver\n");
+	ret = misc_register(&usdpaa_miscdev);
+	if (ret)
+		pr_err("fsl-usdpaa-irq: failed to register misc device\n");
+	return ret;
+}
+
+static void __exit usdpaa_irq_exit(void)
+{
+	misc_deregister(&usdpaa_miscdev);
+}
+
+module_init(usdpaa_irq_init);
+module_exit(usdpaa_irq_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Freescale Semiconductor");
+MODULE_DESCRIPTION("Freescale USDPAA process IRQ driver");
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qbman_driver.c
@@ -0,0 +1,88 @@
+/* Copyright 2013 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *	 notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *	 notice, this list of conditions and the following disclaimer in the
+ *	 documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *	 names of its contributors may be used to endorse or promote products
+ *	 derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/time.h>
+#include "qman_private.h"
+#include "bman_private.h"
+__init void qman_init_early(void);
+__init void bman_init_early(void);
+
+static __init int qbman_init(void)
+{
+	struct device_node *dn;
+	u32 is_portal_available;
+
+	bman_init();
+	qman_init();
+
+	is_portal_available = 0;
+	for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
+		if (!of_device_is_available(dn))
+			continue;
+		else
+			is_portal_available = 1;
+	}
+
+	if (!qman_have_ccsr() && is_portal_available) {
+		struct qman_fq fq = {
+				.fqid = 1
+		};
+		struct qm_mcr_queryfq_np np;
+		int err, retry = CONFIG_FSL_QMAN_INIT_TIMEOUT;
+		struct timespec nowts, diffts, startts = current_kernel_time();
+		/* Loop while querying given fqid succeeds or time out */
+		while (1) {
+			err = qman_query_fq_np(&fq, &np);
+			if (!err) {
+				/* success, control-plane has configured QMan */
+				break;
+			} else if (err != -ERANGE) {
+				pr_err("QMan: I/O error, continuing anyway\n");
+				break;
+			}
+			nowts = current_kernel_time();
+			diffts = timespec_sub(nowts, startts);
+			if (diffts.tv_sec > 0) {
+				if (!retry--) {
+					pr_err("QMan: time out, control-plane"
+								" dead?\n");
+					break;
+				}
+				pr_warn("QMan: polling for the control-plane"
+							" (%d)\n", retry);
+			}
+		}
+	}
+	bman_resource_init();
+	qman_resource_init();
+	return 0;
+}
+subsys_initcall(qbman_init);
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_config.c
@@ -0,0 +1,1199 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/cacheflush.h>
+#include "qman_private.h"
+#include <linux/highmem.h>
+#include <linux/of_reserved_mem.h>
+
+/* Last updated for v00.800 of the BG */
+
+/* Register offsets */
+#define REG_QCSP_LIO_CFG(n)	(0x0000 + ((n) * 0x10))
+#define REG_QCSP_IO_CFG(n)	(0x0004 + ((n) * 0x10))
+#define REG_QCSP_DD_CFG(n)	(0x000c + ((n) * 0x10))
+#define REG_DD_CFG		0x0200
+#define REG_DCP_CFG(n)		(0x0300 + ((n) * 0x10))
+#define REG_DCP_DD_CFG(n)	(0x0304 + ((n) * 0x10))
+#define REG_DCP_DLM_AVG(n)	(0x030c + ((n) * 0x10))
+#define REG_PFDR_FPC		0x0400
+#define REG_PFDR_FP_HEAD	0x0404
+#define REG_PFDR_FP_TAIL	0x0408
+#define REG_PFDR_FP_LWIT	0x0410
+#define REG_PFDR_CFG		0x0414
+#define REG_SFDR_CFG		0x0500
+#define REG_SFDR_IN_USE		0x0504
+#define REG_WQ_CS_CFG(n)	(0x0600 + ((n) * 0x04))
+#define REG_WQ_DEF_ENC_WQID	0x0630
+#define REG_WQ_SC_DD_CFG(n)	(0x640 + ((n) * 0x04))
+#define REG_WQ_PC_DD_CFG(n)	(0x680 + ((n) * 0x04))
+#define REG_WQ_DC0_DD_CFG(n)	(0x6c0 + ((n) * 0x04))
+#define REG_WQ_DC1_DD_CFG(n)	(0x700 + ((n) * 0x04))
+#define REG_WQ_DCn_DD_CFG(n)	(0x6c0 + ((n) * 0x40)) /* n=2,3 */
+#define REG_CM_CFG		0x0800
+#define REG_ECSR		0x0a00
+#define REG_ECIR		0x0a04
+#define REG_EADR		0x0a08
+#define REG_ECIR2		0x0a0c
+#define REG_EDATA(n)		(0x0a10 + ((n) * 0x04))
+#define REG_SBEC(n)		(0x0a80 + ((n) * 0x04))
+#define REG_MCR			0x0b00
+#define REG_MCP(n)		(0x0b04 + ((n) * 0x04))
+#define REG_MISC_CFG		0x0be0
+#define REG_HID_CFG		0x0bf0
+#define REG_IDLE_STAT		0x0bf4
+#define REG_IP_REV_1		0x0bf8
+#define REG_IP_REV_2		0x0bfc
+#define REG_FQD_BARE		0x0c00
+#define REG_PFDR_BARE		0x0c20
+#define REG_offset_BAR		0x0004	/* relative to REG_[FQD|PFDR]_BARE */
+#define REG_offset_AR		0x0010	/* relative to REG_[FQD|PFDR]_BARE */
+#define REG_QCSP_BARE		0x0c80
+#define REG_QCSP_BAR		0x0c84
+#define REG_CI_SCHED_CFG	0x0d00
+#define REG_SRCIDR		0x0d04
+#define REG_LIODNR		0x0d08
+#define REG_CI_RLM_AVG		0x0d14
+#define REG_ERR_ISR		0x0e00	/* + "enum qm_isr_reg" */
+#define REG_REV3_QCSP_LIO_CFG(n)	(0x1000 + ((n) * 0x10))
+#define REG_REV3_QCSP_IO_CFG(n)	(0x1004 + ((n) * 0x10))
+#define REG_REV3_QCSP_DD_CFG(n)	(0x100c + ((n) * 0x10))
+#define REG_CEETM_CFG_IDX	0x900
+#define REG_CEETM_CFG_PRES	0x904
+#define REG_CEETM_XSFDR_IN_USE	0x908
+
+/* Assists for QMAN_MCR */
+#define MCR_INIT_PFDR		0x01000000
+#define MCR_get_rslt(v)		(u8)((v) >> 24)
+#define MCR_rslt_idle(r)	(!rslt || (rslt >= 0xf0))
+#define MCR_rslt_ok(r)		(rslt == 0xf0)
+#define MCR_rslt_eaccess(r)	(rslt == 0xf8)
+#define MCR_rslt_inval(r)	(rslt == 0xff)
+
+struct qman;
+
+/* Follows WQ_CS_CFG0-5 */
+enum qm_wq_class {
+	qm_wq_portal = 0,
+	qm_wq_pool = 1,
+	qm_wq_fman0 = 2,
+	qm_wq_fman1 = 3,
+	qm_wq_caam = 4,
+	qm_wq_pme = 5,
+	qm_wq_first = qm_wq_portal,
+	qm_wq_last = qm_wq_pme
+};
+
+/* Follows FQD_[BARE|BAR|AR] and PFDR_[BARE|BAR|AR] */
+enum qm_memory {
+	qm_memory_fqd,
+	qm_memory_pfdr
+};
+
+/* Used by all error interrupt registers except 'inhibit' */
+#define QM_EIRQ_CIDE	0x20000000	/* Corenet Initiator Data Error */
+#define QM_EIRQ_CTDE	0x10000000	/* Corenet Target Data Error */
+#define QM_EIRQ_CITT	0x08000000	/* Corenet Invalid Target Transaction */
+#define QM_EIRQ_PLWI	0x04000000	/* PFDR Low Watermark */
+#define QM_EIRQ_MBEI	0x02000000	/* Multi-bit ECC Error */
+#define QM_EIRQ_SBEI	0x01000000	/* Single-bit ECC Error */
+#define QM_EIRQ_PEBI	0x00800000	/* PFDR Enqueues Blocked Interrupt */
+#define QM_EIRQ_IFSI	0x00020000	/* Invalid FQ Flow Control State */
+#define QM_EIRQ_ICVI	0x00010000	/* Invalid Command Verb */
+#define QM_EIRQ_IDDI	0x00000800	/* Invalid Dequeue (Direct-connect) */
+#define QM_EIRQ_IDFI	0x00000400	/* Invalid Dequeue FQ */
+#define QM_EIRQ_IDSI	0x00000200	/* Invalid Dequeue Source */
+#define QM_EIRQ_IDQI	0x00000100	/* Invalid Dequeue Queue */
+#define QM_EIRQ_IECE	0x00000010	/* Invalid Enqueue Configuration */
+#define QM_EIRQ_IEOI	0x00000008	/* Invalid Enqueue Overflow */
+#define QM_EIRQ_IESI	0x00000004	/* Invalid Enqueue State */
+#define QM_EIRQ_IECI	0x00000002	/* Invalid Enqueue Channel */
+#define QM_EIRQ_IEQI	0x00000001	/* Invalid Enqueue Queue */
+
+/* QMAN_ECIR valid error bit */
+#define PORTAL_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IESI | QM_EIRQ_IEOI | \
+				QM_EIRQ_IDQI | QM_EIRQ_IDSI | QM_EIRQ_IDFI | \
+				QM_EIRQ_IDDI | QM_EIRQ_ICVI | QM_EIRQ_IFSI)
+#define FQID_ECSR_ERR	(QM_EIRQ_IEQI | QM_EIRQ_IECI | QM_EIRQ_IESI | \
+			QM_EIRQ_IEOI | QM_EIRQ_IDQI | QM_EIRQ_IDFI | \
+			QM_EIRQ_IFSI)
+
+union qman_ecir {
+	u32 ecir_raw;
+	struct {
+		u32 __reserved:2;
+		u32 portal_type:1;
+		u32 portal_num:5;
+		u32 fqid:24;
+	} __packed info;
+};
+
+union qman_ecir2 {
+	u32 ecir2_raw;
+	struct {
+		u32 portal_type:1;
+		u32 __reserved:21;
+		u32 portal_num:10;
+	} __packed info;
+};
+
+union qman_eadr {
+	u32 eadr_raw;
+	struct {
+		u32 __reserved1:4;
+		u32 memid:4;
+		u32 __reserved2:12;
+		u32 eadr:12;
+	} __packed info;
+	struct {
+		u32 __reserved1:3;
+		u32 memid:5;
+		u32 __reserved:8;
+		u32 eadr:16;
+	} __packed info_rev3;
+};
+
+struct qman_hwerr_txt {
+	u32 mask;
+	const char *txt;
+};
+
+#define QMAN_HWE_TXT(a, b) { .mask = QM_EIRQ_##a, .txt = b }
+
+static const struct qman_hwerr_txt qman_hwerr_txts[] = {
+	QMAN_HWE_TXT(CIDE, "Corenet Initiator Data Error"),
+	QMAN_HWE_TXT(CTDE, "Corenet Target Data Error"),
+	QMAN_HWE_TXT(CITT, "Corenet Invalid Target Transaction"),
+	QMAN_HWE_TXT(PLWI, "PFDR Low Watermark"),
+	QMAN_HWE_TXT(MBEI, "Multi-bit ECC Error"),
+	QMAN_HWE_TXT(SBEI, "Single-bit ECC Error"),
+	QMAN_HWE_TXT(PEBI, "PFDR Enqueues Blocked Interrupt"),
+	QMAN_HWE_TXT(ICVI, "Invalid Command Verb"),
+	QMAN_HWE_TXT(IFSI, "Invalid Flow Control State"),
+	QMAN_HWE_TXT(IDDI, "Invalid Dequeue (Direct-connect)"),
+	QMAN_HWE_TXT(IDFI, "Invalid Dequeue FQ"),
+	QMAN_HWE_TXT(IDSI, "Invalid Dequeue Source"),
+	QMAN_HWE_TXT(IDQI, "Invalid Dequeue Queue"),
+	QMAN_HWE_TXT(IECE, "Invalid Enqueue Configuration"),
+	QMAN_HWE_TXT(IEOI, "Invalid Enqueue Overflow"),
+	QMAN_HWE_TXT(IESI, "Invalid Enqueue State"),
+	QMAN_HWE_TXT(IECI, "Invalid Enqueue Channel"),
+	QMAN_HWE_TXT(IEQI, "Invalid Enqueue Queue")
+};
+#define QMAN_HWE_COUNT (sizeof(qman_hwerr_txts)/sizeof(struct qman_hwerr_txt))
+
+struct qman_error_info_mdata {
+	u16 addr_mask;
+	u16 bits;
+	const char *txt;
+};
+
+#define QMAN_ERR_MDATA(a, b, c) { .addr_mask = a, .bits = b, .txt = c}
+static const struct qman_error_info_mdata error_mdata[] = {
+	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 0"),
+	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 1"),
+	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 2"),
+	QMAN_ERR_MDATA(0x01FF, 24, "FQD cache tag memory 3"),
+	QMAN_ERR_MDATA(0x0FFF, 512, "FQD cache memory"),
+	QMAN_ERR_MDATA(0x07FF, 128, "SFDR memory"),
+	QMAN_ERR_MDATA(0x01FF, 72, "WQ context memory"),
+	QMAN_ERR_MDATA(0x00FF, 240, "CGR memory"),
+	QMAN_ERR_MDATA(0x00FF, 302, "Internal Order Restoration List memory"),
+	QMAN_ERR_MDATA(0x01FF, 256, "SW portal ring memory"),
+	QMAN_ERR_MDATA(0x07FF, 181, "CEETM class queue descriptor memory"),
+	QMAN_ERR_MDATA(0x0FFF, 140, "CEETM extended SFDR memory"),
+	QMAN_ERR_MDATA(0x0FFF, 25, "CEETM logical FQ mapping memory"),
+	QMAN_ERR_MDATA(0x0FFF, 96, "CEETM dequeue context memory"),
+	QMAN_ERR_MDATA(0x07FF, 396, "CEETM ccgr memory"),
+	QMAN_ERR_MDATA(0x00FF, 146, "CEETM CQ channel shaping memory"),
+	QMAN_ERR_MDATA(0x007F, 256, "CEETM CQ channel scheduling memory"),
+	QMAN_ERR_MDATA(0x01FF, 88, "CEETM dequeue statistics memory"),
+};
+#define QMAN_ERR_MDATA_COUNT \
+	(sizeof(error_mdata)/sizeof(struct qman_error_info_mdata))
+
+/* Add this in Kconfig */
+#define QMAN_ERRS_TO_UNENABLE (QM_EIRQ_PLWI | QM_EIRQ_PEBI)
+
+/**
+ * qm_err_isr_<reg>_<verb> - Manipulate global interrupt registers
+ * @v: for accessors that write values, this is the 32-bit value
+ *
+ * Manipulates QMAN_ERR_ISR, QMAN_ERR_IER, QMAN_ERR_ISDR, QMAN_ERR_IIR. All
+ * manipulations except qm_err_isr_[un]inhibit() use 32-bit masks composed of
+ * the QM_EIRQ_*** definitions. Note that "qm_err_isr_enable_write" means
+ * "write the enable register" rather than "enable the write register"!
+ */
+#define qm_err_isr_status_read(qm)	\
+		__qm_err_isr_read(qm, qm_isr_status)
+#define qm_err_isr_status_clear(qm, m)	\
+		__qm_err_isr_write(qm, qm_isr_status, m)
+#define qm_err_isr_enable_read(qm)	\
+		__qm_err_isr_read(qm, qm_isr_enable)
+#define qm_err_isr_enable_write(qm, v)	\
+		__qm_err_isr_write(qm, qm_isr_enable, v)
+#define qm_err_isr_disable_read(qm)	\
+		__qm_err_isr_read(qm, qm_isr_disable)
+#define qm_err_isr_disable_write(qm, v)	\
+		__qm_err_isr_write(qm, qm_isr_disable, v)
+#define qm_err_isr_inhibit(qm)		\
+		__qm_err_isr_write(qm, qm_isr_inhibit, 1)
+#define qm_err_isr_uninhibit(qm)	\
+		__qm_err_isr_write(qm, qm_isr_inhibit, 0)
+
+/*
+ * TODO: unimplemented registers
+ *
+ * Keeping a list here of Qman registers I have not yet covered;
+ * QCSP_DD_IHRSR, QCSP_DD_IHRFR, QCSP_DD_HASR,
+ * DCP_DD_IHRSR, DCP_DD_IHRFR, DCP_DD_HASR, CM_CFG,
+ * QMAN_EECC, QMAN_SBET, QMAN_EINJ, QMAN_SBEC0-12
+ */
+
+/* Encapsulate "struct qman *" as a cast of the register space address. */
+
+static struct qman *qm_create(void *regs)
+{
+	return (struct qman *)regs;
+}
+
+static inline u32 __qm_in(struct qman *qm, u32 offset)
+{
+	return in_be32((void *)qm + offset);
+}
+static inline void __qm_out(struct qman *qm, u32 offset, u32 val)
+{
+	out_be32((void *)qm + offset, val);
+}
+#define qm_in(reg)		__qm_in(qm, REG_##reg)
+#define qm_out(reg, val)	__qm_out(qm, REG_##reg, val)
+
+static u32 __qm_err_isr_read(struct qman *qm, enum qm_isr_reg n)
+{
+	return __qm_in(qm, REG_ERR_ISR + (n << 2));
+}
+
+static void __qm_err_isr_write(struct qman *qm, enum qm_isr_reg n, u32 val)
+{
+	__qm_out(qm, REG_ERR_ISR + (n << 2), val);
+}
+
+static void qm_set_dc(struct qman *qm, enum qm_dc_portal portal,
+			int ed, u8 sernd)
+{
+	DPA_ASSERT(!ed || (portal == qm_dc_portal_fman0) ||
+			(portal == qm_dc_portal_fman1));
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		qm_out(DCP_CFG(portal), (ed ? 0x1000 : 0) | (sernd & 0x3ff));
+	else
+		qm_out(DCP_CFG(portal), (ed ? 0x100 : 0) | (sernd & 0x1f));
+}
+
+static void qm_set_wq_scheduling(struct qman *qm, enum qm_wq_class wq_class,
+			u8 cs_elev, u8 csw2, u8 csw3, u8 csw4, u8 csw5,
+			u8 csw6, u8 csw7)
+{
+	qm_out(WQ_CS_CFG(wq_class), ((cs_elev & 0xff) << 24) |
+		((csw2 & 0x7) << 20) | ((csw3 & 0x7) << 16) |
+		((csw4 & 0x7) << 12) | ((csw5 & 0x7) << 8) |
+		((csw6 & 0x7) << 4) | (csw7 & 0x7));
+}
+
+static void qm_set_hid(struct qman *qm)
+{
+	qm_out(HID_CFG, 0);
+}
+
+static void qm_set_corenet_initiator(struct qman *qm)
+{
+	qm_out(CI_SCHED_CFG,
+		0x80000000 | /* write srcciv enable */
+		(CONFIG_FSL_QMAN_CI_SCHED_CFG_SRCCIV << 24) |
+		(CONFIG_FSL_QMAN_CI_SCHED_CFG_SRQ_W << 8) |
+		(CONFIG_FSL_QMAN_CI_SCHED_CFG_RW_W << 4) |
+		CONFIG_FSL_QMAN_CI_SCHED_CFG_BMAN_W);
+}
+
+static void qm_get_version(struct qman *qm, u16 *id, u8 *major, u8 *minor,
+			u8 *cfg)
+{
+	u32 v = qm_in(IP_REV_1);
+	u32 v2 = qm_in(IP_REV_2);
+	*id = (v >> 16);
+	*major = (v >> 8) & 0xff;
+	*minor = v & 0xff;
+	*cfg = v2 & 0xff;
+}
+
+static void qm_set_memory(struct qman *qm, enum qm_memory memory, u64 ba,
+			int enable, int prio, int stash, u32 size)
+{
+	u32 offset = (memory == qm_memory_fqd) ? REG_FQD_BARE : REG_PFDR_BARE;
+	u32 exp = ilog2(size);
+	/* choke if size isn't within range */
+	DPA_ASSERT((size >= 4096) && (size <= 1073741824) &&
+			is_power_of_2(size));
+	/* choke if 'ba' has lower-alignment than 'size' */
+	DPA_ASSERT(!(ba & (size - 1)));
+	__qm_out(qm, offset, upper_32_bits(ba));
+	__qm_out(qm, offset + REG_offset_BAR, lower_32_bits(ba));
+	__qm_out(qm, offset + REG_offset_AR,
+		(enable ? 0x80000000 : 0) |
+		(prio ? 0x40000000 : 0) |
+		(stash ? 0x20000000 : 0) |
+		(exp - 1));
+}
+
+static void qm_set_pfdr_threshold(struct qman *qm, u32 th, u8 k)
+{
+	qm_out(PFDR_FP_LWIT, th & 0xffffff);
+	qm_out(PFDR_CFG, k);
+}
+
+static void qm_set_sfdr_threshold(struct qman *qm, u16 th)
+{
+	qm_out(SFDR_CFG, th & 0x3ff);
+}
+
+static int qm_init_pfdr(struct qman *qm, u32 pfdr_start, u32 num)
+{
+	u8 rslt = MCR_get_rslt(qm_in(MCR));
+
+	DPA_ASSERT(pfdr_start && !(pfdr_start & 7) && !(num & 7) && num);
+	/* Make sure the command interface is 'idle' */
+	if (!MCR_rslt_idle(rslt))
+		panic("QMAN_MCR isn't idle");
+
+	/* Write the MCR command params then the verb */
+	qm_out(MCP(0), pfdr_start);
+	/* TODO: remove this - it's a workaround for a model bug that is
+	 * corrected in more recent versions. We use the workaround until
+	 * everyone has upgraded. */
+	qm_out(MCP(1), (pfdr_start + num - 16));
+	lwsync();
+	qm_out(MCR, MCR_INIT_PFDR);
+	/* Poll for the result */
+	do {
+		rslt = MCR_get_rslt(qm_in(MCR));
+	} while (!MCR_rslt_idle(rslt));
+	if (MCR_rslt_ok(rslt))
+		return 0;
+	if (MCR_rslt_eaccess(rslt))
+		return -EACCES;
+	if (MCR_rslt_inval(rslt))
+		return -EINVAL;
+	pr_crit("Unexpected result from MCR_INIT_PFDR: %02x\n", rslt);
+	return -ENOSYS;
+}
+
+/*****************/
+/* Config driver */
+/*****************/
+
+#define DEFAULT_FQD_SZ	(PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ)
+#define DEFAULT_PFDR_SZ	(PAGE_SIZE << CONFIG_FSL_QMAN_PFDR_SZ)
+
+/* We support only one of these */
+static struct qman *qm;
+static struct device_node *qm_node;
+
+/* And this state belongs to 'qm'. It is set during fsl_qman_init(), but used
+ * during qman_init_ccsr(). */
+static dma_addr_t fqd_a, pfdr_a;
+static size_t fqd_sz = DEFAULT_FQD_SZ, pfdr_sz = DEFAULT_PFDR_SZ;
+
+static int qman_fqd(struct reserved_mem *rmem)
+{
+	fqd_a = rmem->base;
+	fqd_sz = rmem->size;
+
+	WARN_ON(!(fqd_a && fqd_sz));
+
+	return 0;
+}
+RESERVEDMEM_OF_DECLARE(qman_fqd, "fsl,qman-fqd", qman_fqd);
+
+static int qman_pfdr(struct reserved_mem *rmem)
+{
+	pfdr_a = rmem->base;
+	pfdr_sz = rmem->size;
+
+	WARN_ON(!(pfdr_a && pfdr_sz));
+
+	return 0;
+}
+RESERVEDMEM_OF_DECLARE(qman_fbpr, "fsl,qman-pfdr", qman_pfdr);
+
+size_t get_qman_fqd_size()
+{
+	return fqd_sz;
+}
+
+/* Parse the <name> property to extract the memory location and size and
+ * memblock_reserve() it. If it isn't supplied, memblock_alloc() the default
+ * size. Also flush this memory range from data cache so that QMAN originated
+ * transactions for this memory region could be marked non-coherent.
+ */
+static __init int parse_mem_property(struct device_node *node, const char *name,
+				dma_addr_t *addr, size_t *sz, int zero)
+{
+	int ret;
+
+	/* If using a "zero-pma", don't try to zero it, even if you asked */
+	if (zero && of_find_property(node, "zero-pma", &ret)) {
+		pr_info("  it's a 'zero-pma', not zeroing from s/w\n");
+		zero = 0;
+	}
+
+	if (zero) {
+		/* map as cacheable, non-guarded */
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+		void __iomem *tmpp = ioremap_cache(*addr, *sz);
+#else
+		void __iomem *tmpp = ioremap(*addr, *sz);
+#endif
+
+		if (!tmpp)
+			return -ENOMEM;
+		memset_io(tmpp, 0, *sz);
+		flush_dcache_range((unsigned long)tmpp,
+				   (unsigned long)tmpp + *sz);
+		iounmap(tmpp);
+	}
+
+	return 0;
+}
+
+/* TODO:
+ * - there is obviously no handling of errors,
+ * - the calls to qm_set_memory() hard-code the priority and CPC-stashing for
+ *   both memory resources to zero.
+ */
+static int __init fsl_qman_init(struct device_node *node)
+{
+	struct resource res;
+	resource_size_t len;
+	u32 __iomem *regs;
+	const char *s;
+	int ret, standby = 0;
+	u16 id;
+	u8 major, minor, cfg;
+	ret = of_address_to_resource(node, 0, &res);
+	if (ret) {
+		pr_err("Can't get %s property '%s'\n", node->full_name, "reg");
+		return ret;
+	}
+	s = of_get_property(node, "fsl,hv-claimable", &ret);
+	if (s && !strcmp(s, "standby"))
+		standby = 1;
+	if (!standby) {
+		ret = parse_mem_property(node, "fsl,qman-fqd",
+					&fqd_a, &fqd_sz, 1);
+		pr_info("qman-fqd addr 0x%llx size 0x%zx\n",
+				(unsigned long long)fqd_a, fqd_sz);
+		BUG_ON(ret);
+		ret = parse_mem_property(node, "fsl,qman-pfdr",
+					&pfdr_a, &pfdr_sz, 0);
+		pr_info("qman-pfdr addr 0x%llx size 0x%zx\n",
+				(unsigned long long)pfdr_a, pfdr_sz);
+		BUG_ON(ret);
+	}
+	/* Global configuration */
+	len = resource_size(&res);
+	if (len != (unsigned long)len)
+		return -EINVAL;
+	regs = ioremap(res.start, (unsigned long)len);
+	qm = qm_create(regs);
+	qm_node = node;
+	qm_get_version(qm, &id, &major, &minor, &cfg);
+	pr_info("Qman ver:%04x,%02x,%02x,%02x\n", id, major, minor, cfg);
+	if (!qman_ip_rev) {
+		if ((major == 1) && (minor == 0)) {
+			pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n");
+			iounmap(regs);
+			return -ENODEV;
+		} else if ((major == 1) && (minor == 1))
+			qman_ip_rev = QMAN_REV11;
+		else if	((major == 1) && (minor == 2))
+			qman_ip_rev = QMAN_REV12;
+		else if ((major == 2) && (minor == 0))
+			qman_ip_rev = QMAN_REV20;
+		else if ((major == 3) && (minor == 0))
+			qman_ip_rev = QMAN_REV30;
+		else if ((major == 3) && (minor == 1))
+			qman_ip_rev = QMAN_REV31;
+		else if ((major == 3) && (minor == 2))
+			qman_ip_rev = QMAN_REV32;
+		else {
+			pr_warn("unknown Qman version, default to rev1.1\n");
+			qman_ip_rev = QMAN_REV11;
+		}
+		qman_ip_cfg = cfg;
+	}
+
+	if (standby) {
+		pr_info("  -> in standby mode\n");
+		return 0;
+	}
+	return 0;
+}
+
+int qman_have_ccsr(void)
+{
+	return qm ? 1 : 0;
+}
+
+__init int qman_init_early(void)
+{
+	struct device_node *dn;
+	int ret;
+
+	for_each_compatible_node(dn, NULL, "fsl,qman") {
+		if (qm)
+			pr_err("%s: only one 'fsl,qman' allowed\n",
+				dn->full_name);
+		else {
+			if (!of_device_is_available(dn))
+				continue;
+
+			ret = fsl_qman_init(dn);
+			BUG_ON(ret);
+		}
+	}
+	return 0;
+}
+postcore_initcall_sync(qman_init_early);
+
+static void log_edata_bits(u32 bit_count)
+{
+	u32 i, j, mask = 0xffffffff;
+
+	pr_warn("Qman ErrInt, EDATA:\n");
+	i = bit_count/32;
+	if (bit_count%32) {
+		i++;
+		mask = ~(mask << bit_count%32);
+	}
+	j = 16-i;
+	pr_warn("  0x%08x\n", qm_in(EDATA(j)) & mask);
+	j++;
+	for (; j < 16; j++)
+		pr_warn("  0x%08x\n", qm_in(EDATA(j)));
+}
+
+static void log_additional_error_info(u32 isr_val, u32 ecsr_val)
+{
+	union qman_ecir ecir_val;
+	union qman_eadr eadr_val;
+
+	ecir_val.ecir_raw = qm_in(ECIR);
+	/* Is portal info valid */
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
+		union qman_ecir2 ecir2_val;
+		ecir2_val.ecir2_raw = qm_in(ECIR2);
+		if (ecsr_val & PORTAL_ECSR_ERR) {
+			pr_warn("Qman ErrInt: %s id %d\n",
+				(ecir2_val.info.portal_type) ?
+				"DCP" : "SWP", ecir2_val.info.portal_num);
+		}
+		if (ecsr_val & (FQID_ECSR_ERR | QM_EIRQ_IECE)) {
+			pr_warn("Qman ErrInt: ecir.fqid 0x%x\n",
+				ecir_val.info.fqid);
+		}
+		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
+			eadr_val.eadr_raw = qm_in(EADR);
+			pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n",
+				error_mdata[eadr_val.info_rev3.memid].txt,
+				error_mdata[eadr_val.info_rev3.memid].addr_mask
+					& eadr_val.info_rev3.eadr);
+			log_edata_bits(
+				error_mdata[eadr_val.info_rev3.memid].bits);
+		}
+	} else {
+		if (ecsr_val & PORTAL_ECSR_ERR) {
+			pr_warn("Qman ErrInt: %s id %d\n",
+				(ecir_val.info.portal_type) ?
+				"DCP" : "SWP", ecir_val.info.portal_num);
+		}
+		if (ecsr_val & FQID_ECSR_ERR) {
+			pr_warn("Qman ErrInt: ecir.fqid 0x%x\n",
+				ecir_val.info.fqid);
+		}
+		if (ecsr_val & (QM_EIRQ_SBEI|QM_EIRQ_MBEI)) {
+			eadr_val.eadr_raw = qm_in(EADR);
+			pr_warn("Qman ErrInt: EADR Memory: %s, 0x%x\n",
+				error_mdata[eadr_val.info.memid].txt,
+				error_mdata[eadr_val.info.memid].addr_mask
+					& eadr_val.info.eadr);
+			log_edata_bits(error_mdata[eadr_val.info.memid].bits);
+		}
+	}
+}
+
+/* Qman interrupt handler */
+static irqreturn_t qman_isr(int irq, void *ptr)
+{
+	u32 isr_val, ier_val, ecsr_val, isr_mask, i;
+
+	ier_val = qm_err_isr_enable_read(qm);
+	isr_val = qm_err_isr_status_read(qm);
+	ecsr_val = qm_in(ECSR);
+	isr_mask = isr_val & ier_val;
+
+	if (!isr_mask)
+		return IRQ_NONE;
+	for (i = 0; i < QMAN_HWE_COUNT; i++) {
+		if (qman_hwerr_txts[i].mask & isr_mask) {
+			pr_warn("Qman ErrInt: %s\n", qman_hwerr_txts[i].txt);
+			if (qman_hwerr_txts[i].mask & ecsr_val) {
+				log_additional_error_info(isr_mask, ecsr_val);
+				/* Re-arm error capture registers */
+				qm_out(ECSR, ecsr_val);
+			}
+			if (qman_hwerr_txts[i].mask & QMAN_ERRS_TO_UNENABLE) {
+				pr_devel("Qman un-enabling error 0x%x\n",
+					qman_hwerr_txts[i].mask);
+				ier_val &= ~qman_hwerr_txts[i].mask;
+				qm_err_isr_enable_write(qm, ier_val);
+			}
+		}
+	}
+	qm_err_isr_status_clear(qm, isr_val);
+	return IRQ_HANDLED;
+}
+
+static int __bind_irq(void)
+{
+	int ret, err_irq;
+
+	err_irq = of_irq_to_resource(qm_node, 0, NULL);
+	if (err_irq == 0) {
+		pr_info("Can't get %s property '%s'\n", qm_node->full_name,
+			"interrupts");
+		return -ENODEV;
+	}
+	ret = request_irq(err_irq, qman_isr, IRQF_SHARED, "qman-err", qm_node);
+	if (ret)  {
+		pr_err("request_irq() failed %d for '%s'\n", ret,
+			qm_node->full_name);
+		return -ENODEV;
+	}
+	/* Write-to-clear any stale bits, (eg. starvation being asserted prior
+	 * to resource allocation during driver init). */
+	qm_err_isr_status_clear(qm, 0xffffffff);
+	/* Enable Error Interrupts */
+	qm_err_isr_enable_write(qm, 0xffffffff);
+	return 0;
+}
+
+int qman_init_ccsr(struct device_node *node)
+{
+	int ret;
+	if (!qman_have_ccsr())
+		return 0;
+	if (node != qm_node)
+		return -EINVAL;
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	/* TEMP for LS1043 : should be done in uboot */
+	qm_out(QCSP_BARE, 0x5);
+	qm_out(QCSP_BAR, 0x0);
+#endif
+	/* FQD memory */
+	qm_set_memory(qm, qm_memory_fqd, fqd_a, 1, 0, 0, fqd_sz);
+	/* PFDR memory */
+	qm_set_memory(qm, qm_memory_pfdr, pfdr_a, 1, 0, 0, pfdr_sz);
+	qm_init_pfdr(qm, 8, pfdr_sz / 64 - 8);
+	/* thresholds */
+	qm_set_pfdr_threshold(qm, 512, 64);
+	qm_set_sfdr_threshold(qm, 128);
+	/* clear stale PEBI bit from interrupt status register */
+	qm_err_isr_status_clear(qm, QM_EIRQ_PEBI);
+	/* corenet initiator settings */
+	qm_set_corenet_initiator(qm);
+	/* HID settings */
+	qm_set_hid(qm);
+	/* Set scheduling weights to defaults */
+	for (ret = qm_wq_first; ret <= qm_wq_last; ret++)
+		qm_set_wq_scheduling(qm, ret, 0, 0, 0, 0, 0, 0, 0);
+	/* We are not prepared to accept ERNs for hardware enqueues */
+	qm_set_dc(qm, qm_dc_portal_fman0, 1, 0);
+	qm_set_dc(qm, qm_dc_portal_fman1, 1, 0);
+	/* Initialise Error Interrupt Handler */
+	ret = __bind_irq();
+	if (ret)
+		return ret;
+	return 0;
+}
+
+#define LIO_CFG_LIODN_MASK 0x0fff0000
+void qman_liodn_fixup(u16 channel)
+{
+	static int done;
+	static u32 liodn_offset;
+	u32 before, after;
+	int idx = channel - QM_CHANNEL_SWPORTAL0;
+
+	if (!qman_have_ccsr())
+		return;
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		before = qm_in(REV3_QCSP_LIO_CFG(idx));
+	else
+		before = qm_in(QCSP_LIO_CFG(idx));
+	if (!done) {
+		liodn_offset = before & LIO_CFG_LIODN_MASK;
+		done = 1;
+		return;
+	}
+	after = (before & (~LIO_CFG_LIODN_MASK)) | liodn_offset;
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		qm_out(REV3_QCSP_LIO_CFG(idx), after);
+	else
+		qm_out(QCSP_LIO_CFG(idx), after);
+}
+
+#define IO_CFG_SDEST_MASK 0x00ff0000
+int qman_set_sdest(u16 channel, unsigned int cpu_idx)
+{
+	int idx = channel - QM_CHANNEL_SWPORTAL0;
+	u32 before, after;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+	if ((qman_ip_rev & 0xFF00) == QMAN_REV31) {
+		/* LS1043A - only one L2 cache */
+		cpu_idx = 0;
+	}
+
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
+		before = qm_in(REV3_QCSP_IO_CFG(idx));
+		/* Each pair of vcpu share the same SRQ(SDEST) */
+		cpu_idx /= 2;
+		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
+		qm_out(REV3_QCSP_IO_CFG(idx), after);
+	} else {
+		before = qm_in(QCSP_IO_CFG(idx));
+		after = (before & (~IO_CFG_SDEST_MASK)) | (cpu_idx << 16);
+		qm_out(QCSP_IO_CFG(idx), after);
+	}
+	return 0;
+}
+
+#define MISC_CFG_WPM_MASK 0x00000002
+int qm_set_wpm(int wpm)
+{
+	u32 before;
+	u32 after;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+
+	before = qm_in(MISC_CFG);
+	after = (before & (~MISC_CFG_WPM_MASK)) | (wpm << 1);
+	qm_out(MISC_CFG, after);
+	return 0;
+}
+
+int qm_get_wpm(int *wpm)
+{
+	u32 before;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+
+	before = qm_in(MISC_CFG);
+	*wpm = (before & MISC_CFG_WPM_MASK) >> 1;
+	return 0;
+}
+
+/* CEETM_CFG_PRES register has PRES field which is calculated by:
+ *    PRES = (2^22 / credit update reference period) * QMan clock period
+ *         = (2^22 * 10^9)/ CONFIG_QMAN_CEETM_UPDATE_PERIOD) / qman_clk
+ */
+
+int qman_ceetm_set_prescaler(enum qm_dc_portal portal)
+{
+	u64 temp;
+	u16 pres;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+
+	temp = 0x400000 * 100;
+	do_div(temp, CONFIG_QMAN_CEETM_UPDATE_PERIOD);
+	temp *= 10000000;
+	do_div(temp, qman_clk);
+	pres = (u16) temp;
+	qm_out(CEETM_CFG_IDX, portal);
+	qm_out(CEETM_CFG_PRES, pres);
+	return 0;
+}
+
+int qman_ceetm_get_prescaler(u16 *pres)
+{
+	if (!qman_have_ccsr())
+		return -ENODEV;
+	*pres = (u16)qm_in(CEETM_CFG_PRES);
+	return 0;
+}
+
+#define DCP_CFG_CEETME_MASK 0xFFFF0000
+#define QM_SP_ENABLE_CEETM(n) (0x80000000 >> (n))
+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal)
+{
+	u32 dcp_cfg;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+
+	dcp_cfg = qm_in(DCP_CFG(portal));
+	dcp_cfg |= QM_SP_ENABLE_CEETM(sub_portal);
+	qm_out(DCP_CFG(portal), dcp_cfg);
+	return 0;
+}
+
+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal)
+{
+	u32 dcp_cfg;
+
+	if (!qman_have_ccsr())
+		return -ENODEV;
+	dcp_cfg = qm_in(DCP_CFG(portal));
+	dcp_cfg &= ~(QM_SP_ENABLE_CEETM(sub_portal));
+	qm_out(DCP_CFG(portal), dcp_cfg);
+	return 0;
+}
+
+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num)
+{
+	if (!qman_have_ccsr())
+		return -ENODEV;
+	*num = qm_in(CEETM_XSFDR_IN_USE);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_get_xsfdr);
+
+#ifdef CONFIG_SYSFS
+
+#define DRV_NAME	"fsl-qman"
+#define DCP_MAX_ID	3
+#define DCP_MIN_ID	0
+
+static ssize_t show_pfdr_fpc(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_FPC));
+};
+
+static ssize_t show_dlm_avg(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	u32 data;
+	int i;
+
+	if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i))
+		return -EINVAL;
+	if (i < DCP_MIN_ID || i > DCP_MAX_ID)
+		return -EINVAL;
+	data = qm_in(DCP_DLM_AVG(i));
+	return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
+			(data & 0x000000ff)*390625);
+};
+
+static ssize_t set_dlm_avg(struct device *dev,
+	struct device_attribute *dev_attr, const char *buf, size_t count)
+{
+	unsigned long val;
+	int i;
+
+	if (!sscanf(dev_attr->attr.name, "dcp%d_dlm_avg", &i))
+		return -EINVAL;
+	if (i < DCP_MIN_ID || i > DCP_MAX_ID)
+		return -EINVAL;
+	if (kstrtoul(buf, 0, &val)) {
+		dev_dbg(dev, "invalid input %s\n", buf);
+		return -EINVAL;
+	}
+	qm_out(DCP_DLM_AVG(i), val);
+	return count;
+};
+
+static ssize_t show_pfdr_cfg(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(PFDR_CFG));
+};
+
+static ssize_t set_pfdr_cfg(struct device *dev,
+	struct device_attribute *dev_attr, const char *buf, size_t count)
+{
+	unsigned long val;
+
+	if (kstrtoul(buf, 0, &val)) {
+		dev_dbg(dev, "invalid input %s\n", buf);
+		return -EINVAL;
+	}
+	qm_out(PFDR_CFG, val);
+	return count;
+};
+
+static ssize_t show_sfdr_in_use(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SFDR_IN_USE));
+};
+
+static ssize_t show_idle_stat(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(IDLE_STAT));
+};
+
+static ssize_t show_ci_rlm_avg(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	u32 data = qm_in(CI_RLM_AVG);
+	return snprintf(buf, PAGE_SIZE, "%d.%08d\n", data>>8,
+			(data & 0x000000ff)*390625);
+};
+
+static ssize_t set_ci_rlm_avg(struct device *dev,
+	struct device_attribute *dev_attr, const char *buf, size_t count)
+{
+	unsigned long val;
+
+	if (kstrtoul(buf, 0, &val)) {
+		dev_dbg(dev, "invalid input %s\n", buf);
+		return -EINVAL;
+	}
+	qm_out(CI_RLM_AVG, val);
+	return count;
+};
+
+static ssize_t show_err_isr(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "0x%08x\n", qm_in(ERR_ISR));
+};
+
+#define SBEC_MAX_ID	14
+#define SBEC_MIN_ID	0
+
+static ssize_t show_sbec(struct device *dev,
+	struct device_attribute *dev_attr, char *buf)
+{
+	int i;
+
+	if (!sscanf(dev_attr->attr.name, "sbec_%d", &i))
+		return -EINVAL;
+	if (i < SBEC_MIN_ID || i > SBEC_MAX_ID)
+		return -EINVAL;
+	return snprintf(buf, PAGE_SIZE, "%u\n", qm_in(SBEC(i)));
+};
+
+static DEVICE_ATTR(pfdr_fpc, S_IRUSR, show_pfdr_fpc, NULL);
+static DEVICE_ATTR(pfdr_cfg, S_IRUSR, show_pfdr_cfg, set_pfdr_cfg);
+static DEVICE_ATTR(idle_stat, S_IRUSR, show_idle_stat, NULL);
+static DEVICE_ATTR(ci_rlm_avg, (S_IRUSR|S_IWUSR),
+		show_ci_rlm_avg, set_ci_rlm_avg);
+static DEVICE_ATTR(err_isr, S_IRUSR, show_err_isr, NULL);
+static DEVICE_ATTR(sfdr_in_use, S_IRUSR, show_sfdr_in_use, NULL);
+
+static DEVICE_ATTR(dcp0_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
+static DEVICE_ATTR(dcp1_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
+static DEVICE_ATTR(dcp2_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
+static DEVICE_ATTR(dcp3_dlm_avg, (S_IRUSR|S_IWUSR), show_dlm_avg, set_dlm_avg);
+
+static DEVICE_ATTR(sbec_0, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_1, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_2, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_3, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_4, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_5, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_6, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_7, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_8, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_9, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_10, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_11, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_12, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_13, S_IRUSR, show_sbec, NULL);
+static DEVICE_ATTR(sbec_14, S_IRUSR, show_sbec, NULL);
+
+static struct attribute *qman_dev_attributes[] = {
+	&dev_attr_pfdr_fpc.attr,
+	&dev_attr_pfdr_cfg.attr,
+	&dev_attr_idle_stat.attr,
+	&dev_attr_ci_rlm_avg.attr,
+	&dev_attr_err_isr.attr,
+	&dev_attr_dcp0_dlm_avg.attr,
+	&dev_attr_dcp1_dlm_avg.attr,
+	&dev_attr_dcp2_dlm_avg.attr,
+	&dev_attr_dcp3_dlm_avg.attr,
+	/* sfdr_in_use will be added if necessary */
+	NULL
+};
+
+static struct attribute *qman_dev_ecr_attributes[] = {
+	&dev_attr_sbec_0.attr,
+	&dev_attr_sbec_1.attr,
+	&dev_attr_sbec_2.attr,
+	&dev_attr_sbec_3.attr,
+	&dev_attr_sbec_4.attr,
+	&dev_attr_sbec_5.attr,
+	&dev_attr_sbec_6.attr,
+	&dev_attr_sbec_7.attr,
+	&dev_attr_sbec_8.attr,
+	&dev_attr_sbec_9.attr,
+	&dev_attr_sbec_10.attr,
+	&dev_attr_sbec_11.attr,
+	&dev_attr_sbec_12.attr,
+	&dev_attr_sbec_13.attr,
+	&dev_attr_sbec_14.attr,
+	NULL
+};
+
+/* root level */
+static const struct attribute_group qman_dev_attr_grp = {
+	.name = NULL,
+	.attrs = qman_dev_attributes
+};
+static const struct attribute_group qman_dev_ecr_grp = {
+	.name = "error_capture",
+	.attrs = qman_dev_ecr_attributes
+};
+
+static int of_fsl_qman_remove(struct platform_device *ofdev)
+{
+	sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
+	return 0;
+};
+
+static int of_fsl_qman_probe(struct platform_device *ofdev)
+{
+	int ret;
+
+	ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
+	if (ret)
+		goto done;
+	ret = sysfs_add_file_to_group(&ofdev->dev.kobj,
+		&dev_attr_sfdr_in_use.attr, qman_dev_attr_grp.name);
+	if (ret)
+		goto del_group_0;
+	ret = sysfs_create_group(&ofdev->dev.kobj, &qman_dev_ecr_grp);
+	if (ret)
+		goto del_group_0;
+
+	goto done;
+
+del_group_0:
+	sysfs_remove_group(&ofdev->dev.kobj, &qman_dev_attr_grp);
+done:
+	if (ret)
+		dev_err(&ofdev->dev,
+				"Cannot create dev attributes ret=%d\n", ret);
+	return ret;
+};
+
+static struct of_device_id of_fsl_qman_ids[] = {
+	{
+		.compatible = "fsl,qman",
+	},
+	{}
+};
+MODULE_DEVICE_TABLE(of, of_fsl_qman_ids);
+
+#ifdef CONFIG_SUSPEND
+
+static u32 saved_isdr;
+static int qman_pm_suspend_noirq(struct device *dev)
+{
+	uint32_t idle_state;
+
+	suspend_unused_qportal();
+	/* save isdr, disable all, clear isr */
+	saved_isdr = qm_err_isr_disable_read(qm);
+	qm_err_isr_disable_write(qm, 0xffffffff);
+	qm_err_isr_status_clear(qm, 0xffffffff);
+	idle_state = qm_in(IDLE_STAT);
+	if (!(idle_state & 0x1)) {
+		pr_err("Qman not idle 0x%x aborting\n", idle_state);
+		qm_err_isr_disable_write(qm, saved_isdr);
+		resume_unused_qportal();
+		return -EBUSY;
+	}
+#ifdef CONFIG_PM_DEBUG
+	pr_info("Qman suspend code, IDLE_STAT = 0x%x\n", idle_state);
+#endif
+	return 0;
+}
+
+static int qman_pm_resume_noirq(struct device *dev)
+{
+	/* restore isdr */
+	qm_err_isr_disable_write(qm, saved_isdr);
+	resume_unused_qportal();
+	return 0;
+}
+#else
+#define qman_pm_suspend_noirq	NULL
+#define qman_pm_resume_noirq	NULL
+#endif
+
+static const struct dev_pm_ops qman_pm_ops = {
+	.suspend_noirq = qman_pm_suspend_noirq,
+	.resume_noirq = qman_pm_resume_noirq,
+};
+
+static struct platform_driver of_fsl_qman_driver = {
+	.driver = {
+		.owner = THIS_MODULE,
+		.name = DRV_NAME,
+		.of_match_table = of_fsl_qman_ids,
+		.pm = &qman_pm_ops,
+	},
+	.probe = of_fsl_qman_probe,
+	.remove      = of_fsl_qman_remove,
+};
+
+static int qman_ctrl_init(void)
+{
+	return platform_driver_register(&of_fsl_qman_driver);
+}
+
+static void qman_ctrl_exit(void)
+{
+	platform_driver_unregister(&of_fsl_qman_driver);
+}
+
+module_init(qman_ctrl_init);
+module_exit(qman_ctrl_exit);
+
+#endif /* CONFIG_SYSFS */
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_debugfs.c
@@ -0,0 +1,1594 @@
+/* Copyright 2010-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#include "qman_private.h"
+
+#define MAX_FQID (0x00ffffff)
+#define QM_FQD_BLOCK_SIZE     64
+#define QM_FQD_AR             (0xC10)
+
+static u32 fqid_max;
+static u64 qman_ccsr_start;
+static u64 qman_ccsr_size;
+
+static const char * const state_txt[] = {
+	"Out of Service",
+	"Retired",
+	"Tentatively Scheduled",
+	"Truly Scheduled",
+	"Parked",
+	"Active, Active Held or Held Suspended",
+	"Unknown State 6",
+	"Unknown State 7",
+	NULL,
+};
+
+static const u8 fqd_states[] = {
+	QM_MCR_NP_STATE_OOS, QM_MCR_NP_STATE_RETIRED, QM_MCR_NP_STATE_TEN_SCHED,
+	QM_MCR_NP_STATE_TRU_SCHED, QM_MCR_NP_STATE_PARKED,
+	QM_MCR_NP_STATE_ACTIVE};
+
+struct mask_to_text {
+	u16 mask;
+	const char *txt;
+};
+
+struct mask_filter_s {
+	u16 mask;
+	u8 filter;
+};
+
+static const struct mask_filter_s mask_filter[] = {
+	{QM_FQCTRL_PREFERINCACHE, 0},
+	{QM_FQCTRL_PREFERINCACHE, 1},
+	{QM_FQCTRL_HOLDACTIVE, 0},
+	{QM_FQCTRL_HOLDACTIVE, 1},
+	{QM_FQCTRL_AVOIDBLOCK, 0},
+	{QM_FQCTRL_AVOIDBLOCK, 1},
+	{QM_FQCTRL_FORCESFDR, 0},
+	{QM_FQCTRL_FORCESFDR, 1},
+	{QM_FQCTRL_CPCSTASH, 0},
+	{QM_FQCTRL_CPCSTASH, 1},
+	{QM_FQCTRL_CTXASTASHING, 0},
+	{QM_FQCTRL_CTXASTASHING, 1},
+	{QM_FQCTRL_ORP, 0},
+	{QM_FQCTRL_ORP, 1},
+	{QM_FQCTRL_TDE, 0},
+	{QM_FQCTRL_TDE, 1},
+	{QM_FQCTRL_CGE, 0},
+	{QM_FQCTRL_CGE, 1}
+};
+
+static const struct mask_to_text fq_ctrl_text_list[] = {
+	{
+		.mask = QM_FQCTRL_PREFERINCACHE,
+		.txt = "Prefer in cache",
+	},
+	{
+		.mask = QM_FQCTRL_HOLDACTIVE,
+		.txt =  "Hold active in portal",
+	},
+	{
+		.mask = QM_FQCTRL_AVOIDBLOCK,
+		.txt = "Avoid Blocking",
+	},
+	{
+		.mask = QM_FQCTRL_FORCESFDR,
+		.txt = "High-priority SFDRs",
+	},
+	{
+		.mask = QM_FQCTRL_CPCSTASH,
+		.txt = "CPC Stash Enable",
+	},
+	{
+		.mask = QM_FQCTRL_CTXASTASHING,
+		.txt =  "Context-A stashing",
+	},
+	{
+		.mask = QM_FQCTRL_ORP,
+		.txt =  "ORP Enable",
+	},
+	{
+		.mask = QM_FQCTRL_TDE,
+		.txt = "Tail-Drop Enable",
+	},
+	{
+		.mask = QM_FQCTRL_CGE,
+		.txt = "Congestion Group Enable",
+	},
+	{
+		.mask = 0,
+		.txt = NULL,
+	}
+};
+
+static const char *get_fqd_ctrl_text(u16 mask)
+{
+	int i = 0;
+
+	while (fq_ctrl_text_list[i].txt != NULL) {
+		if (fq_ctrl_text_list[i].mask == mask)
+			return fq_ctrl_text_list[i].txt;
+		i++;
+	}
+	return NULL;
+}
+
+static const struct mask_to_text stashing_text_list[] = {
+	{
+		.mask = QM_STASHING_EXCL_CTX,
+		.txt = "FQ Ctx Stash"
+	},
+	{
+		.mask = QM_STASHING_EXCL_DATA,
+		.txt =  "Frame Data Stash",
+	},
+	{
+		.mask = QM_STASHING_EXCL_ANNOTATION,
+		.txt = "Frame Annotation Stash",
+	},
+	{
+		.mask = 0,
+		.txt = NULL,
+	},
+};
+
+static int user_input_convert(const char __user *user_buf, size_t count,
+				unsigned long *val)
+{
+	char buf[12];
+
+	if (count > sizeof(buf) - 1)
+		return -EINVAL;
+	if (copy_from_user(buf, user_buf, count))
+		return -EFAULT;
+	buf[count] = '\0';
+	if (kstrtoul(buf, 0, val))
+		return -EINVAL;
+	return 0;
+}
+
+struct line_buffer_fq {
+	u32 buf[8];
+	u32 buf_cnt;
+	int line_cnt;
+};
+
+static void add_to_line_buffer(struct line_buffer_fq *line_buf, u32 fqid,
+			struct seq_file *file)
+{
+	line_buf->buf[line_buf->buf_cnt] = fqid;
+	line_buf->buf_cnt++;
+	if (line_buf->buf_cnt == 8) {
+		/* Buffer is full, flush it */
+		if (line_buf->line_cnt != 0)
+			seq_puts(file, ",\n");
+		seq_printf(file, "0x%06x,0x%06x,0x%06x,0x%06x,0x%06x,"
+			"0x%06x,0x%06x,0x%06x",
+			line_buf->buf[0], line_buf->buf[1], line_buf->buf[2],
+			line_buf->buf[3], line_buf->buf[4], line_buf->buf[5],
+			line_buf->buf[6], line_buf->buf[7]);
+		line_buf->buf_cnt = 0;
+		line_buf->line_cnt++;
+	}
+}
+
+static void flush_line_buffer(struct line_buffer_fq *line_buf,
+				struct seq_file *file)
+{
+	if (line_buf->buf_cnt) {
+		int y = 0;
+		if (line_buf->line_cnt != 0)
+			seq_puts(file, ",\n");
+		while (y != line_buf->buf_cnt) {
+			if (y+1 == line_buf->buf_cnt)
+				seq_printf(file, "0x%06x", line_buf->buf[y]);
+			else
+				seq_printf(file, "0x%06x,", line_buf->buf[y]);
+			y++;
+		}
+		line_buf->line_cnt++;
+	}
+	if (line_buf->line_cnt)
+		seq_putc(file, '\n');
+}
+
+static struct dentry *dfs_root; /* debugfs root directory */
+
+/*******************************************************************************
+ *  Query Frame Queue Non Programmable Fields
+ ******************************************************************************/
+struct query_fq_np_fields_data_s {
+	u32 fqid;
+};
+static struct query_fq_np_fields_data_s query_fq_np_fields_data = {
+	.fqid = 1,
+};
+
+static int query_fq_np_fields_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_queryfq_np np;
+	struct qman_fq fq;
+
+	fq.fqid = query_fq_np_fields_data.fqid;
+	ret = qman_query_fq_np(&fq, &np);
+	if (ret)
+		return ret;
+	/* Print state */
+	seq_printf(file, "Query FQ Non Programmable Fields Result fqid 0x%x\n",
+			fq.fqid);
+	seq_printf(file, " force eligible pending: %s\n",
+		(np.state & QM_MCR_NP_STATE_FE) ? "yes" : "no");
+	seq_printf(file, " retirement pending: %s\n",
+		(np.state & QM_MCR_NP_STATE_R) ? "yes" : "no");
+	seq_printf(file, " state: %s\n",
+		state_txt[np.state & QM_MCR_NP_STATE_MASK]);
+	seq_printf(file, " fq_link: 0x%x\n", np.fqd_link);
+	seq_printf(file, " odp_seq: %u\n", np.odp_seq);
+	seq_printf(file, " orp_nesn: %u\n", np.orp_nesn);
+	seq_printf(file, " orp_ea_hseq: %u\n", np.orp_ea_hseq);
+	seq_printf(file, " orp_ea_tseq: %u\n", np.orp_ea_tseq);
+	seq_printf(file, " orp_ea_hptr: 0x%x\n", np.orp_ea_hptr);
+	seq_printf(file, " orp_ea_tptr: 0x%x\n", np.orp_ea_tptr);
+	seq_printf(file, " pfdr_hptr: 0x%x\n", np.pfdr_hptr);
+	seq_printf(file, " pfdr_tptr: 0x%x\n", np.pfdr_tptr);
+	seq_printf(file, " is: ics_surp contains a %s\n",
+		(np.is) ? "deficit" : "surplus");
+	seq_printf(file, " ics_surp: %u\n", np.ics_surp);
+	seq_printf(file, " byte_cnt: %u\n", np.byte_cnt);
+	seq_printf(file, " frm_cnt: %u\n", np.frm_cnt);
+	seq_printf(file, " ra1_sfdr: 0x%x\n", np.ra1_sfdr);
+	seq_printf(file, " ra2_sfdr: 0x%x\n", np.ra2_sfdr);
+	seq_printf(file, " od1_sfdr: 0x%x\n", np.od1_sfdr);
+	seq_printf(file, " od2_sfdr: 0x%x\n", np.od2_sfdr);
+	seq_printf(file, " od3_sfdr: 0x%x\n", np.od3_sfdr);
+	return 0;
+}
+
+static int query_fq_np_fields_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, query_fq_np_fields_show, NULL);
+}
+
+static ssize_t query_fq_np_fields_write(struct file *f,
+			const char __user *buf, size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > MAX_FQID)
+		return -EINVAL;
+	query_fq_np_fields_data.fqid = (u32)val;
+	return count;
+}
+
+static const struct file_operations query_fq_np_fields_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_fq_np_fields_open,
+	.read           = seq_read,
+	.write		= query_fq_np_fields_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  Frame Queue Programmable Fields
+ ******************************************************************************/
+struct query_fq_fields_data_s {
+	u32 fqid;
+};
+
+static struct query_fq_fields_data_s query_fq_fields_data = {
+	.fqid = 1,
+};
+
+static int query_fq_fields_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_fqd fqd;
+	struct qman_fq fq;
+	int i = 0;
+
+	memset(&fqd, 0, sizeof(struct qm_fqd));
+	fq.fqid = query_fq_fields_data.fqid;
+	ret = qman_query_fq(&fq, &fqd);
+	if (ret)
+		return ret;
+	seq_printf(file, "Query FQ Programmable Fields Result fqid 0x%x\n",
+			fq.fqid);
+	seq_printf(file, " orprws: %u\n", fqd.orprws);
+	seq_printf(file, " oa: %u\n", fqd.oa);
+	seq_printf(file, " olws: %u\n", fqd.olws);
+
+	seq_printf(file, " cgid: %u\n", fqd.cgid);
+
+	if ((fqd.fq_ctrl & QM_FQCTRL_MASK) == 0)
+		seq_puts(file, " fq_ctrl: None\n");
+	else {
+		i = 0;
+		seq_puts(file, " fq_ctrl:\n");
+		while (fq_ctrl_text_list[i].txt != NULL) {
+			if ((fqd.fq_ctrl & QM_FQCTRL_MASK) &
+					fq_ctrl_text_list[i].mask)
+				seq_printf(file, "  %s\n",
+					fq_ctrl_text_list[i].txt);
+			i++;
+		}
+	}
+	seq_printf(file, " dest_channel: %u\n", fqd.dest.channel);
+	seq_printf(file, " dest_wq: %u\n", fqd.dest.wq);
+	seq_printf(file, " ics_cred: %u\n", fqd.ics_cred);
+	seq_printf(file, " td_mant: %u\n", fqd.td.mant);
+	seq_printf(file, " td_exp: %u\n", fqd.td.exp);
+
+	seq_printf(file, " ctx_b: 0x%x\n", fqd.context_b);
+
+	seq_printf(file, " ctx_a: 0x%llx\n", qm_fqd_stashing_get64(&fqd));
+	/* Any stashing configured */
+	if ((fqd.context_a.stashing.exclusive & 0x7) == 0)
+		seq_puts(file, " ctx_a_stash_exclusive: None\n");
+	else {
+		seq_puts(file, " ctx_a_stash_exclusive:\n");
+		i = 0;
+		while (stashing_text_list[i].txt != NULL) {
+			if ((fqd.fq_ctrl & 0x7) & stashing_text_list[i].mask)
+				seq_printf(file, "  %s\n",
+					stashing_text_list[i].txt);
+			i++;
+		}
+	}
+	seq_printf(file, " ctx_a_stash_annotation_cl: %u\n",
+			fqd.context_a.stashing.annotation_cl);
+	seq_printf(file, " ctx_a_stash_data_cl: %u\n",
+			fqd.context_a.stashing.data_cl);
+	seq_printf(file, " ctx_a_stash_context_cl: %u\n",
+			fqd.context_a.stashing.context_cl);
+	return 0;
+}
+
+static int query_fq_fields_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, query_fq_fields_show, NULL);
+}
+
+static ssize_t query_fq_fields_write(struct file *f,
+			const char __user *buf, size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > MAX_FQID)
+		return -EINVAL;
+	query_fq_fields_data.fqid = (u32)val;
+	return count;
+}
+
+static const struct file_operations query_fq_fields_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_fq_fields_open,
+	.read           = seq_read,
+	.write		= query_fq_fields_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ * Query WQ lengths
+ ******************************************************************************/
+struct query_wq_lengths_data_s {
+	union {
+		u16 channel_wq; /* ignores wq (3 lsbits) */
+		struct {
+			u16 id:13; /* qm_channel */
+			u16 __reserved:3;
+		} __packed channel;
+	};
+};
+static struct query_wq_lengths_data_s query_wq_lengths_data;
+static int query_wq_lengths_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_querywq wq;
+	int i;
+
+	memset(&wq, 0, sizeof(struct qm_mcr_querywq));
+	wq.channel.id = query_wq_lengths_data.channel.id;
+	ret = qman_query_wq(0, &wq);
+	if (ret)
+		return ret;
+	seq_printf(file, "Query Result For Channel: 0x%x\n", wq.channel.id);
+	for (i = 0; i < 8; i++)
+		/* mask out upper 4 bits since they are not part of length */
+		seq_printf(file, " wq%d_len : %u\n", i, wq.wq_len[i] & 0x0fff);
+	return 0;
+}
+
+static int query_wq_lengths_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, query_wq_lengths_show, NULL);
+}
+
+static ssize_t query_wq_lengths_write(struct file *f,
+			const char __user *buf, size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > 0xfff8)
+		return -EINVAL;
+	query_wq_lengths_data.channel.id = (u16)val;
+	return count;
+}
+
+static const struct file_operations query_wq_lengths_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_wq_lengths_open,
+	.read           = seq_read,
+	.write		= query_wq_lengths_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  Query CGR
+ ******************************************************************************/
+struct query_cgr_s {
+	u8 cgid;
+};
+static struct query_cgr_s query_cgr_data;
+
+static int query_cgr_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_querycgr cgrd;
+	struct qman_cgr cgr;
+	int i, j;
+	u32 mask;
+
+	memset(&cgr, 0, sizeof(cgr));
+	memset(&cgrd, 0, sizeof(cgrd));
+	cgr.cgrid = query_cgr_data.cgid;
+	ret = qman_query_cgr(&cgr, &cgrd);
+	if (ret)
+		return ret;
+	seq_printf(file, "Query CGR id 0x%x\n", cgr.cgrid);
+	seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		cgrd.cgr.wr_parm_g.MA, cgrd.cgr.wr_parm_g.Mn,
+		cgrd.cgr.wr_parm_g.SA, cgrd.cgr.wr_parm_g.Sn,
+		cgrd.cgr.wr_parm_g.Pn);
+
+	seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		cgrd.cgr.wr_parm_y.MA, cgrd.cgr.wr_parm_y.Mn,
+		cgrd.cgr.wr_parm_y.SA, cgrd.cgr.wr_parm_y.Sn,
+		cgrd.cgr.wr_parm_y.Pn);
+
+	seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		cgrd.cgr.wr_parm_r.MA, cgrd.cgr.wr_parm_r.Mn,
+		cgrd.cgr.wr_parm_r.SA, cgrd.cgr.wr_parm_r.Sn,
+		cgrd.cgr.wr_parm_r.Pn);
+
+	seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
+		cgrd.cgr.wr_en_g, cgrd.cgr.wr_en_y, cgrd.cgr.wr_en_r);
+
+	seq_printf(file, " cscn_en: %u\n", cgrd.cgr.cscn_en);
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30) {
+		seq_puts(file, " cscn_targ_dcp:\n");
+		mask = 0x80000000;
+		for (i = 0; i < 32; i++) {
+			if (cgrd.cgr.cscn_targ & mask)
+				seq_printf(file, "  send CSCN to dcp %u\n",
+								(31 - i));
+			mask >>= 1;
+		}
+
+		seq_puts(file, " cscn_targ_swp:\n");
+		for (i = 0; i < 4; i++) {
+			mask = 0x80000000;
+			for (j = 0; j < 32; j++) {
+				if (cgrd.cscn_targ_swp[i] & mask)
+					seq_printf(file, "  send CSCN to swp"
+						" %u\n", (127 - (i * 32) - j));
+				mask >>= 1;
+			}
+		}
+	} else {
+		seq_printf(file, " cscn_targ: %u\n", cgrd.cgr.cscn_targ);
+	}
+	seq_printf(file, " cstd_en: %u\n", cgrd.cgr.cstd_en);
+	seq_printf(file, " cs: %u\n", cgrd.cgr.cs);
+
+	seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n",
+		cgrd.cgr.cs_thres.TA, cgrd.cgr.cs_thres.Tn);
+
+	seq_printf(file, " mode: %s\n",
+		(cgrd.cgr.mode & QMAN_CGR_MODE_FRAME) ?
+		"frame count" : "byte count");
+	seq_printf(file, " i_bcnt: %llu\n", qm_mcr_querycgr_i_get64(&cgrd));
+	seq_printf(file, " a_bcnt: %llu\n", qm_mcr_querycgr_a_get64(&cgrd));
+
+	return 0;
+}
+
+static int query_cgr_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, query_cgr_show, NULL);
+}
+
+static ssize_t query_cgr_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > 0xff)
+		return -EINVAL;
+	query_cgr_data.cgid = (u8)val;
+	return count;
+}
+
+static const struct file_operations query_cgr_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_cgr_open,
+	.read           = seq_read,
+	.write		= query_cgr_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  Test Write CGR
+ ******************************************************************************/
+struct test_write_cgr_s {
+	u64 i_bcnt;
+	u8 cgid;
+};
+static struct test_write_cgr_s test_write_cgr_data;
+
+static int testwrite_cgr_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_cgrtestwrite result;
+	struct qman_cgr cgr;
+	u64 i_bcnt;
+
+	memset(&cgr, 0, sizeof(struct qman_cgr));
+	memset(&result, 0, sizeof(struct qm_mcr_cgrtestwrite));
+	cgr.cgrid = test_write_cgr_data.cgid;
+	i_bcnt = test_write_cgr_data.i_bcnt;
+	ret = qman_testwrite_cgr(&cgr, i_bcnt, &result);
+	if (ret)
+		return ret;
+	seq_printf(file, "CGR Test Write CGR id 0x%x\n", cgr.cgrid);
+	seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		result.cgr.wr_parm_g.MA, result.cgr.wr_parm_g.Mn,
+		result.cgr.wr_parm_g.SA, result.cgr.wr_parm_g.Sn,
+		result.cgr.wr_parm_g.Pn);
+	seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		result.cgr.wr_parm_y.MA, result.cgr.wr_parm_y.Mn,
+		result.cgr.wr_parm_y.SA, result.cgr.wr_parm_y.Sn,
+		result.cgr.wr_parm_y.Pn);
+	seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		result.cgr.wr_parm_r.MA, result.cgr.wr_parm_r.Mn,
+		result.cgr.wr_parm_r.SA, result.cgr.wr_parm_r.Sn,
+		result.cgr.wr_parm_r.Pn);
+	seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
+		result.cgr.wr_en_g, result.cgr.wr_en_y, result.cgr.wr_en_r);
+	seq_printf(file, " cscn_en: %u\n", result.cgr.cscn_en);
+	seq_printf(file, " cscn_targ: %u\n", result.cgr.cscn_targ);
+	seq_printf(file, " cstd_en: %u\n", result.cgr.cstd_en);
+	seq_printf(file, " cs: %u\n", result.cgr.cs);
+	seq_printf(file, " cs_thresh_TA: %u, cs_thresh_Tn: %u\n",
+		result.cgr.cs_thres.TA, result.cgr.cs_thres.Tn);
+
+	/* Add Mode for Si 2 */
+	seq_printf(file, " mode: %s\n",
+		(result.cgr.mode & QMAN_CGR_MODE_FRAME) ?
+		"frame count" : "byte count");
+
+	seq_printf(file, " i_bcnt: %llu\n",
+		qm_mcr_cgrtestwrite_i_get64(&result));
+	seq_printf(file, " a_bcnt: %llu\n",
+		qm_mcr_cgrtestwrite_a_get64(&result));
+	seq_printf(file, " wr_prob_g: %u\n", result.wr_prob_g);
+	seq_printf(file, " wr_prob_y: %u\n", result.wr_prob_y);
+	seq_printf(file, " wr_prob_r: %u\n", result.wr_prob_r);
+	return 0;
+}
+
+static int testwrite_cgr_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, testwrite_cgr_show, NULL);
+}
+
+static const struct file_operations testwrite_cgr_fops = {
+	.owner          = THIS_MODULE,
+	.open		= testwrite_cgr_open,
+	.read           = seq_read,
+	.release	= single_release,
+};
+
+
+static int testwrite_cgr_ibcnt_show(struct seq_file *file, void *offset)
+{
+	seq_printf(file, "i_bcnt: %llu\n", test_write_cgr_data.i_bcnt);
+	return 0;
+}
+static int testwrite_cgr_ibcnt_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, testwrite_cgr_ibcnt_show, NULL);
+}
+
+static ssize_t testwrite_cgr_ibcnt_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	test_write_cgr_data.i_bcnt = val;
+	return count;
+}
+
+static const struct file_operations teswrite_cgr_ibcnt_fops = {
+	.owner          = THIS_MODULE,
+	.open		= testwrite_cgr_ibcnt_open,
+	.read           = seq_read,
+	.write		= testwrite_cgr_ibcnt_write,
+	.release	= single_release,
+};
+
+static int testwrite_cgr_cgrid_show(struct seq_file *file, void *offset)
+{
+	seq_printf(file, "cgrid: %u\n", (u32)test_write_cgr_data.cgid);
+	return 0;
+}
+static int testwrite_cgr_cgrid_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, testwrite_cgr_cgrid_show, NULL);
+}
+
+static ssize_t testwrite_cgr_cgrid_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > 0xff)
+		return -EINVAL;
+	test_write_cgr_data.cgid = (u8)val;
+	return count;
+}
+
+static const struct file_operations teswrite_cgr_cgrid_fops = {
+	.owner          = THIS_MODULE,
+	.open		= testwrite_cgr_cgrid_open,
+	.read           = seq_read,
+	.write		= testwrite_cgr_cgrid_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  Query Congestion State
+ ******************************************************************************/
+static int query_congestion_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_querycongestion cs;
+	int i, j, in_cong = 0;
+	u32 mask;
+
+	memset(&cs, 0, sizeof(struct qm_mcr_querycongestion));
+	ret = qman_query_congestion(&cs);
+	if (ret)
+		return ret;
+	seq_puts(file, "Query Congestion Result\n");
+	for (i = 0; i < 8; i++) {
+		mask = 0x80000000;
+		for (j = 0; j < 32; j++) {
+			if (cs.state.__state[i] & mask) {
+				in_cong = 1;
+				seq_printf(file, " cg %u: %s\n", (i*32)+j,
+					"in congestion");
+			}
+			mask >>= 1;
+		}
+	}
+	if (!in_cong)
+		seq_puts(file, " All congestion groups not congested.\n");
+	return 0;
+}
+
+static int query_congestion_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, query_congestion_show, NULL);
+}
+
+static const struct file_operations query_congestion_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_congestion_open,
+	.read           = seq_read,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  Query CCGR
+ ******************************************************************************/
+struct query_ccgr_s {
+	u32 ccgid;
+};
+static struct query_ccgr_s query_ccgr_data;
+
+static int query_ccgr_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_ceetm_ccgr_query ccgr_query;
+	struct qm_mcc_ceetm_ccgr_query query_opts;
+	int i, j;
+	u32 mask;
+
+	memset(&ccgr_query, 0, sizeof(struct qm_mcr_ceetm_ccgr_query));
+	memset(&query_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_query));
+
+	if ((qman_ip_rev & 0xFF00) < QMAN_REV30)
+		return -EINVAL;
+
+	seq_printf(file, "Query CCGID %x\n", query_ccgr_data.ccgid);
+	query_opts.dcpid = ((query_ccgr_data.ccgid & 0xFF000000) >> 24);
+	query_opts.ccgrid = query_ccgr_data.ccgid & 0x000001FF;
+	ret = qman_ceetm_query_ccgr(&query_opts, &ccgr_query);
+	if (ret)
+		return ret;
+	seq_printf(file, "Query CCGR id %x in DCP %d\n", query_opts.ccgrid,
+						query_opts.dcpid);
+	seq_printf(file, " wr_parm_g MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		ccgr_query.cm_query.wr_parm_g.MA,
+		ccgr_query.cm_query.wr_parm_g.Mn,
+		ccgr_query.cm_query.wr_parm_g.SA,
+		ccgr_query.cm_query.wr_parm_g.Sn,
+		ccgr_query.cm_query.wr_parm_g.Pn);
+
+	seq_printf(file, " wr_parm_y MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		ccgr_query.cm_query.wr_parm_y.MA,
+		ccgr_query.cm_query.wr_parm_y.Mn,
+		ccgr_query.cm_query.wr_parm_y.SA,
+		ccgr_query.cm_query.wr_parm_y.Sn,
+		ccgr_query.cm_query.wr_parm_y.Pn);
+
+	seq_printf(file, " wr_parm_r MA: %u, Mn: %u, SA: %u, Sn: %u, Pn: %u\n",
+		ccgr_query.cm_query.wr_parm_r.MA,
+		ccgr_query.cm_query.wr_parm_r.Mn,
+		ccgr_query.cm_query.wr_parm_r.SA,
+		ccgr_query.cm_query.wr_parm_r.Sn,
+		ccgr_query.cm_query.wr_parm_r.Pn);
+
+	seq_printf(file, " wr_en_g: %u, wr_en_y: %u, we_en_r: %u\n",
+		ccgr_query.cm_query.ctl_wr_en_g,
+		ccgr_query.cm_query.ctl_wr_en_y,
+		ccgr_query.cm_query.ctl_wr_en_r);
+
+	seq_printf(file, " cscn_en: %u\n", ccgr_query.cm_query.ctl_cscn_en);
+	seq_puts(file, " cscn_targ_dcp:\n");
+	mask = 0x80000000;
+	for (i = 0; i < 32; i++) {
+		if (ccgr_query.cm_query.cscn_targ_dcp & mask)
+			seq_printf(file, "  send CSCN to dcp %u\n", (31 - i));
+		mask >>= 1;
+	}
+
+	seq_puts(file, " cscn_targ_swp:\n");
+	for (i = 0; i < 4; i++) {
+		mask = 0x80000000;
+		for (j = 0; j < 32; j++) {
+			if (ccgr_query.cm_query.cscn_targ_swp[i] & mask)
+				seq_printf(file, "  send CSCN to swp"
+					"%u\n", (127 - (i * 32) - j));
+			mask >>= 1;
+		}
+	}
+
+	seq_printf(file, " td_en: %u\n", ccgr_query.cm_query.ctl_td_en);
+
+	seq_printf(file, " cs_thresh_in_TA: %u, cs_thresh_in_Tn: %u\n",
+			ccgr_query.cm_query.cs_thres.TA,
+			ccgr_query.cm_query.cs_thres.Tn);
+
+	seq_printf(file, " cs_thresh_out_TA: %u, cs_thresh_out_Tn: %u\n",
+			ccgr_query.cm_query.cs_thres_x.TA,
+			ccgr_query.cm_query.cs_thres_x.Tn);
+
+	seq_printf(file, " td_thresh_TA: %u, td_thresh_Tn: %u\n",
+			ccgr_query.cm_query.td_thres.TA,
+			ccgr_query.cm_query.td_thres.Tn);
+
+	seq_printf(file, " mode: %s\n",
+			(ccgr_query.cm_query.ctl_mode &
+			QMAN_CGR_MODE_FRAME) ?
+			"frame count" : "byte count");
+	seq_printf(file, " i_cnt: %llu\n", (u64)ccgr_query.cm_query.i_cnt);
+	seq_printf(file, " a_cnt: %llu\n", (u64)ccgr_query.cm_query.a_cnt);
+
+	return 0;
+}
+
+static int query_ccgr_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, query_ccgr_show, NULL);
+}
+
+static ssize_t query_ccgr_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	query_ccgr_data.ccgid = val;
+	return count;
+}
+
+static const struct file_operations query_ccgr_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_ccgr_open,
+	.read           = seq_read,
+	.write		= query_ccgr_write,
+	.release	= single_release,
+};
+/*******************************************************************************
+ *  QMan register
+ ******************************************************************************/
+struct qman_register_s {
+	u32 val;
+};
+static struct qman_register_s qman_register_data;
+
+static void init_ccsrmempeek(void)
+{
+	struct device_node *dn;
+	const u32 *regaddr_p;
+
+	dn = of_find_compatible_node(NULL, NULL, "fsl,qman");
+	if (!dn) {
+		pr_info("No fsl,qman node\n");
+		return;
+	}
+	regaddr_p = of_get_address(dn, 0, &qman_ccsr_size, NULL);
+	if (!regaddr_p) {
+		of_node_put(dn);
+		return;
+	}
+	qman_ccsr_start = of_translate_address(dn, regaddr_p);
+	of_node_put(dn);
+}
+/* This function provides access to QMan ccsr memory map */
+static int qman_ccsrmempeek(u32 *val, u32 offset)
+{
+	void __iomem *addr;
+	u64 phys_addr;
+
+	if (!qman_ccsr_start)
+		return -EINVAL;
+
+	if (offset > (qman_ccsr_size - sizeof(u32)))
+		return -EINVAL;
+
+	phys_addr = qman_ccsr_start + offset;
+	addr = ioremap(phys_addr, sizeof(u32));
+	if (!addr) {
+		pr_err("ccsrmempeek, ioremap failed\n");
+		return -EINVAL;
+	}
+	*val = in_be32(addr);
+	iounmap(addr);
+	return 0;
+}
+
+static int qman_ccsrmempeek_show(struct seq_file *file, void *offset)
+{
+	u32 b;
+
+	qman_ccsrmempeek(&b, qman_register_data.val);
+	seq_printf(file, "QMan register offset = 0x%x\n",
+		   qman_register_data.val);
+	seq_printf(file, "value = 0x%08x\n", b);
+
+	return 0;
+}
+
+static int qman_ccsrmempeek_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_ccsrmempeek_show, NULL);
+}
+
+static ssize_t qman_ccsrmempeek_write(struct file *f, const char __user *buf,
+				size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	/* multiple of 4 */
+	if (val > (qman_ccsr_size - sizeof(u32))) {
+		pr_info("Input 0x%lx > 0x%llx\n",
+			val, (qman_ccsr_size - sizeof(u32)));
+		return -EINVAL;
+	}
+	if (val & 0x3) {
+		pr_info("Input 0x%lx not multiple of 4\n", val);
+		return -EINVAL;
+	}
+	qman_register_data.val = val;
+	return count;
+}
+
+static const struct file_operations qman_ccsrmempeek_fops = {
+	.owner          = THIS_MODULE,
+	.open		= qman_ccsrmempeek_open,
+	.read           = seq_read,
+	.write		= qman_ccsrmempeek_write,
+};
+
+/*******************************************************************************
+ *  QMan state
+ ******************************************************************************/
+static int qman_fqd_state_show(struct seq_file *file, void *offset)
+{
+	struct qm_mcr_queryfq_np np;
+	struct qman_fq fq;
+	struct line_buffer_fq line_buf;
+	int ret, i;
+	u8 *state = file->private;
+	u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)];
+
+	memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt));
+	memset(&line_buf, 0, sizeof(line_buf));
+
+	seq_printf(file, "List of fq ids in state: %s\n", state_txt[*state]);
+
+	for (i = 1; i < fqid_max; i++) {
+		fq.fqid = i;
+		ret = qman_query_fq_np(&fq, &np);
+		if (ret)
+			return ret;
+		if (*state == (np.state & QM_MCR_NP_STATE_MASK))
+			add_to_line_buffer(&line_buf, fq.fqid, file);
+		/* Keep a summary count of all states */
+		if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states))
+			qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++;
+	}
+	flush_line_buffer(&line_buf, file);
+
+	for (i = 0; i < ARRAY_SIZE(fqd_states); i++) {
+		seq_printf(file, "%s count = %u\n", state_txt[i],
+			   qm_fq_state_cnt[i]);
+	}
+	return 0;
+}
+
+static int qman_fqd_state_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_fqd_state_show, inode->i_private);
+}
+
+static const struct file_operations qman_fqd_state_fops =  {
+	.owner          = THIS_MODULE,
+	.open		= qman_fqd_state_open,
+	.read           = seq_read,
+};
+
+static int qman_fqd_ctrl_show(struct seq_file *file, void *offset)
+{
+	struct qm_fqd fqd;
+	struct qman_fq fq;
+	u32 fq_en_cnt = 0, fq_di_cnt = 0;
+	int ret, i;
+	struct mask_filter_s *data = file->private;
+	const char *ctrl_txt = get_fqd_ctrl_text(data->mask);
+	struct line_buffer_fq line_buf;
+
+	memset(&line_buf, 0, sizeof(line_buf));
+	seq_printf(file, "List of fq ids with: %s :%s\n",
+		ctrl_txt, (data->filter) ? "enabled" : "disabled");
+	for (i = 1; i < fqid_max; i++) {
+		fq.fqid = i;
+		memset(&fqd, 0, sizeof(struct qm_fqd));
+		ret = qman_query_fq(&fq, &fqd);
+		if (ret)
+			return ret;
+		if (data->filter) {
+			if (fqd.fq_ctrl & data->mask)
+				add_to_line_buffer(&line_buf, fq.fqid, file);
+		} else {
+			if (!(fqd.fq_ctrl & data->mask))
+				add_to_line_buffer(&line_buf, fq.fqid, file);
+		}
+		if (fqd.fq_ctrl & data->mask)
+			fq_en_cnt++;
+		else
+			fq_di_cnt++;
+	}
+	flush_line_buffer(&line_buf, file);
+
+	seq_printf(file, "Total FQD with: %s :  enabled = %u\n",
+		   ctrl_txt, fq_en_cnt);
+	seq_printf(file, "Total FQD with: %s : disabled = %u\n",
+		   ctrl_txt, fq_di_cnt);
+	return 0;
+}
+
+/*******************************************************************************
+ *  QMan ctrl CGE, TDE, ORP, CTX, CPC, SFDR, BLOCK, HOLD, CACHE
+ ******************************************************************************/
+static int qman_fqd_ctrl_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_fqd_ctrl_show, inode->i_private);
+}
+
+static const struct file_operations qman_fqd_ctrl_fops =  {
+	.owner          = THIS_MODULE,
+	.open		= qman_fqd_ctrl_open,
+	.read           = seq_read,
+};
+
+/*******************************************************************************
+ *  QMan ctrl summary
+ ******************************************************************************/
+/*******************************************************************************
+ *  QMan summary state
+ ******************************************************************************/
+static int qman_fqd_non_prog_summary_show(struct seq_file *file, void *offset)
+{
+	struct qm_mcr_queryfq_np np;
+	struct qman_fq fq;
+	int ret, i;
+	u32 qm_fq_state_cnt[ARRAY_SIZE(fqd_states)];
+
+	memset(qm_fq_state_cnt, 0, sizeof(qm_fq_state_cnt));
+
+	for (i = 1; i < fqid_max; i++) {
+		fq.fqid = i;
+		ret = qman_query_fq_np(&fq, &np);
+		if (ret)
+			return ret;
+		/* Keep a summary count of all states */
+		if ((np.state & QM_MCR_NP_STATE_MASK) < ARRAY_SIZE(fqd_states))
+			qm_fq_state_cnt[(np.state & QM_MCR_NP_STATE_MASK)]++;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(fqd_states); i++) {
+		seq_printf(file, "%s count = %u\n", state_txt[i],
+			   qm_fq_state_cnt[i]);
+	}
+	return 0;
+}
+
+static int qman_fqd_prog_summary_show(struct seq_file *file, void *offset)
+{
+	struct qm_fqd fqd;
+	struct qman_fq fq;
+	int ret, i , j;
+	u32 qm_prog_cnt[ARRAY_SIZE(mask_filter)/2];
+
+	memset(qm_prog_cnt, 0, sizeof(qm_prog_cnt));
+
+	for (i = 1; i < fqid_max; i++) {
+		memset(&fqd, 0, sizeof(struct qm_fqd));
+		fq.fqid = i;
+		ret = qman_query_fq(&fq, &fqd);
+		if (ret)
+			return ret;
+		/* Keep a summary count of all states */
+		for (j = 0; j < ARRAY_SIZE(mask_filter); j += 2)
+			if ((fqd.fq_ctrl & QM_FQCTRL_MASK) &
+					mask_filter[j].mask)
+				qm_prog_cnt[j/2]++;
+	}
+	for (i = 0; i < ARRAY_SIZE(mask_filter) / 2; i++) {
+		seq_printf(file, "%s count = %u\n",
+			get_fqd_ctrl_text(mask_filter[i*2].mask),
+			   qm_prog_cnt[i]);
+	}
+	return 0;
+}
+
+static int qman_fqd_summary_show(struct seq_file *file, void *offset)
+{
+	int ret;
+
+	/* Display summary of non programmable fields */
+	ret = qman_fqd_non_prog_summary_show(file, offset);
+	if (ret)
+		return ret;
+	seq_puts(file, "-----------------------------------------\n");
+	/* Display programmable fields */
+	ret = qman_fqd_prog_summary_show(file, offset);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int qman_fqd_summary_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_fqd_summary_show, NULL);
+}
+
+static const struct file_operations qman_fqd_summary_fops =  {
+	.owner          = THIS_MODULE,
+	.open		= qman_fqd_summary_open,
+	.read           = seq_read,
+};
+
+/*******************************************************************************
+ *  QMan destination work queue
+ ******************************************************************************/
+struct qman_dest_wq_s {
+	u16 wq_id;
+};
+static struct qman_dest_wq_s qman_dest_wq_data = {
+	.wq_id = 0,
+};
+
+static int qman_fqd_dest_wq_show(struct seq_file *file, void *offset)
+{
+	struct qm_fqd fqd;
+	struct qman_fq fq;
+	int ret, i;
+	u16 *wq, wq_id = qman_dest_wq_data.wq_id;
+	struct line_buffer_fq line_buf;
+
+	memset(&line_buf, 0, sizeof(line_buf));
+	/* use vmalloc : need to allocate large memory region and don't
+	 * require the memory to be physically contiguous. */
+	wq = vzalloc(sizeof(u16) * (0xFFFF+1));
+	if (!wq)
+		return -ENOMEM;
+
+	seq_printf(file, "List of fq ids with destination work queue id"
+			" = 0x%x\n", wq_id);
+
+	for (i = 1; i < fqid_max; i++) {
+		fq.fqid = i;
+		memset(&fqd, 0, sizeof(struct qm_fqd));
+		ret = qman_query_fq(&fq, &fqd);
+		if (ret) {
+			vfree(wq);
+			return ret;
+		}
+		if (wq_id == fqd.dest_wq)
+			add_to_line_buffer(&line_buf, fq.fqid, file);
+		wq[fqd.dest_wq]++;
+	}
+	flush_line_buffer(&line_buf, file);
+
+	seq_puts(file, "Summary of all FQD destination work queue values\n");
+	for (i = 0; i < 0xFFFF; i++) {
+		if (wq[i])
+			seq_printf(file, "Channel: 0x%x WQ: 0x%x WQ_ID: 0x%x, "
+				"count = %u\n", i >> 3, i & 0x3, i, wq[i]);
+	}
+	vfree(wq);
+	return 0;
+}
+
+static ssize_t qman_fqd_dest_wq_write(struct file *f, const char __user *buf,
+				      size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > 0xFFFF)
+		return -EINVAL;
+	qman_dest_wq_data.wq_id = val;
+	return count;
+}
+
+static int qman_fqd_dest_wq_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_fqd_dest_wq_show, NULL);
+}
+
+static const struct file_operations qman_fqd_dest_wq_fops =  {
+	.owner          = THIS_MODULE,
+	.open		= qman_fqd_dest_wq_open,
+	.read           = seq_read,
+	.write		= qman_fqd_dest_wq_write,
+};
+
+/*******************************************************************************
+ *  QMan Intra-Class Scheduling Credit
+ ******************************************************************************/
+static int qman_fqd_cred_show(struct seq_file *file, void *offset)
+{
+	struct qm_fqd fqd;
+	struct qman_fq fq;
+	int ret, i;
+	u32 fq_cnt = 0;
+	struct line_buffer_fq line_buf;
+
+	memset(&line_buf, 0, sizeof(line_buf));
+	seq_puts(file, "List of fq ids with Intra-Class Scheduling Credit > 0"
+			"\n");
+
+	for (i = 1; i < fqid_max; i++) {
+		fq.fqid = i;
+		memset(&fqd, 0, sizeof(struct qm_fqd));
+		ret = qman_query_fq(&fq, &fqd);
+		if (ret)
+			return ret;
+		if (fqd.ics_cred > 0) {
+			add_to_line_buffer(&line_buf, fq.fqid, file);
+			fq_cnt++;
+		}
+	}
+	flush_line_buffer(&line_buf, file);
+
+	seq_printf(file, "Total FQD with ics_cred > 0 = %d\n", fq_cnt);
+	return 0;
+}
+
+static int qman_fqd_cred_open(struct inode *inode, struct file *file)
+{
+	return single_open(file, qman_fqd_cred_show, NULL);
+}
+
+static const struct file_operations qman_fqd_cred_fops =  {
+	.owner          = THIS_MODULE,
+	.open		= qman_fqd_cred_open,
+	.read           = seq_read,
+};
+
+/*******************************************************************************
+ *  Class Queue Fields
+ ******************************************************************************/
+struct query_cq_fields_data_s {
+	u32 cqid;
+};
+
+static struct query_cq_fields_data_s query_cq_fields_data = {
+	.cqid = 1,
+};
+
+static int query_cq_fields_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	struct qm_mcr_ceetm_cq_query query_result;
+	unsigned int cqid;
+	unsigned int portal;
+
+	if ((qman_ip_rev & 0xFF00) < QMAN_REV30)
+		return -EINVAL;
+
+	cqid = query_cq_fields_data.cqid & 0x00FFFFFF;
+	portal = query_cq_fields_data.cqid >> 24;
+	if (portal > qm_dc_portal_fman1)
+		return -EINVAL;
+
+	ret = qman_ceetm_query_cq(cqid, portal, &query_result);
+	if (ret)
+		return ret;
+	seq_printf(file, "Query CQ Fields Result cqid 0x%x on DCP %d\n",
+								cqid, portal);
+	seq_printf(file, " ccgid: %u\n", query_result.ccgid);
+	seq_printf(file, " state: %u\n", query_result.state);
+	seq_printf(file, " pfdr_hptr: %u\n", query_result.pfdr_hptr);
+	seq_printf(file, " pfdr_tptr: %u\n", query_result.pfdr_tptr);
+	seq_printf(file, " od1_xsfdr: %u\n", query_result.od1_xsfdr);
+	seq_printf(file, " od2_xsfdr: %u\n", query_result.od2_xsfdr);
+	seq_printf(file, " od3_xsfdr: %u\n", query_result.od3_xsfdr);
+	seq_printf(file, " od4_xsfdr: %u\n", query_result.od4_xsfdr);
+	seq_printf(file, " od5_xsfdr: %u\n", query_result.od5_xsfdr);
+	seq_printf(file, " od6_xsfdr: %u\n", query_result.od6_xsfdr);
+	seq_printf(file, " ra1_xsfdr: %u\n", query_result.ra1_xsfdr);
+	seq_printf(file, " ra2_xsfdr: %u\n", query_result.ra2_xsfdr);
+	seq_printf(file, " frame_count: %u\n", query_result.frm_cnt);
+
+	return 0;
+}
+
+static int query_cq_fields_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, query_cq_fields_show, NULL);
+}
+
+static ssize_t query_cq_fields_write(struct file *f,
+			const char __user *buf, size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	query_cq_fields_data.cqid = (u32)val;
+	return count;
+}
+
+static const struct file_operations query_cq_fields_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_cq_fields_open,
+	.read           = seq_read,
+	.write		= query_cq_fields_write,
+	.release	= single_release,
+};
+
+/*******************************************************************************
+ *  READ CEETM_XSFDR_IN_USE
+ ******************************************************************************/
+struct query_ceetm_xsfdr_data_s {
+	enum qm_dc_portal dcp_portal;
+};
+
+static struct query_ceetm_xsfdr_data_s query_ceetm_xsfdr_data;
+
+static int query_ceetm_xsfdr_show(struct seq_file *file, void *offset)
+{
+	int ret;
+	unsigned int xsfdr_in_use;
+	enum qm_dc_portal portal;
+
+
+	if (qman_ip_rev < QMAN_REV31)
+		return -EINVAL;
+
+	portal = query_ceetm_xsfdr_data.dcp_portal;
+	ret = qman_ceetm_get_xsfdr(portal, &xsfdr_in_use);
+	if (ret) {
+		seq_printf(file, "Read CEETM_XSFDR_IN_USE on DCP %d failed\n",
+								portal);
+		return ret;
+	}
+
+	seq_printf(file, "DCP%d: CEETM_XSFDR_IN_USE number is %u\n", portal,
+						(xsfdr_in_use & 0x1FFF));
+	return 0;
+}
+
+static int query_ceetm_xsfdr_open(struct inode *inode,
+					struct file *file)
+{
+	return single_open(file, query_ceetm_xsfdr_show, NULL);
+}
+
+static ssize_t query_ceetm_xsfdr_write(struct file *f,
+			const char __user *buf, size_t count, loff_t *off)
+{
+	int ret;
+	unsigned long val;
+
+	ret = user_input_convert(buf, count, &val);
+	if (ret)
+		return ret;
+	if (val > qm_dc_portal_fman1)
+		return -EINVAL;
+	query_ceetm_xsfdr_data.dcp_portal = (u32)val;
+	return count;
+}
+
+static const struct file_operations query_ceetm_xsfdr_fops = {
+	.owner          = THIS_MODULE,
+	.open		= query_ceetm_xsfdr_open,
+	.read           = seq_read,
+	.write		= query_ceetm_xsfdr_write,
+	.release	= single_release,
+};
+
+/* helper macros used in qman_debugfs_module_init */
+#define QMAN_DBGFS_ENTRY(name, mode, parent, data, fops) \
+	do { \
+		d = debugfs_create_file(name, \
+			mode, parent, \
+			data, \
+			fops); \
+		if (d == NULL) { \
+			ret = -ENOMEM; \
+			goto _return; \
+		} \
+	} while (0)
+
+/* dfs_root as parent */
+#define QMAN_DBGFS_ENTRY_ROOT(name, mode, data, fops) \
+	QMAN_DBGFS_ENTRY(name, mode, dfs_root, data, fops)
+
+/* fqd_root as parent */
+#define QMAN_DBGFS_ENTRY_FQDROOT(name, mode, data, fops) \
+	QMAN_DBGFS_ENTRY(name, mode, fqd_root, data, fops)
+
+/* fqd state */
+#define QMAN_DBGFS_ENTRY_FQDSTATE(name, index) \
+	QMAN_DBGFS_ENTRY_FQDROOT(name, S_IRUGO, \
+	(void *)&mask_filter[index], &qman_fqd_ctrl_fops)
+
+static int __init qman_debugfs_module_init(void)
+{
+	int ret = 0;
+	struct dentry *d, *fqd_root;
+	u32 reg;
+
+	fqid_max = 0;
+	init_ccsrmempeek();
+	if (qman_ccsr_start) {
+		if (!qman_ccsrmempeek(&reg, QM_FQD_AR)) {
+			/* extract the size of the FQD window */
+			reg = reg & 0x3f;
+			/* calculate valid frame queue descriptor range */
+			fqid_max = (1 << (reg + 1)) / QM_FQD_BLOCK_SIZE;
+		}
+	}
+	dfs_root = debugfs_create_dir("qman", NULL);
+	fqd_root = debugfs_create_dir("fqd", dfs_root);
+	if (dfs_root == NULL || fqd_root == NULL) {
+		ret = -ENOMEM;
+		pr_err("Cannot create qman/fqd debugfs dir\n");
+		goto _return;
+	}
+	if (fqid_max) {
+		QMAN_DBGFS_ENTRY_ROOT("ccsrmempeek", S_IRUGO | S_IWUGO,
+				NULL, &qman_ccsrmempeek_fops);
+	}
+	QMAN_DBGFS_ENTRY_ROOT("query_fq_np_fields", S_IRUGO | S_IWUGO,
+		&query_fq_np_fields_data, &query_fq_np_fields_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("query_fq_fields", S_IRUGO | S_IWUGO,
+		&query_fq_fields_data, &query_fq_fields_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("query_wq_lengths", S_IRUGO | S_IWUGO,
+		&query_wq_lengths_data, &query_wq_lengths_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("query_cgr", S_IRUGO | S_IWUGO,
+		&query_cgr_data, &query_cgr_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("query_congestion", S_IRUGO,
+		NULL, &query_congestion_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr", S_IRUGO,
+		NULL, &testwrite_cgr_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_cgrid", S_IRUGO | S_IWUGO,
+		NULL, &teswrite_cgr_cgrid_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("testwrite_cgr_ibcnt", S_IRUGO | S_IWUGO,
+		NULL, &teswrite_cgr_ibcnt_fops);
+
+	QMAN_DBGFS_ENTRY_ROOT("query_ceetm_ccgr", S_IRUGO | S_IWUGO,
+		&query_ccgr_data, &query_ccgr_fops);
+	/* Create files with fqd_root as parent */
+
+	QMAN_DBGFS_ENTRY_FQDROOT("stateoos", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_OOS], &qman_fqd_state_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("state_retired", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_RETIRED],
+		&qman_fqd_state_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("state_tentatively_sched", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_TEN_SCHED],
+		&qman_fqd_state_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("state_truly_sched", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_TRU_SCHED],
+		&qman_fqd_state_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("state_parked", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_PARKED],
+		&qman_fqd_state_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("state_active", S_IRUGO,
+		(void *)&fqd_states[QM_MCR_NP_STATE_ACTIVE],
+		&qman_fqd_state_fops);
+	QMAN_DBGFS_ENTRY_ROOT("query_cq_fields", S_IRUGO | S_IWUGO,
+		&query_cq_fields_data, &query_cq_fields_fops);
+	QMAN_DBGFS_ENTRY_ROOT("query_ceetm_xsfdr_in_use", S_IRUGO | S_IWUGO,
+		&query_ceetm_xsfdr_data, &query_ceetm_xsfdr_fops);
+
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("cge_enable", 17);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("cge_disable", 16);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("tde_enable", 15);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("tde_disable", 14);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("orp_enable", 13);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("orp_disable", 12);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_enable", 11);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("ctx_a_stashing_disable", 10);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("cpc_enable", 9);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("cpc_disable", 8);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_enable", 7);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("sfdr_disable", 6);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_enable", 5);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("avoid_blocking_disable", 4);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_enable", 3);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("hold_active_disable", 2);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_enable", 1);
+
+	QMAN_DBGFS_ENTRY_FQDSTATE("prefer_in_cache_disable", 0);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("summary", S_IRUGO,
+		NULL, &qman_fqd_summary_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("wq", S_IRUGO | S_IWUGO,
+		NULL, &qman_fqd_dest_wq_fops);
+
+	QMAN_DBGFS_ENTRY_FQDROOT("cred", S_IRUGO,
+		NULL, &qman_fqd_cred_fops);
+
+	return 0;
+
+_return:
+	debugfs_remove_recursive(dfs_root);
+	return ret;
+}
+
+static void __exit qman_debugfs_module_exit(void)
+{
+	debugfs_remove_recursive(dfs_root);
+}
+
+module_init(qman_debugfs_module_init);
+module_exit(qman_debugfs_module_exit);
+MODULE_LICENSE("Dual BSD/GPL");
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_driver.c
@@ -0,0 +1,980 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *	 notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *	 notice, this list of conditions and the following disclaimer in the
+ *	 documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *	 names of its contributors may be used to endorse or promote products
+ *	 derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_private.h"
+
+#include <asm/smp.h>	/* hard_smp_processor_id() if !CONFIG_SMP */
+#ifdef CONFIG_HOTPLUG_CPU
+#include <linux/cpu.h>
+#endif
+
+/* Global variable containing revision id (even on non-control plane systems
+ * where CCSR isn't available) */
+u16 qman_ip_rev;
+EXPORT_SYMBOL(qman_ip_rev);
+u8 qman_ip_cfg;
+EXPORT_SYMBOL(qman_ip_cfg);
+u16 qm_channel_pool1 = QMAN_CHANNEL_POOL1;
+EXPORT_SYMBOL(qm_channel_pool1);
+u16 qm_channel_caam = QMAN_CHANNEL_CAAM;
+EXPORT_SYMBOL(qm_channel_caam);
+u16 qm_channel_pme = QMAN_CHANNEL_PME;
+EXPORT_SYMBOL(qm_channel_pme);
+u16 qm_channel_dce = QMAN_CHANNEL_DCE;
+EXPORT_SYMBOL(qm_channel_dce);
+u16 qman_portal_max;
+EXPORT_SYMBOL(qman_portal_max);
+
+u32 qman_clk;
+struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX];
+/* the qman ceetm instances on the given SoC */
+u8 num_ceetms;
+
+/* For these variables, and the portal-initialisation logic, the
+ * comments in bman_driver.c apply here so won't be repeated. */
+static struct qman_portal *shared_portals[NR_CPUS];
+static int num_shared_portals;
+static int shared_portals_idx;
+static LIST_HEAD(unused_pcfgs);
+static DEFINE_SPINLOCK(unused_pcfgs_lock);
+
+/* A SDQCR mask comprising all the available/visible pool channels */
+static u32 pools_sdqcr;
+
+#define STR_ERR_NOPROP	    "No '%s' property in node %s\n"
+#define STR_ERR_CELL	    "'%s' is not a %d-cell range in node %s\n"
+#define STR_FQID_RANGE	    "fsl,fqid-range"
+#define STR_POOL_CHAN_RANGE "fsl,pool-channel-range"
+#define STR_CGRID_RANGE	     "fsl,cgrid-range"
+
+/* A "fsl,fqid-range" node; release the given range to the allocator */
+static __init int fsl_fqid_range_init(struct device_node *node)
+{
+	int ret;
+	const u32 *range = of_get_property(node, STR_FQID_RANGE, &ret);
+	if (!range) {
+		pr_err(STR_ERR_NOPROP, STR_FQID_RANGE, node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err(STR_ERR_CELL, STR_FQID_RANGE, 2, node->full_name);
+		return -EINVAL;
+	}
+	qman_seed_fqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	pr_info("Qman: FQID allocator includes range %d:%d\n",
+		be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	return 0;
+}
+
+/* A "fsl,pool-channel-range" node; add to the SDQCR mask only */
+static __init int fsl_pool_channel_range_sdqcr(struct device_node *node)
+{
+	int ret;
+	const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret);
+	if (!chanid) {
+		pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name);
+		return -EINVAL;
+	}
+	for (ret = 0; ret < be32_to_cpu(chanid[1]); ret++)
+		pools_sdqcr |= QM_SDQCR_CHANNELS_POOL_CONV(be32_to_cpu(chanid[0]) + ret);
+	return 0;
+}
+
+/* A "fsl,pool-channel-range" node; release the given range to the allocator */
+static __init int fsl_pool_channel_range_init(struct device_node *node)
+{
+	int ret;
+	const u32 *chanid = of_get_property(node, STR_POOL_CHAN_RANGE, &ret);
+	if (!chanid) {
+		pr_err(STR_ERR_NOPROP, STR_POOL_CHAN_RANGE, node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err(STR_ERR_CELL, STR_POOL_CHAN_RANGE, 1, node->full_name);
+		return -EINVAL;
+	}
+	qman_seed_pool_range(be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1]));
+	pr_info("Qman: pool channel allocator includes range %d:%d\n",
+		be32_to_cpu(chanid[0]), be32_to_cpu(chanid[1]));
+	return 0;
+}
+
+/* A "fsl,cgrid-range" node; release the given range to the allocator */
+static __init int fsl_cgrid_range_init(struct device_node *node)
+{
+	struct qman_cgr cgr;
+	int ret, errors = 0;
+	const u32 *range = of_get_property(node, STR_CGRID_RANGE, &ret);
+	if (!range) {
+		pr_err(STR_ERR_NOPROP, STR_CGRID_RANGE, node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err(STR_ERR_CELL, STR_CGRID_RANGE, 2, node->full_name);
+		return -EINVAL;
+	}
+	qman_seed_cgrid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	pr_info("Qman: CGRID allocator includes range %d:%d\n",
+		be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	for (cgr.cgrid = 0; cgr.cgrid < __CGR_NUM; cgr.cgrid++) {
+		ret = qman_modify_cgr(&cgr, QMAN_CGR_FLAG_USE_INIT, NULL);
+		if (ret)
+			errors++;
+	}
+	if (errors)
+		pr_err("Warning: %d error%s while initialising CGRs %d:%d\n",
+			errors, (errors > 1) ? "s" : "", range[0], range[1]);
+	return 0;
+}
+
+static __init int fsl_ceetm_init(struct device_node *node)
+{
+	enum qm_dc_portal dcp_portal;
+	struct qm_ceetm_sp *sp;
+	struct qm_ceetm_lni *lni;
+	int ret, i;
+	const u32 *range;
+
+	/* Find LFQID range */
+	range = of_get_property(node, "fsl,ceetm-lfqid-range", &ret);
+	if (!range) {
+		pr_err("No fsl,ceetm-lfqid-range in node %s\n",
+							node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err("fsl,ceetm-lfqid-range is not a 2-cell range in node"
+					" %s\n", node->full_name);
+		return -EINVAL;
+	}
+
+	dcp_portal = (be32_to_cpu(range[0]) & 0x0F0000) >> 16;
+	if (dcp_portal > qm_dc_portal_fman1) {
+		pr_err("The DCP portal %d doesn't support CEETM\n", dcp_portal);
+		return -EINVAL;
+	}
+
+	if (dcp_portal == qm_dc_portal_fman0)
+		qman_seed_ceetm0_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	if (dcp_portal == qm_dc_portal_fman1)
+		qman_seed_ceetm1_lfqid_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	pr_debug("Qman: The lfqid allocator of CEETM %d includes range"
+		 " 0x%x:0x%x\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+
+	qman_ceetms[dcp_portal].idx = dcp_portal;
+	INIT_LIST_HEAD(&qman_ceetms[dcp_portal].sub_portals);
+	INIT_LIST_HEAD(&qman_ceetms[dcp_portal].lnis);
+
+	/* Find Sub-portal range */
+	range = of_get_property(node, "fsl,ceetm-sp-range", &ret);
+	if (!range) {
+		pr_err("No fsl,ceetm-sp-range in node %s\n", node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err("fsl,ceetm-sp-range is not a 2-cell range in node %s\n",
+							node->full_name);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < be32_to_cpu(range[1]); i++) {
+		sp = kzalloc(sizeof(*sp), GFP_KERNEL);
+		if (!sp) {
+			pr_err("Can't alloc memory for sub-portal %d\n",
+							range[0] + i);
+			return -ENOMEM;
+		}
+		sp->idx = be32_to_cpu(range[0]) + i;
+		sp->dcp_idx = dcp_portal;
+		sp->is_claimed = 0;
+		list_add_tail(&sp->node, &qman_ceetms[dcp_portal].sub_portals);
+		sp++;
+	}
+	pr_debug("Qman: Reserve sub-portal %d:%d for CEETM %d\n",
+		 be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal);
+	qman_ceetms[dcp_portal].sp_range[0] = be32_to_cpu(range[0]);
+	qman_ceetms[dcp_portal].sp_range[1] = be32_to_cpu(range[1]);
+
+	/* Find LNI range */
+	range = of_get_property(node, "fsl,ceetm-lni-range", &ret);
+	if (!range) {
+		pr_err("No fsl,ceetm-lni-range in node %s\n", node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err("fsl,ceetm-lni-range is not a 2-cell range in node %s\n",
+							node->full_name);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < be32_to_cpu(range[1]); i++) {
+		lni = kzalloc(sizeof(*lni), GFP_KERNEL);
+		if (!lni) {
+			pr_err("Can't alloc memory for LNI %d\n",
+							range[0] + i);
+			return -ENOMEM;
+		}
+		lni->idx = be32_to_cpu(range[0]) + i;
+		lni->dcp_idx = dcp_portal;
+		lni->is_claimed = 0;
+		INIT_LIST_HEAD(&lni->channels);
+		list_add_tail(&lni->node, &qman_ceetms[dcp_portal].lnis);
+		lni++;
+	}
+	pr_debug("Qman: Reserve LNI %d:%d for CEETM %d\n",
+		 be32_to_cpu(range[0]), be32_to_cpu(range[1]), dcp_portal);
+	qman_ceetms[dcp_portal].lni_range[0] = be32_to_cpu(range[0]);
+	qman_ceetms[dcp_portal].lni_range[1] = be32_to_cpu(range[1]);
+
+	/* Find CEETM channel range */
+	range = of_get_property(node, "fsl,ceetm-channel-range", &ret);
+	if (!range) {
+		pr_err("No fsl,ceetm-channel-range in node %s\n",
+							node->full_name);
+		return -EINVAL;
+	}
+	if (ret != 8) {
+		pr_err("fsl,ceetm-channel-range is not a 2-cell range in node"
+						"%s\n", node->full_name);
+		return -EINVAL;
+	}
+
+	if (dcp_portal == qm_dc_portal_fman0)
+		qman_seed_ceetm0_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	if (dcp_portal == qm_dc_portal_fman1)
+		qman_seed_ceetm1_channel_range(be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+	pr_debug("Qman: The channel allocator of CEETM %d includes"
+		 " range %d:%d\n", dcp_portal, be32_to_cpu(range[0]), be32_to_cpu(range[1]));
+
+	/* Set CEETM PRES register */
+	ret = qman_ceetm_set_prescaler(dcp_portal);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static void qman_get_ip_revision(struct device_node *dn)
+{
+	u16 ip_rev = 0;
+	u8 ip_cfg = QMAN_REV_CFG_0;
+	for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
+		if (!of_device_is_available(dn))
+			continue;
+		if (of_device_is_compatible(dn, "fsl,qman-portal-1.0") ||
+			of_device_is_compatible(dn, "fsl,qman-portal-1.0.0")) {
+			pr_err("QMAN rev1.0 on P4080 rev1 is not supported!\n");
+			BUG_ON(1);
+		} else if (of_device_is_compatible(dn, "fsl,qman-portal-1.1") ||
+			of_device_is_compatible(dn, "fsl,qman-portal-1.1.0")) {
+			ip_rev = QMAN_REV11;
+			qman_portal_max = 10;
+		} else if (of_device_is_compatible(dn, "fsl,qman-portal-1.2") ||
+			of_device_is_compatible(dn, "fsl,qman-portal-1.2.0")) {
+			ip_rev = QMAN_REV12;
+			qman_portal_max = 10;
+		} else if (of_device_is_compatible(dn, "fsl,qman-portal-2.0") ||
+			of_device_is_compatible(dn, "fsl,qman-portal-2.0.0")) {
+			ip_rev = QMAN_REV20;
+			qman_portal_max = 3;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.0.0")) {
+			ip_rev = QMAN_REV30;
+			qman_portal_max = 50;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.0.1")) {
+			ip_rev = QMAN_REV30;
+			qman_portal_max = 25;
+			ip_cfg = QMAN_REV_CFG_1;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.1.0")) {
+			ip_rev = QMAN_REV31;
+			qman_portal_max = 50;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.1.1")) {
+			ip_rev = QMAN_REV31;
+			qman_portal_max = 25;
+			ip_cfg = QMAN_REV_CFG_1;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.1.2")) {
+			ip_rev = QMAN_REV31;
+			qman_portal_max = 18;
+			ip_cfg = QMAN_REV_CFG_2;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.1.3")) {
+			ip_rev = QMAN_REV31;
+			qman_portal_max = 10;
+			ip_cfg = QMAN_REV_CFG_3;
+		} else if (of_device_is_compatible(dn,
+						"fsl,qman-portal-3.2.0")) {
+			ip_rev = QMAN_REV32;
+			qman_portal_max = 10;
+			ip_cfg = QMAN_REV_CFG_3; // TODO: Verify for ls1043
+		} else {
+			pr_warn("unknown QMan version in portal node,"
+				"default to rev1.1\n");
+			ip_rev = QMAN_REV11;
+			qman_portal_max = 10;
+		}
+
+		if (!qman_ip_rev) {
+			if (ip_rev) {
+				qman_ip_rev = ip_rev;
+				qman_ip_cfg = ip_cfg;
+			} else {
+				pr_warn("unknown Qman version,"
+					" default to rev1.1\n");
+				qman_ip_rev = QMAN_REV11;
+				qman_ip_cfg = QMAN_REV_CFG_0;
+			}
+		} else if (ip_rev && (qman_ip_rev != ip_rev))
+			pr_warn("Revision=0x%04x, but portal '%s' has"
+							" 0x%04x\n",
+			qman_ip_rev, dn->full_name, ip_rev);
+		if (qman_ip_rev == ip_rev)
+			break;
+	}
+}
+
+/* Parse a portal node, perform generic mapping duties and return the config. It
+ * is not known at this stage for what purpose (or even if) the portal will be
+ * used. */
+static struct qm_portal_config * __init parse_pcfg(struct device_node *node)
+{
+	struct qm_portal_config *pcfg;
+	const u32 *index_p, *channel_p;
+	u32 index, channel;
+	int irq, ret;
+	resource_size_t len;
+
+	pcfg = kmalloc(sizeof(*pcfg), GFP_KERNEL);
+	if (!pcfg) {
+		pr_err("can't allocate portal config");
+		return NULL;
+	}
+
+	/*
+	 * This is a *horrible hack*, but the IOMMU/PAMU driver needs a
+	 * 'struct device' in order to get the PAMU stashing setup and the QMan
+	 * portal [driver] won't function at all without ring stashing
+	 *
+	 * Making the QMan portal driver nice and proper is part of the
+	 * upstreaming effort
+	 */
+	pcfg->dev.bus = &platform_bus_type;
+	pcfg->dev.of_node = node;
+#ifdef CONFIG_FSL_PAMU
+	pcfg->dev.archdata.iommu_domain = NULL;
+#endif
+
+	ret = of_address_to_resource(node, DPA_PORTAL_CE,
+				&pcfg->addr_phys[DPA_PORTAL_CE]);
+	if (ret) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"reg::CE");
+		goto err;
+	}
+	ret = of_address_to_resource(node, DPA_PORTAL_CI,
+				&pcfg->addr_phys[DPA_PORTAL_CI]);
+	if (ret) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"reg::CI");
+		goto err;
+	}
+	index_p = of_get_property(node, "cell-index", &ret);
+	if (!index_p || (ret != 4)) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"cell-index");
+		goto err;
+	}
+	index = be32_to_cpu(*index_p);
+	if (index >= qman_portal_max) {
+		pr_err("QMan portal index %d is beyond max (%d)\n",
+		       index, qman_portal_max);
+		goto err;
+	}
+
+	channel_p = of_get_property(node, "fsl,qman-channel-id", &ret);
+	if (!channel_p || (ret != 4)) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"fsl,qman-channel-id");
+		goto err;
+	}
+	channel = be32_to_cpu(*channel_p);
+	if (channel != (index + QM_CHANNEL_SWPORTAL0))
+		pr_err("Warning: node %s has mismatched %s and %s\n",
+			node->full_name, "cell-index", "fsl,qman-channel-id");
+	pcfg->public_cfg.channel = channel;
+	pcfg->public_cfg.cpu = -1;
+	irq = irq_of_parse_and_map(node, 0);
+	if (irq == 0) {
+		pr_err("Can't get %s property '%s'\n", node->full_name,
+			"interrupts");
+		goto err;
+	}
+	pcfg->public_cfg.irq = irq;
+	pcfg->public_cfg.index = index;
+#ifdef CONFIG_FSL_QMAN_CONFIG
+	/* We need the same LIODN offset for all portals */
+	qman_liodn_fixup(pcfg->public_cfg.channel);
+#endif
+
+	len = resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]);
+	if (len != (unsigned long)len)
+		goto err;
+
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_cache_ns(
+                                pcfg->addr_phys[DPA_PORTAL_CE].start,
+                                resource_size(&pcfg->addr_phys[DPA_PORTAL_CE]));
+
+        pcfg->addr_virt[DPA_PORTAL_CI] = ioremap(
+                                pcfg->addr_phys[DPA_PORTAL_CI].start,
+                                resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]));
+#else
+	pcfg->addr_virt[DPA_PORTAL_CE] = ioremap_prot(
+				pcfg->addr_phys[DPA_PORTAL_CE].start,
+				(unsigned long)len,
+				0);
+	pcfg->addr_virt[DPA_PORTAL_CI] = ioremap_prot(
+				pcfg->addr_phys[DPA_PORTAL_CI].start,
+				resource_size(&pcfg->addr_phys[DPA_PORTAL_CI]),
+				_PAGE_GUARDED | _PAGE_NO_CACHE);
+#endif
+	return pcfg;
+err:
+	kfree(pcfg);
+	return NULL;
+}
+
+static struct qm_portal_config *get_pcfg(struct list_head *list)
+{
+	struct qm_portal_config *pcfg;
+	if (list_empty(list))
+		return NULL;
+	pcfg = list_entry(list->prev, struct qm_portal_config, list);
+	list_del(&pcfg->list);
+	return pcfg;
+}
+
+static struct qm_portal_config *get_pcfg_idx(struct list_head *list, u32 idx)
+{
+	struct qm_portal_config *pcfg;
+	if (list_empty(list))
+		return NULL;
+	list_for_each_entry(pcfg, list, list) {
+		if (pcfg->public_cfg.index == idx) {
+			list_del(&pcfg->list);
+			return pcfg;
+		}
+	}
+	return NULL;
+}
+
+static void portal_set_cpu(struct qm_portal_config *pcfg, int cpu)
+{
+#ifdef CONFIG_FSL_PAMU
+	int ret;
+	int window_count = 1;
+	struct iommu_domain_geometry geom_attr;
+	struct pamu_stash_attribute stash_attr;
+
+	pcfg->iommu_domain = iommu_domain_alloc(&platform_bus_type);
+	if (!pcfg->iommu_domain) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_alloc() failed",
+			   __func__);
+		goto _no_iommu;
+	}
+	geom_attr.aperture_start = 0;
+	geom_attr.aperture_end =
+		((dma_addr_t)1 << min(8 * sizeof(dma_addr_t), (size_t)36)) - 1;
+	geom_attr.force_aperture = true;
+	ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_GEOMETRY,
+				    &geom_attr);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_set_attr(pcfg->iommu_domain, DOMAIN_ATTR_WINDOWS,
+				    &window_count);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	stash_attr.cpu = cpu;
+	stash_attr.cache = PAMU_ATTR_CACHE_L1;
+	/* set stash information for the window */
+	stash_attr.window = 0;
+	ret = iommu_domain_set_attr(pcfg->iommu_domain,
+				    DOMAIN_ATTR_FSL_PAMU_STASH,
+				    &stash_attr);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_window_enable(pcfg->iommu_domain, 0, 0, 1ULL << 36,
+					 IOMMU_READ | IOMMU_WRITE);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_window_enable() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_attach_device(pcfg->iommu_domain, &pcfg->dev);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_device_attach() = %d",
+			   __func__, ret);
+		goto _iommu_domain_free;
+	}
+	ret = iommu_domain_set_attr(pcfg->iommu_domain,
+				    DOMAIN_ATTR_FSL_PAMU_ENABLE,
+				    &window_count);
+	if (ret < 0) {
+		pr_err(KBUILD_MODNAME ":%s(): iommu_domain_set_attr() = %d",
+			   __func__, ret);
+		goto _iommu_detach_device;
+	}
+
+_no_iommu:
+#endif
+#ifdef CONFIG_FSL_QMAN_CONFIG
+	if (qman_set_sdest(pcfg->public_cfg.channel, cpu))
+#endif
+		pr_warn("Failed to set QMan portal's stash request queue\n");
+
+	return;
+
+#ifdef CONFIG_FSL_PAMU
+_iommu_detach_device:
+	iommu_detach_device(pcfg->iommu_domain, NULL);
+_iommu_domain_free:
+	iommu_domain_free(pcfg->iommu_domain);
+#endif
+}
+
+struct qm_portal_config *qm_get_unused_portal_idx(u32 idx)
+{
+	struct qm_portal_config *ret;
+	spin_lock(&unused_pcfgs_lock);
+	if (idx == QBMAN_ANY_PORTAL_IDX)
+		ret = get_pcfg(&unused_pcfgs);
+	else
+		ret = get_pcfg_idx(&unused_pcfgs, idx);
+	spin_unlock(&unused_pcfgs_lock);
+	/* Bind stashing LIODNs to the CPU we are currently executing on, and
+	 * set the portal to use the stashing request queue corresonding to the
+	 * cpu as well. The user-space driver assumption is that the pthread has
+	 * to already be affine to one cpu only before opening a portal. If that
+	 * check is circumvented, the only risk is a performance degradation -
+	 * stashing will go to whatever cpu they happened to be running on when
+	 * opening the device file, and if that isn't the cpu they subsequently
+	 * bind to and do their polling on, tough. */
+	if (ret)
+		portal_set_cpu(ret, hard_smp_processor_id());
+	return ret;
+}
+
+struct qm_portal_config *qm_get_unused_portal(void)
+{
+	return qm_get_unused_portal_idx(QBMAN_ANY_PORTAL_IDX);
+}
+
+void qm_put_unused_portal(struct qm_portal_config *pcfg)
+{
+	spin_lock(&unused_pcfgs_lock);
+	list_add(&pcfg->list, &unused_pcfgs);
+	spin_unlock(&unused_pcfgs_lock);
+}
+
+static struct qman_portal *init_pcfg(struct qm_portal_config *pcfg)
+{
+	struct qman_portal *p;
+
+	pcfg->iommu_domain = NULL;
+	portal_set_cpu(pcfg, pcfg->public_cfg.cpu);
+	p = qman_create_affine_portal(pcfg, NULL);
+	if (p) {
+		u32 irq_sources = 0;
+		/* Determine what should be interrupt-vs-poll driven */
+#ifdef CONFIG_FSL_DPA_PIRQ_SLOW
+		irq_sources |= QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI |
+			       QM_PIRQ_CSCI | QM_PIRQ_CCSCI;
+#endif
+#ifdef CONFIG_FSL_DPA_PIRQ_FAST
+		irq_sources |= QM_PIRQ_DQRI;
+#endif
+		qman_p_irqsource_add(p, irq_sources);
+		pr_info("Qman portal %sinitialised, cpu %d\n",
+			pcfg->public_cfg.is_shared ? "(shared) " : "",
+			pcfg->public_cfg.cpu);
+	} else
+		pr_crit("Qman portal failure on cpu %d\n",
+			pcfg->public_cfg.cpu);
+	return p;
+}
+
+static void init_slave(int cpu)
+{
+	struct qman_portal *p;
+	struct cpumask oldmask = *tsk_cpus_allowed(current);
+	set_cpus_allowed_ptr(current, get_cpu_mask(cpu));
+	p = qman_create_affine_slave(shared_portals[shared_portals_idx++], cpu);
+	if (!p)
+		pr_err("Qman slave portal failure on cpu %d\n", cpu);
+	else
+		pr_info("Qman portal %sinitialised, cpu %d\n", "(slave) ", cpu);
+	set_cpus_allowed_ptr(current, &oldmask);
+	if (shared_portals_idx >= num_shared_portals)
+		shared_portals_idx = 0;
+}
+
+static struct cpumask want_unshared __initdata;
+static struct cpumask want_shared __initdata;
+
+static int __init parse_qportals(char *str)
+{
+	return parse_portals_bootarg(str, &want_shared, &want_unshared,
+				     "qportals");
+}
+__setup("qportals=", parse_qportals);
+
+static void qman_portal_update_sdest(const struct qm_portal_config *pcfg,
+							unsigned int cpu)
+{
+#ifdef CONFIG_FSL_PAMU
+	struct pamu_stash_attribute stash_attr;
+	int ret;
+
+	if (pcfg->iommu_domain) {
+		stash_attr.cpu = cpu;
+		stash_attr.cache = PAMU_ATTR_CACHE_L1;
+		/* set stash information for the window */
+		stash_attr.window = 0;
+		ret = iommu_domain_set_attr(pcfg->iommu_domain,
+				DOMAIN_ATTR_FSL_PAMU_STASH, &stash_attr);
+		if (ret < 0) {
+			pr_err("Failed to update pamu stash setting\n");
+			return;
+		}
+	}
+#endif
+#ifdef CONFIG_FSL_QMAN_CONFIG
+	if (qman_set_sdest(pcfg->public_cfg.channel, cpu))
+		pr_warn("Failed to update portal's stash request queue\n");
+#endif
+}
+
+static void qman_offline_cpu(unsigned int cpu)
+{
+	struct qman_portal *p;
+	const struct qm_portal_config *pcfg;
+	p = (struct qman_portal *)affine_portals[cpu];
+	if (p) {
+		pcfg = qman_get_qm_portal_config(p);
+		if (pcfg) {
+			irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(0));
+			qman_portal_update_sdest(pcfg, 0);
+		}
+	}
+}
+
+#ifdef CONFIG_HOTPLUG_CPU
+static void qman_online_cpu(unsigned int cpu)
+{
+	struct qman_portal *p;
+	const struct qm_portal_config *pcfg;
+	p = (struct qman_portal *)affine_portals[cpu];
+	if (p) {
+		pcfg = qman_get_qm_portal_config(p);
+		if (pcfg) {
+			irq_set_affinity(pcfg->public_cfg.irq, cpumask_of(cpu));
+			qman_portal_update_sdest(pcfg, cpu);
+		}
+	}
+}
+
+static int qman_hotplug_cpu_callback(struct notifier_block *nfb,
+				unsigned long action, void *hcpu)
+{
+	unsigned int cpu = (unsigned long)hcpu;
+
+	switch (action) {
+	case CPU_ONLINE:
+	case CPU_ONLINE_FROZEN:
+		qman_online_cpu(cpu);
+		break;
+	case CPU_DOWN_PREPARE:
+	case CPU_DOWN_PREPARE_FROZEN:
+		qman_offline_cpu(cpu);
+	default:
+		break;
+	}
+	return NOTIFY_OK;
+}
+
+static struct notifier_block qman_hotplug_cpu_notifier = {
+	.notifier_call = qman_hotplug_cpu_callback,
+};
+#endif /* CONFIG_HOTPLUG_CPU */
+
+__init int qman_init(void)
+{
+	struct cpumask slave_cpus;
+	struct cpumask unshared_cpus = *cpu_none_mask;
+	struct cpumask shared_cpus = *cpu_none_mask;
+	LIST_HEAD(unshared_pcfgs);
+	LIST_HEAD(shared_pcfgs);
+	struct device_node *dn;
+	struct qm_portal_config *pcfg;
+	struct qman_portal *p;
+	int cpu, ret;
+	const u32 *clk;
+	struct cpumask offline_cpus;
+
+	/* Initialise the Qman (CCSR) device */
+	for_each_compatible_node(dn, NULL, "fsl,qman") {
+		if (!qman_init_ccsr(dn))
+			pr_info("Qman err interrupt handler present\n");
+		else
+			pr_err("Qman CCSR setup failed\n");
+
+		clk = of_get_property(dn, "clock-frequency", NULL);
+		if (!clk)
+			pr_warn("Can't find Qman clock frequency\n");
+		else
+			qman_clk = be32_to_cpu(*clk);
+	}
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+	/* Setup lookup table for FQ demux */
+	ret = qman_setup_fq_lookup_table(get_qman_fqd_size()/64);
+	if (ret)
+		return ret;
+#endif
+
+	/* Get qman ip revision */
+	qman_get_ip_revision(dn);
+	if ((qman_ip_rev & 0xff00) >= QMAN_REV30) {
+		qm_channel_pool1 = QMAN_CHANNEL_POOL1_REV3;
+		qm_channel_caam = QMAN_CHANNEL_CAAM_REV3;
+		qm_channel_pme = QMAN_CHANNEL_PME_REV3;
+	}
+
+	if ((qman_ip_rev == QMAN_REV31) && (qman_ip_cfg == QMAN_REV_CFG_2))
+		qm_channel_dce = QMAN_CHANNEL_DCE_QMANREV312;
+
+	/*
+	 * Parse the ceetm node to get how many ceetm instances are supported
+	 * on the current silicon. num_ceetms must be confirmed before portals
+	 * are intiailized.
+	 */
+	num_ceetms = 0;
+	for_each_compatible_node(dn, NULL, "fsl,qman-ceetm")
+		num_ceetms++;
+
+	/* Parse pool channels into the SDQCR mask. (Must happen before portals
+	 * are initialised.) */
+	for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") {
+		ret = fsl_pool_channel_range_sdqcr(dn);
+		if (ret)
+			return ret;
+	}
+
+	memset(affine_portals, 0, sizeof(void *) * num_possible_cpus());
+	/* Initialise portals. See bman_driver.c for comments */
+	for_each_compatible_node(dn, NULL, "fsl,qman-portal") {
+		if (!of_device_is_available(dn))
+			continue;
+		pcfg = parse_pcfg(dn);
+		if (pcfg) {
+			pcfg->public_cfg.pools = pools_sdqcr;
+			list_add_tail(&pcfg->list, &unused_pcfgs);
+		}
+	}
+	for_each_possible_cpu(cpu) {
+		if (cpumask_test_cpu(cpu, &want_shared)) {
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &shared_pcfgs);
+			cpumask_set_cpu(cpu, &shared_cpus);
+		}
+		if (cpumask_test_cpu(cpu, &want_unshared)) {
+			if (cpumask_test_cpu(cpu, &shared_cpus))
+				continue;
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &unshared_pcfgs);
+			cpumask_set_cpu(cpu, &unshared_cpus);
+		}
+	}
+	if (list_empty(&shared_pcfgs) && list_empty(&unshared_pcfgs)) {
+		for_each_online_cpu(cpu) {
+			pcfg = get_pcfg(&unused_pcfgs);
+			if (!pcfg)
+				break;
+			pcfg->public_cfg.cpu = cpu;
+			list_add_tail(&pcfg->list, &unshared_pcfgs);
+			cpumask_set_cpu(cpu, &unshared_cpus);
+		}
+	}
+	cpumask_andnot(&slave_cpus, cpu_possible_mask, &shared_cpus);
+	cpumask_andnot(&slave_cpus, &slave_cpus, &unshared_cpus);
+	if (cpumask_empty(&slave_cpus)) {
+		if (!list_empty(&shared_pcfgs)) {
+			cpumask_or(&unshared_cpus, &unshared_cpus,
+				   &shared_cpus);
+			cpumask_clear(&shared_cpus);
+			list_splice_tail(&shared_pcfgs, &unshared_pcfgs);
+			INIT_LIST_HEAD(&shared_pcfgs);
+		}
+	} else {
+		if (list_empty(&shared_pcfgs)) {
+			pcfg = get_pcfg(&unshared_pcfgs);
+			if (!pcfg) {
+				pr_crit("No QMan portals available!\n");
+				return 0;
+			}
+			cpumask_clear_cpu(pcfg->public_cfg.cpu, &unshared_cpus);
+			cpumask_set_cpu(pcfg->public_cfg.cpu, &shared_cpus);
+			list_add_tail(&pcfg->list, &shared_pcfgs);
+		}
+	}
+	list_for_each_entry(pcfg, &unshared_pcfgs, list) {
+		pcfg->public_cfg.is_shared = 0;
+		p = init_pcfg(pcfg);
+		if (!p) {
+			pr_crit("Unable to configure portals\n");
+			return 0;
+		}
+	}
+	list_for_each_entry(pcfg, &shared_pcfgs, list) {
+		pcfg->public_cfg.is_shared = 1;
+		p = init_pcfg(pcfg);
+		if (p)
+			shared_portals[num_shared_portals++] = p;
+	}
+	if (!cpumask_empty(&slave_cpus))
+		for_each_cpu(cpu, &slave_cpus)
+			init_slave(cpu);
+	pr_info("Qman portals initialised\n");
+	cpumask_andnot(&offline_cpus, cpu_possible_mask, cpu_online_mask);
+	for_each_cpu(cpu, &offline_cpus)
+		qman_offline_cpu(cpu);
+#ifdef CONFIG_HOTPLUG_CPU
+	register_hotcpu_notifier(&qman_hotplug_cpu_notifier);
+#endif
+	return 0;
+}
+
+__init int qman_resource_init(void)
+{
+	struct device_node *dn;
+	int ret;
+
+	/* Initialise FQID allocation ranges */
+	for_each_compatible_node(dn, NULL, "fsl,fqid-range") {
+		ret = fsl_fqid_range_init(dn);
+		if (ret)
+			return ret;
+	}
+	/* Initialise CGRID allocation ranges */
+	for_each_compatible_node(dn, NULL, "fsl,cgrid-range") {
+		ret = fsl_cgrid_range_init(dn);
+		if (ret)
+			return ret;
+	}
+	/* Parse pool channels into the allocator. (Must happen after portals
+	 * are initialised.) */
+	for_each_compatible_node(dn, NULL, "fsl,pool-channel-range") {
+		ret = fsl_pool_channel_range_init(dn);
+		if (ret)
+			return ret;
+	}
+
+	/* Parse CEETM */
+	for_each_compatible_node(dn, NULL, "fsl,qman-ceetm") {
+		ret = fsl_ceetm_init(dn);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+#ifdef CONFIG_SUSPEND
+void suspend_unused_qportal(void)
+{
+	struct qm_portal_config *pcfg;
+
+	if (list_empty(&unused_pcfgs))
+		return;
+
+	list_for_each_entry(pcfg, &unused_pcfgs, list) {
+#ifdef CONFIG_PM_DEBUG
+		pr_info("Need to save qportal %d\n", pcfg->public_cfg.index);
+#endif
+		/* save isdr, disable all via isdr, clear isr */
+		pcfg->saved_isdr =
+			__raw_readl(pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
+		__raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
+					0xe08);
+		__raw_writel(0xffffffff, pcfg->addr_virt[DPA_PORTAL_CI] +
+					0xe00);
+	}
+	return;
+}
+
+void resume_unused_qportal(void)
+{
+	struct qm_portal_config *pcfg;
+
+	if (list_empty(&unused_pcfgs))
+		return;
+
+	list_for_each_entry(pcfg, &unused_pcfgs, list) {
+#ifdef CONFIG_PM_DEBUG
+		pr_info("Need to resume qportal %d\n", pcfg->public_cfg.index);
+#endif
+		/* restore isdr */
+		__raw_writel(pcfg->saved_isdr,
+				pcfg->addr_virt[DPA_PORTAL_CI] + 0xe08);
+	}
+	return;
+}
+#endif
+
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_high.c
@@ -0,0 +1,5568 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_low.h"
+
+/* Compilation constants */
+#define DQRR_MAXFILL	15
+#define EQCR_ITHRESH	4	/* if EQCR congests, interrupt threshold */
+#define IRQNAME		"QMan portal %d"
+#define MAX_IRQNAME	16	/* big enough for "QMan portal %d" */
+
+/* Divide 'n' by 'd', rounding down if 'r' is negative, rounding up if it's
+ * positive, and rounding to the closest value if it's zero. NB, this macro
+ * implicitly upgrades parameters to unsigned 64-bit, so feed it with types
+ * that are compatible with this. NB, these arguments should not be expressions
+ * unless it is safe for them to be evaluated multiple times. Eg. do not pass
+ * in "some_value++" as a parameter to the macro! */
+#define ROUNDING(n, d, r) \
+	(((r) < 0) ? div64_u64((n), (d)) : \
+	(((r) > 0) ? div64_u64(((n) + (d) - 1), (d)) : \
+	div64_u64(((n) + ((d) / 2)), (d))))
+
+/* Lock/unlock frame queues, subject to the "LOCKED" flag. This is about
+ * inter-processor locking only. Note, FQLOCK() is always called either under a
+ * local_irq_save() or from interrupt context - hence there's no need for irq
+ * protection (and indeed, attempting to nest irq-protection doesn't work, as
+ * the "irq en/disable" machinery isn't recursive...). */
+#define FQLOCK(fq) \
+	do { \
+		struct qman_fq *__fq478 = (fq); \
+		if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
+			spin_lock(&__fq478->fqlock); \
+	} while (0)
+#define FQUNLOCK(fq) \
+	do { \
+		struct qman_fq *__fq478 = (fq); \
+		if (fq_isset(__fq478, QMAN_FQ_FLAG_LOCKED)) \
+			spin_unlock(&__fq478->fqlock); \
+	} while (0)
+
+static inline void fq_set(struct qman_fq *fq, u32 mask)
+{
+	set_bits(mask, &fq->flags);
+}
+static inline void fq_clear(struct qman_fq *fq, u32 mask)
+{
+	clear_bits(mask, &fq->flags);
+}
+static inline int fq_isset(struct qman_fq *fq, u32 mask)
+{
+	return fq->flags & mask;
+}
+static inline int fq_isclear(struct qman_fq *fq, u32 mask)
+{
+	return !(fq->flags & mask);
+}
+
+struct qman_portal {
+	struct qm_portal p;
+	unsigned long bits; /* PORTAL_BITS_*** - dynamic, strictly internal */
+	unsigned long irq_sources;
+	u32 use_eqcr_ci_stashing;
+	u32 slowpoll;	/* only used when interrupts are off */
+	struct qman_fq *vdqcr_owned; /* only 1 volatile dequeue at a time */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	struct qman_fq *eqci_owned; /* only 1 enqueue WAIT_SYNC at a time */
+#endif
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	raw_spinlock_t sharing_lock; /* only used if is_shared */
+	int is_shared;
+	struct qman_portal *sharing_redirect;
+#endif
+	u32 sdqcr;
+	int dqrr_disable_ref;
+	/* A portal-specific handler for DCP ERNs. If this is NULL, the global
+	 * handler is called instead. */
+	qman_cb_dc_ern cb_dc_ern;
+	/* When the cpu-affine portal is activated, this is non-NULL */
+	const struct qm_portal_config *config;
+	/* This is needed for providing a non-NULL device to dma_map_***() */
+	struct platform_device *pdev;
+	struct dpa_rbtree retire_table;
+	char irqname[MAX_IRQNAME];
+	/* 2-element array. cgrs[0] is mask, cgrs[1] is snapshot. */
+	struct qman_cgrs *cgrs;
+	/* linked-list of CSCN handlers. */
+	struct list_head cgr_cbs;
+	/* list lock */
+	spinlock_t cgr_lock;
+	/* 2-element array. ccgrs[0] is mask, ccgrs[1] is snapshot. */
+	struct qman_ccgrs *ccgrs[QMAN_CEETM_MAX];
+	/* 256-element array, each is a linked-list of CCSCN handlers. */
+	struct list_head ccgr_cbs[QMAN_CEETM_MAX];
+	/* list lock */
+	spinlock_t ccgr_lock;
+	/* track if memory was allocated by the driver */
+	u8 alloced;
+	/* power management data */
+	u32 save_isdr;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	/* Keep a shadow copy of the DQRR on LE systems
+	   as the SW needs to do byteswaps of read only
+	   memory.  Must be aligned to the size of the
+	   ring to ensure easy index calcualtions based
+	   on address */
+	struct qm_dqrr_entry shadow_dqrr[QM_DQRR_SIZE]
+	            __attribute__((aligned(512)));
+#endif
+};
+
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+#define PORTAL_IRQ_LOCK(p, irqflags) \
+	do { \
+		if ((p)->is_shared) \
+			raw_spin_lock_irqsave(&(p)->sharing_lock, irqflags); \
+		else \
+			local_irq_save(irqflags); \
+	} while (0)
+#define PORTAL_IRQ_UNLOCK(p, irqflags) \
+	do { \
+		if ((p)->is_shared) \
+			raw_spin_unlock_irqrestore(&(p)->sharing_lock, \
+						   irqflags); \
+		else \
+			local_irq_restore(irqflags); \
+	} while (0)
+#else
+#define PORTAL_IRQ_LOCK(p, irqflags) local_irq_save(irqflags)
+#define PORTAL_IRQ_UNLOCK(p, irqflags) local_irq_restore(irqflags)
+#endif
+
+/* Global handler for DCP ERNs. Used when the portal receiving the message does
+ * not have a portal-specific handler. */
+static qman_cb_dc_ern cb_dc_ern;
+
+static cpumask_t affine_mask;
+static DEFINE_SPINLOCK(affine_mask_lock);
+static u16 affine_channels[NR_CPUS];
+static DEFINE_PER_CPU(struct qman_portal, qman_affine_portal);
+void *affine_portals[NR_CPUS];
+
+/* "raw" gets the cpu-local struct whether it's a redirect or not. */
+static inline struct qman_portal *get_raw_affine_portal(void)
+{
+	return &get_cpu_var(qman_affine_portal);
+}
+/* For ops that can redirect, this obtains the portal to use */
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+static inline struct qman_portal *get_affine_portal(void)
+{
+	struct qman_portal *p = get_raw_affine_portal();
+	if (p->sharing_redirect)
+		return p->sharing_redirect;
+	return p;
+}
+#else
+#define get_affine_portal() get_raw_affine_portal()
+#endif
+/* For every "get", there must be a "put" */
+static inline void put_affine_portal(void)
+{
+	put_cpu_var(qman_affine_portal);
+}
+/* Exception: poll functions assume the caller is cpu-affine and in no risk of
+ * re-entrance, which are the two reasons we usually use the get/put_cpu_var()
+ * semantic - ie. to disable pre-emption. Some use-cases expect the execution
+ * context to remain as non-atomic during poll-triggered callbacks as it was
+ * when the poll API was first called (eg. NAPI), so we go out of our way in
+ * this case to not disable pre-emption. */
+static inline struct qman_portal *get_poll_portal(void)
+{
+	return &get_cpu_var(qman_affine_portal);
+}
+#define put_poll_portal()
+
+/* This gives a FQID->FQ lookup to cover the fact that we can't directly demux
+ * retirement notifications (the fact they are sometimes h/w-consumed means that
+ * contextB isn't always a s/w demux - and as we can't know which case it is
+ * when looking at the notification, we have to use the slow lookup for all of
+ * them). NB, it's possible to have multiple FQ objects refer to the same FQID
+ * (though at most one of them should be the consumer), so this table isn't for
+ * all FQs - FQs are added when retirement commands are issued, and removed when
+ * they complete, which also massively reduces the size of this table. */
+IMPLEMENT_DPA_RBTREE(fqtree, struct qman_fq, node, fqid);
+
+/* This is what everything can wait on, even if it migrates to a different cpu
+ * to the one whose affine portal it is waiting on. */
+static DECLARE_WAIT_QUEUE_HEAD(affine_queue);
+
+static inline int table_push_fq(struct qman_portal *p, struct qman_fq *fq)
+{
+	int ret = fqtree_push(&p->retire_table, fq);
+	if (ret)
+		pr_err("ERROR: double FQ-retirement %d\n", fq->fqid);
+	return ret;
+}
+
+static inline void table_del_fq(struct qman_portal *p, struct qman_fq *fq)
+{
+	fqtree_del(&p->retire_table, fq);
+}
+
+static inline struct qman_fq *table_find_fq(struct qman_portal *p, u32 fqid)
+{
+	return fqtree_find(&p->retire_table, fqid);
+}
+
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+static void **qman_fq_lookup_table;
+static size_t qman_fq_lookup_table_size;
+
+int qman_setup_fq_lookup_table(size_t num_entries)
+{
+	num_entries++;
+	/* Allocate 1 more entry since the first entry is not used */
+	qman_fq_lookup_table = vzalloc((num_entries * sizeof(void *)));
+	if (!qman_fq_lookup_table) {
+		pr_err("QMan: Could not allocate fq lookup table\n");
+		return -ENOMEM;
+	}
+	qman_fq_lookup_table_size = num_entries;
+	pr_info("QMan: Allocated lookup table at %p, entry count %lu\n",
+			qman_fq_lookup_table,
+			(unsigned long)qman_fq_lookup_table_size);
+	return 0;
+}
+
+/* global structure that maintains fq object mapping */
+static DEFINE_SPINLOCK(fq_hash_table_lock);
+
+static int find_empty_fq_table_entry(u32 *entry, struct qman_fq *fq)
+{
+	u32 i;
+
+	spin_lock(&fq_hash_table_lock);
+	/* Can't use index zero because this has special meaning
+	 * in context_b field. */
+	for (i = 1; i < qman_fq_lookup_table_size; i++) {
+		if (qman_fq_lookup_table[i] == NULL) {
+			*entry = i;
+			qman_fq_lookup_table[i] = fq;
+			spin_unlock(&fq_hash_table_lock);
+			return 0;
+		}
+	}
+	spin_unlock(&fq_hash_table_lock);
+	return -ENOMEM;
+}
+
+static void clear_fq_table_entry(u32 entry)
+{
+	spin_lock(&fq_hash_table_lock);
+	BUG_ON(entry >= qman_fq_lookup_table_size);
+	qman_fq_lookup_table[entry] = NULL;
+	spin_unlock(&fq_hash_table_lock);
+}
+
+static inline struct qman_fq *get_fq_table_entry(u32 entry)
+{
+	BUG_ON(entry >= qman_fq_lookup_table_size);
+	return qman_fq_lookup_table[entry];
+}
+#endif
+
+static inline void cpu_to_hw_fqd(struct qm_fqd *fqd)
+{
+	/* Byteswap the FQD to HW format */
+	fqd->fq_ctrl = cpu_to_be16(fqd->fq_ctrl);
+	fqd->dest_wq = cpu_to_be16(fqd->dest_wq);
+	fqd->ics_cred = cpu_to_be16(fqd->ics_cred);
+	fqd->context_b = cpu_to_be32(fqd->context_b);
+	fqd->context_a.opaque = cpu_to_be64(fqd->context_a.opaque);
+}
+
+static inline void hw_fqd_to_cpu(struct qm_fqd *fqd)
+{
+	/* Byteswap the FQD to CPU format */
+	fqd->fq_ctrl = be16_to_cpu(fqd->fq_ctrl);
+	fqd->dest_wq = be16_to_cpu(fqd->dest_wq);
+	fqd->ics_cred = be16_to_cpu(fqd->ics_cred);
+	fqd->context_b = be32_to_cpu(fqd->context_b);
+	fqd->context_a.opaque = be64_to_cpu(fqd->context_a.opaque);
+}
+
+/* Swap a 40 bit address */
+static inline u64 cpu_to_be40(u64 in)
+{
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	return in;
+#else
+	u64 out = 0;
+	u8 *p = (u8 *) &out;
+	p[0] = in >> 32;
+	p[1] = in >> 24;
+	p[2] = in >> 16;
+	p[3] = in >> 8;
+	p[4] = in >> 0;
+	return out;
+#endif
+}
+static inline u64 be40_to_cpu(u64 in)
+{
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	return in;
+#else
+	u64 out = 0;
+	u8 *pout = (u8 *) &out;
+	u8 *pin = (u8 *) &in;
+	pout[0] = pin[4];
+	pout[1] = pin[3];
+	pout[2] = pin[2];
+	pout[3] = pin[1];
+	pout[4] = pin[0];
+	return out;
+#endif
+}
+
+/* Swap a 24 bit value */
+static inline u32 cpu_to_be24(u32 in)
+{
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	return in;
+#else
+	u32 out = 0;
+	u8 *p = (u8 *) &out;
+	p[0] = in >> 16;
+	p[1] = in >> 8;
+	p[2] = in >> 0;
+	return out;
+#endif
+}
+
+static inline u32 be24_to_cpu(u32 in)
+{
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	return in;
+#else
+	u32 out = 0;
+	u8 *pout = (u8 *) &out;
+	u8 *pin = (u8 *) &in;
+	pout[0] = pin[2];
+	pout[1] = pin[1];
+	pout[2] = pin[0];
+	return out;
+#endif
+}
+
+static inline u64 be48_to_cpu(u64 in)
+{
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	return in;
+#else
+	u64 out = 0;
+	u8 *pout = (u8 *) &out;
+	u8 *pin = (u8 *) &in;
+
+	pout[0] = pin[5];
+	pout[1] = pin[4];
+	pout[2] = pin[3];
+	pout[3] = pin[2];
+	pout[4] = pin[1];
+	pout[5] = pin[0];
+	return out;
+#endif
+}
+static inline void cpu_to_hw_fd(struct qm_fd *fd)
+{
+	fd->addr = cpu_to_be40(fd->addr);
+	fd->status = cpu_to_be32(fd->status);
+	fd->opaque = cpu_to_be32(fd->opaque);
+}
+
+static inline void hw_fd_to_cpu(struct qm_fd *fd)
+{
+	fd->addr = be40_to_cpu(fd->addr);
+	fd->status = be32_to_cpu(fd->status);
+	fd->opaque = be32_to_cpu(fd->opaque);
+}
+
+static inline void hw_cq_query_to_cpu(struct qm_mcr_ceetm_cq_query *cq_query)
+{
+	cq_query->ccgid = be16_to_cpu(cq_query->ccgid);
+	cq_query->state = be16_to_cpu(cq_query->state);
+	cq_query->pfdr_hptr = be24_to_cpu(cq_query->pfdr_hptr);
+	cq_query->pfdr_tptr = be24_to_cpu(cq_query->pfdr_tptr);
+	cq_query->od1_xsfdr = be16_to_cpu(cq_query->od1_xsfdr);
+	cq_query->od2_xsfdr = be16_to_cpu(cq_query->od2_xsfdr);
+	cq_query->od3_xsfdr = be16_to_cpu(cq_query->od3_xsfdr);
+	cq_query->od4_xsfdr = be16_to_cpu(cq_query->od4_xsfdr);
+	cq_query->od5_xsfdr = be16_to_cpu(cq_query->od5_xsfdr);
+	cq_query->od6_xsfdr = be16_to_cpu(cq_query->od6_xsfdr);
+	cq_query->ra1_xsfdr = be16_to_cpu(cq_query->ra1_xsfdr);
+	cq_query->ra2_xsfdr = be16_to_cpu(cq_query->ra2_xsfdr);
+	cq_query->frm_cnt = be24_to_cpu(cq_query->frm_cnt);
+}
+
+static inline void hw_ccgr_query_to_cpu(struct qm_mcr_ceetm_ccgr_query *ccgr_q)
+{
+	int i;
+
+	ccgr_q->cm_query.cscn_targ_dcp =
+		be16_to_cpu(ccgr_q->cm_query.cscn_targ_dcp);
+	ccgr_q->cm_query.i_cnt = be40_to_cpu(ccgr_q->cm_query.i_cnt);
+	ccgr_q->cm_query.a_cnt = be40_to_cpu(ccgr_q->cm_query.a_cnt);
+	for (i = 0; i < ARRAY_SIZE(ccgr_q->cm_query.cscn_targ_swp); i++)
+		ccgr_q->cm_query.cscn_targ_swp[i] =
+			be32_to_cpu(ccgr_q->cm_query.cscn_targ_swp[i]);
+}
+
+/* In the case that slow- and fast-path handling are both done by qman_poll()
+ * (ie. because there is no interrupt handling), we ought to balance how often
+ * we do the fast-path poll versus the slow-path poll. We'll use two decrementer
+ * sources, so we call the fast poll 'n' times before calling the slow poll
+ * once. The idle decrementer constant is used when the last slow-poll detected
+ * no work to do, and the busy decrementer constant when the last slow-poll had
+ * work to do. */
+#define SLOW_POLL_IDLE   1000
+#define SLOW_POLL_BUSY   10
+static u32 __poll_portal_slow(struct qman_portal *p, u32 is);
+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
+					unsigned int poll_limit);
+
+/* Portal interrupt handler */
+static irqreturn_t portal_isr(__always_unused int irq, void *ptr)
+{
+	struct qman_portal *p = ptr;
+	/*
+	 * The CSCI/CCSCI source is cleared inside __poll_portal_slow(), because
+	 * it could race against a Query Congestion State command also given
+	 * as part of the handling of this interrupt source. We mustn't
+	 * clear it a second time in this top-level function.
+	 */
+	u32 clear = QM_DQAVAIL_MASK | (p->irq_sources &
+		~(QM_PIRQ_CSCI | QM_PIRQ_CCSCI));
+	u32 is = qm_isr_status_read(&p->p) & p->irq_sources;
+	/* DQRR-handling if it's interrupt-driven */
+	if (is & QM_PIRQ_DQRI)
+		__poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
+	/* Handling of anything else that's interrupt-driven */
+	clear |= __poll_portal_slow(p, is);
+	qm_isr_status_clear(&p->p, clear);
+	return IRQ_HANDLED;
+}
+
+/* This inner version is used privately by qman_create_affine_portal(), as well
+ * as by the exported qman_stop_dequeues(). */
+static inline void qman_stop_dequeues_ex(struct qman_portal *p)
+{
+	unsigned long irqflags __maybe_unused;
+	PORTAL_IRQ_LOCK(p, irqflags);
+	if (!(p->dqrr_disable_ref++))
+		qm_dqrr_set_maxfill(&p->p, 0);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+}
+
+static int drain_mr(struct qm_portal *p)
+{
+	const struct qm_mr_entry *msg;
+loop:
+	msg = qm_mr_current(p);
+	if (!msg) {
+		/* if MR was full and h/w had other FQRNI entries to produce, we
+		 * need to allow it time to produce those entries once the
+		 * existing entries are consumed. A worst-case situation
+		 * (fully-loaded system) means h/w sequencers may have to do 3-4
+		 * other things before servicing the portal's MR pump, each of
+		 * which (if slow) may take ~50 qman cycles (which is ~200
+		 * processor cycles). So rounding up and then multiplying this
+		 * worst-case estimate by a factor of 10, just to be
+		 * ultra-paranoid, goes as high as 10,000 cycles. NB, we consume
+		 * one entry at a time, so h/w has an opportunity to produce new
+		 * entries well before the ring has been fully consumed, so
+		 * we're being *really* paranoid here. */
+		u64 now, then = mfatb();
+		do {
+			now = mfatb();
+		} while ((then + 10000) > now);
+		msg = qm_mr_current(p);
+		if (!msg)
+			return 0;
+	}
+	qm_mr_next(p);
+	qm_mr_cci_consume(p, 1);
+	goto loop;
+}
+
+#ifdef CONFIG_SUSPEND
+static int _qman_portal_suspend_noirq(struct device *dev)
+{
+	struct qman_portal *p = (struct qman_portal *)dev->platform_data;
+#ifdef CONFIG_PM_DEBUG
+	struct platform_device *pdev = to_platform_device(dev);
+#endif
+
+	p->save_isdr = qm_isr_disable_read(&p->p);
+	qm_isr_disable_write(&p->p, 0xffffffff);
+	qm_isr_status_clear(&p->p, 0xffffffff);
+#ifdef CONFIG_PM_DEBUG
+	pr_info("Suspend for %s\n", pdev->name);
+#endif
+	return 0;
+}
+
+static int _qman_portal_resume_noirq(struct device *dev)
+{
+	struct qman_portal *p = (struct qman_portal *)dev->platform_data;
+
+	/* restore isdr */
+	qm_isr_disable_write(&p->p, p->save_isdr);
+	return 0;
+}
+#else
+#define _qman_portal_suspend_noirq NULL
+#define _qman_portal_resume_noirq NULL
+#endif
+
+struct dev_pm_domain qman_portal_device_pm_domain = {
+	.ops = {
+		USE_PLATFORM_PM_SLEEP_OPS
+		.suspend_noirq = _qman_portal_suspend_noirq,
+		.resume_noirq = _qman_portal_resume_noirq,
+	}
+};
+
+struct qman_portal *qman_create_portal(
+			struct qman_portal *portal,
+			const struct qm_portal_config *config,
+			const struct qman_cgrs *cgrs)
+{
+	struct qm_portal *__p;
+	char buf[16];
+	int ret;
+	u32 isdr;
+
+	if (!portal) {
+		portal = kmalloc(sizeof(*portal), GFP_KERNEL);
+		if (!portal)
+			return portal;
+		portal->alloced = 1;
+	} else
+		portal->alloced = 0;
+
+	__p = &portal->p;
+
+#ifdef CONFIG_FSL_PAMU
+        /* PAMU is required for stashing */
+        portal->use_eqcr_ci_stashing = ((qman_ip_rev >= QMAN_REV30) ?
+                                                                1 : 0);
+#else
+        portal->use_eqcr_ci_stashing = 0;
+#endif
+
+	/* prep the low-level portal struct with the mapped addresses from the
+	 * config, everything that follows depends on it and "config" is more
+	 * for (de)reference... */
+	__p->addr.addr_ce = config->addr_virt[DPA_PORTAL_CE];
+	__p->addr.addr_ci = config->addr_virt[DPA_PORTAL_CI];
+	/*
+	 * If CI-stashing is used, the current defaults use a threshold of 3,
+	 * and stash with high-than-DQRR priority.
+	 */
+	if (qm_eqcr_init(__p, qm_eqcr_pvb,
+			portal->use_eqcr_ci_stashing ? 3 : 0, 1)) {
+		pr_err("Qman EQCR initialisation failed\n");
+		goto fail_eqcr;
+	}
+	if (qm_dqrr_init(__p, config, qm_dqrr_dpush, qm_dqrr_pvb,
+			qm_dqrr_cdc, DQRR_MAXFILL)) {
+		pr_err("Qman DQRR initialisation failed\n");
+		goto fail_dqrr;
+	}
+	if (qm_mr_init(__p, qm_mr_pvb, qm_mr_cci)) {
+		pr_err("Qman MR initialisation failed\n");
+		goto fail_mr;
+	}
+	if (qm_mc_init(__p)) {
+		pr_err("Qman MC initialisation failed\n");
+		goto fail_mc;
+	}
+	if (qm_isr_init(__p)) {
+		pr_err("Qman ISR initialisation failed\n");
+		goto fail_isr;
+	}
+	/* static interrupt-gating controls */
+	qm_dqrr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_DQRR_ITHRESH);
+	qm_mr_set_ithresh(__p, CONFIG_FSL_QMAN_PIRQ_MR_ITHRESH);
+	qm_isr_set_iperiod(__p, CONFIG_FSL_QMAN_PIRQ_IPERIOD);
+	portal->cgrs = kmalloc(2 * sizeof(*cgrs), GFP_KERNEL);
+	if (!portal->cgrs)
+		goto fail_cgrs;
+	/* initial snapshot is no-depletion */
+	qman_cgrs_init(&portal->cgrs[1]);
+	if (cgrs)
+		portal->cgrs[0] = *cgrs;
+	else
+		/* if the given mask is NULL, assume all CGRs can be seen */
+		qman_cgrs_fill(&portal->cgrs[0]);
+	INIT_LIST_HEAD(&portal->cgr_cbs);
+	spin_lock_init(&portal->cgr_lock);
+	if (num_ceetms) {
+		for (ret = 0; ret < num_ceetms; ret++) {
+			portal->ccgrs[ret] = kmalloc(2 *
+				sizeof(struct qman_ccgrs), GFP_KERNEL);
+			if (!portal->ccgrs[ret])
+				goto fail_ccgrs;
+			qman_ccgrs_init(&portal->ccgrs[ret][1]);
+			qman_ccgrs_fill(&portal->ccgrs[ret][0]);
+			INIT_LIST_HEAD(&portal->ccgr_cbs[ret]);
+		}
+	}
+	spin_lock_init(&portal->ccgr_lock);
+	portal->bits = 0;
+	portal->slowpoll = 0;
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	portal->eqci_owned = NULL;
+#endif
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	raw_spin_lock_init(&portal->sharing_lock);
+	portal->is_shared = config->public_cfg.is_shared;
+	portal->sharing_redirect = NULL;
+#endif
+	portal->sdqcr = QM_SDQCR_SOURCE_CHANNELS | QM_SDQCR_COUNT_UPTO3 |
+			QM_SDQCR_DEDICATED_PRECEDENCE | QM_SDQCR_TYPE_PRIO_QOS |
+			QM_SDQCR_TOKEN_SET(0xab) | QM_SDQCR_CHANNELS_DEDICATED;
+	portal->dqrr_disable_ref = 0;
+	portal->cb_dc_ern = NULL;
+	sprintf(buf, "qportal-%d", config->public_cfg.channel);
+	portal->pdev = platform_device_alloc(buf, -1);
+	if (!portal->pdev) {
+		pr_err("qman_portal - platform_device_alloc() failed\n");
+		goto fail_devalloc;
+	}
+#ifdef CONFIG_ARM
+	portal->pdev->dev.coherent_dma_mask = DMA_BIT_MASK(40);
+	portal->pdev->dev.dma_mask = &portal->pdev->dev.coherent_dma_mask;
+#else
+	if (dma_set_mask(&portal->pdev->dev, DMA_BIT_MASK(40))) {
+		pr_err("qman_portal - dma_set_mask() failed\n");
+		goto fail_devadd;
+	}
+#endif
+	portal->pdev->dev.pm_domain = &qman_portal_device_pm_domain;
+	portal->pdev->dev.platform_data = portal;
+	ret = platform_device_add(portal->pdev);
+	if (ret) {
+		pr_err("qman_portal - platform_device_add() failed\n");
+		goto fail_devadd;
+	}
+	dpa_rbtree_init(&portal->retire_table);
+	isdr = 0xffffffff;
+	qm_isr_disable_write(__p, isdr);
+	portal->irq_sources = 0;
+	qm_isr_enable_write(__p, portal->irq_sources);
+	qm_isr_status_clear(__p, 0xffffffff);
+	snprintf(portal->irqname, MAX_IRQNAME, IRQNAME, config->public_cfg.cpu);
+	if (request_irq(config->public_cfg.irq, portal_isr, 0, portal->irqname,
+				portal)) {
+		pr_err("request_irq() failed\n");
+		goto fail_irq;
+	}
+	if ((config->public_cfg.cpu != -1) &&
+			irq_can_set_affinity(config->public_cfg.irq) &&
+			irq_set_affinity(config->public_cfg.irq,
+				cpumask_of(config->public_cfg.cpu))) {
+		pr_err("irq_set_affinity() failed\n");
+		goto fail_affinity;
+	}
+
+	/* Need EQCR to be empty before continuing */
+	isdr ^= QM_PIRQ_EQCI;
+	qm_isr_disable_write(__p, isdr);
+	ret = qm_eqcr_get_fill(__p);
+	if (ret) {
+		pr_err("Qman EQCR unclean\n");
+		goto fail_eqcr_empty;
+	}
+	isdr ^= (QM_PIRQ_DQRI | QM_PIRQ_MRI);
+	qm_isr_disable_write(__p, isdr);
+	while (qm_dqrr_current(__p) != NULL)
+		qm_dqrr_cdc_consume_n(__p, 0xffff);
+	drain_mr(__p);
+	/* Success */
+	portal->config = config;
+	qm_isr_disable_write(__p, 0);
+	qm_isr_uninhibit(__p);
+	/* Write a sane SDQCR */
+	qm_dqrr_sdqcr_set(__p, portal->sdqcr);
+	return portal;
+fail_eqcr_empty:
+fail_affinity:
+	free_irq(config->public_cfg.irq, portal);
+fail_irq:
+	platform_device_del(portal->pdev);
+fail_devadd:
+	platform_device_put(portal->pdev);
+fail_devalloc:
+	if (num_ceetms)
+		for (ret = 0; ret < num_ceetms; ret++)
+			kfree(portal->ccgrs[ret]);
+fail_ccgrs:
+	kfree(portal->cgrs);
+fail_cgrs:
+	qm_isr_finish(__p);
+fail_isr:
+	qm_mc_finish(__p);
+fail_mc:
+	qm_mr_finish(__p);
+fail_mr:
+	qm_dqrr_finish(__p);
+fail_dqrr:
+	qm_eqcr_finish(__p);
+fail_eqcr:
+	if (portal->alloced)
+		kfree(portal);
+	return NULL;
+}
+
+struct qman_portal *qman_create_affine_portal(
+			const struct qm_portal_config *config,
+			const struct qman_cgrs *cgrs)
+{
+	struct qman_portal *res;
+	struct qman_portal *portal;
+
+	portal = &per_cpu(qman_affine_portal, config->public_cfg.cpu);
+	res = qman_create_portal(portal, config, cgrs);
+	if (res) {
+		spin_lock(&affine_mask_lock);
+		cpumask_set_cpu(config->public_cfg.cpu, &affine_mask);
+		affine_channels[config->public_cfg.cpu] =
+			config->public_cfg.channel;
+		affine_portals[config->public_cfg.cpu] = portal;
+		spin_unlock(&affine_mask_lock);
+	}
+	return res;
+}
+
+/* These checks are BUG_ON()s because the driver is already supposed to avoid
+ * these cases. */
+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect,
+								int cpu)
+{
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	struct qman_portal *p;
+	p = &per_cpu(qman_affine_portal, cpu);
+	/* Check that we don't already have our own portal */
+	BUG_ON(p->config);
+	/* Check that we aren't already slaving to another portal */
+	BUG_ON(p->is_shared);
+	/* Check that 'redirect' is prepared to have us */
+	BUG_ON(!redirect->config->public_cfg.is_shared);
+	/* These are the only elements to initialise when redirecting */
+	p->irq_sources = 0;
+	p->sharing_redirect = redirect;
+	affine_portals[cpu] = p;
+	return p;
+#else
+	BUG();
+	return NULL;
+#endif
+}
+
+void qman_destroy_portal(struct qman_portal *qm)
+{
+	const struct qm_portal_config *pcfg;
+	int i;
+
+	/* Stop dequeues on the portal */
+	qm_dqrr_sdqcr_set(&qm->p, 0);
+
+	/* NB we do this to "quiesce" EQCR. If we add enqueue-completions or
+	 * something related to QM_PIRQ_EQCI, this may need fixing.
+	 * Also, due to the prefetching model used for CI updates in the enqueue
+	 * path, this update will only invalidate the CI cacheline *after*
+	 * working on it, so we need to call this twice to ensure a full update
+	 * irrespective of where the enqueue processing was at when the teardown
+	 * began. */
+	qm_eqcr_cce_update(&qm->p);
+	qm_eqcr_cce_update(&qm->p);
+	pcfg = qm->config;
+
+	free_irq(pcfg->public_cfg.irq, qm);
+
+	kfree(qm->cgrs);
+	if (num_ceetms)
+		for (i = 0; i < num_ceetms; i++)
+			kfree(qm->ccgrs[i]);
+	qm_isr_finish(&qm->p);
+	qm_mc_finish(&qm->p);
+	qm_mr_finish(&qm->p);
+	qm_dqrr_finish(&qm->p);
+	qm_eqcr_finish(&qm->p);
+
+	platform_device_del(qm->pdev);
+	platform_device_put(qm->pdev);
+
+	qm->config = NULL;
+	if (qm->alloced)
+		kfree(qm);
+}
+
+const struct qm_portal_config *qman_destroy_affine_portal(void)
+{
+	/* We don't want to redirect if we're a slave, use "raw" */
+	struct qman_portal *qm = get_raw_affine_portal();
+	const struct qm_portal_config *pcfg;
+	int cpu;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (qm->sharing_redirect) {
+		qm->sharing_redirect = NULL;
+		put_affine_portal();
+		return NULL;
+	}
+	qm->is_shared = 0;
+#endif
+	pcfg = qm->config;
+	cpu = pcfg->public_cfg.cpu;
+
+	qman_destroy_portal(qm);
+
+	spin_lock(&affine_mask_lock);
+	cpumask_clear_cpu(cpu, &affine_mask);
+	spin_unlock(&affine_mask_lock);
+	put_affine_portal();
+	return pcfg;
+}
+
+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal *p)
+{
+	return &p->config->public_cfg;
+}
+EXPORT_SYMBOL(qman_p_get_portal_config);
+
+const struct qman_portal_config *qman_get_portal_config(void)
+{
+	struct qman_portal *p = get_affine_portal();
+	const struct qman_portal_config *ret = qman_p_get_portal_config(p);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_get_portal_config);
+
+/* Inline helper to reduce nesting in __poll_portal_slow() */
+static inline void fq_state_change(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_mr_entry *msg, u8 verb)
+{
+	FQLOCK(fq);
+	switch (verb) {
+	case QM_MR_VERB_FQRL:
+		DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_ORL));
+		fq_clear(fq, QMAN_FQ_STATE_ORL);
+		table_del_fq(p, fq);
+		break;
+	case QM_MR_VERB_FQRN:
+		DPA_ASSERT((fq->state == qman_fq_state_parked) ||
+			(fq->state == qman_fq_state_sched));
+		DPA_ASSERT(fq_isset(fq, QMAN_FQ_STATE_CHANGING));
+		fq_clear(fq, QMAN_FQ_STATE_CHANGING);
+		if (msg->fq.fqs & QM_MR_FQS_NOTEMPTY)
+			fq_set(fq, QMAN_FQ_STATE_NE);
+		if (msg->fq.fqs & QM_MR_FQS_ORLPRESENT)
+			fq_set(fq, QMAN_FQ_STATE_ORL);
+		else
+			table_del_fq(p, fq);
+		fq->state = qman_fq_state_retired;
+		break;
+	case QM_MR_VERB_FQPN:
+		DPA_ASSERT(fq->state == qman_fq_state_sched);
+		DPA_ASSERT(fq_isclear(fq, QMAN_FQ_STATE_CHANGING));
+		fq->state = qman_fq_state_parked;
+	}
+	FQUNLOCK(fq);
+}
+
+static u32 __poll_portal_slow(struct qman_portal *p, u32 is)
+{
+	const struct qm_mr_entry *msg;
+
+	if (is & QM_PIRQ_CSCI) {
+		struct qman_cgrs rr, c;
+		struct qm_mc_result *mcr;
+		struct qman_cgr *cgr;
+		unsigned long irqflags __maybe_unused;
+
+		spin_lock_irqsave(&p->cgr_lock, irqflags);
+		/*
+		 * The CSCI bit must be cleared _before_ issuing the
+		 * Query Congestion State command, to ensure that a long
+		 * CGR State Change callback cannot miss an intervening
+		 * state change.
+		 */
+		qm_isr_status_clear(&p->p, QM_PIRQ_CSCI);
+		qm_mc_start(&p->p);
+		qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
+		while (!(mcr = qm_mc_result(&p->p)))
+			cpu_relax();
+		/* mask out the ones I'm not interested in */
+		qman_cgrs_and(&rr, (const struct qman_cgrs *)
+			&mcr->querycongestion.state, &p->cgrs[0]);
+		/* check previous snapshot for delta, enter/exit congestion */
+		qman_cgrs_xor(&c, &rr, &p->cgrs[1]);
+		/* update snapshot */
+		qman_cgrs_cp(&p->cgrs[1], &rr);
+		/* Invoke callback */
+		list_for_each_entry(cgr, &p->cgr_cbs, node)
+			if (cgr->cb && qman_cgrs_get(&c, cgr->cgrid))
+				cgr->cb(p, cgr, qman_cgrs_get(&rr, cgr->cgrid));
+		spin_unlock_irqrestore(&p->cgr_lock, irqflags);
+	}
+	if (is & QM_PIRQ_CCSCI) {
+		struct qman_ccgrs rr, c, congestion_result;
+		struct qm_mc_result *mcr;
+		struct qm_mc_command *mcc;
+		struct qm_ceetm_ccg *ccg;
+		unsigned long irqflags __maybe_unused;
+		int i, j, k;
+
+		spin_lock_irqsave(&p->ccgr_lock, irqflags);
+		/*
+		 * The CCSCI bit must be cleared _before_ issuing the
+		 * Query Congestion State command, to ensure that a long
+		 * CCGR State Change callback cannot miss an intervening
+		 * state change.
+		 */
+		qm_isr_status_clear(&p->p, QM_PIRQ_CCSCI);
+
+		for (i = 0; i < num_ceetms; i++) {
+			for (j = 0; j < 2; j++) {
+				mcc = qm_mc_start(&p->p);
+				mcc->ccgr_query.ccgrid = cpu_to_be16(
+					CEETM_QUERY_CONGESTION_STATE | j);
+				mcc->ccgr_query.dcpid = i;
+				qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
+				while (!(mcr = qm_mc_result(&p->p)))
+					cpu_relax();
+				for (k = 0; k < 8; k++)
+					mcr->ccgr_query.congestion_state.state.
+						__state[k] = be32_to_cpu(
+						mcr->ccgr_query.
+						congestion_state.state.
+						__state[k]);
+				congestion_result.q[j] =
+					mcr->ccgr_query.congestion_state.state;
+			}
+			/* mask out the ones I'm not interested in */
+			qman_ccgrs_and(&rr, &congestion_result,
+							&p->ccgrs[i][0]);
+			/*
+			 * check previous snapshot for delta, enter/exit
+			 * congestion.
+			 */
+			qman_ccgrs_xor(&c, &rr, &p->ccgrs[i][1]);
+			/* update snapshot */
+			qman_ccgrs_cp(&p->ccgrs[i][1], &rr);
+			/* Invoke callback */
+			list_for_each_entry(ccg, &p->ccgr_cbs[i], cb_node)
+				if (ccg->cb && qman_ccgrs_get(&c,
+					(ccg->parent->idx << 4) | ccg->idx))
+					ccg->cb(ccg, ccg->cb_ctx,
+						qman_ccgrs_get(&rr,
+							(ccg->parent->idx << 4)
+							| ccg->idx));
+		}
+		spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
+	}
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (is & QM_PIRQ_EQCI) {
+		unsigned long irqflags;
+		PORTAL_IRQ_LOCK(p, irqflags);
+		p->eqci_owned = NULL;
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		wake_up(&affine_queue);
+	}
+#endif
+
+	if (is & QM_PIRQ_EQRI) {
+		unsigned long irqflags __maybe_unused;
+		PORTAL_IRQ_LOCK(p, irqflags);
+		qm_eqcr_cce_update(&p->p);
+		qm_eqcr_set_ithresh(&p->p, 0);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		wake_up(&affine_queue);
+	}
+
+	if (is & QM_PIRQ_MRI) {
+		struct qman_fq *fq;
+		u8 verb, num = 0;
+mr_loop:
+		qm_mr_pvb_update(&p->p);
+		msg = qm_mr_current(&p->p);
+		if (!msg)
+			goto mr_done;
+		verb = msg->verb & QM_MR_VERB_TYPE_MASK;
+		/* The message is a software ERN iff the 0x20 bit is set */
+		if (verb & 0x20) {
+			switch (verb) {
+			case QM_MR_VERB_FQRNI:
+				/* nada, we drop FQRNIs on the floor */
+				break;
+			case QM_MR_VERB_FQRN:
+			case QM_MR_VERB_FQRL:
+				/* Lookup in the retirement table */
+				fq = table_find_fq(p, be32_to_cpu(msg->fq.fqid));
+				BUG_ON(!fq);
+				fq_state_change(p, fq, msg, verb);
+				if (fq->cb.fqs)
+					fq->cb.fqs(p, fq, msg);
+				break;
+			case QM_MR_VERB_FQPN:
+				/* Parked */
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+				fq = get_fq_table_entry(
+					be32_to_cpu(msg->fq.contextB));
+#else
+				fq = (void *)(uintptr_t)
+					be32_to_cpu(msg->fq.contextB);
+#endif
+				fq_state_change(p, fq, msg, verb);
+				if (fq->cb.fqs)
+					fq->cb.fqs(p, fq, msg);
+				break;
+			case QM_MR_VERB_DC_ERN:
+				/* DCP ERN */
+				if (p->cb_dc_ern)
+					p->cb_dc_ern(p, msg);
+				else if (cb_dc_ern)
+					cb_dc_ern(p, msg);
+				else {
+					static int warn_once;
+					if (!warn_once) {
+						pr_crit("Leaking DCP ERNs!\n");
+						warn_once = 1;
+					}
+				}
+				break;
+			default:
+				pr_crit("Invalid MR verb 0x%02x\n", verb);
+			}
+		} else {
+			/* Its a software ERN */
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+			pr_info("ROY\n");
+			fq = get_fq_table_entry(be32_to_cpu(msg->ern.tag));
+#else
+			fq = (void *)(uintptr_t)be32_to_cpu(msg->ern.tag);
+#endif
+			fq->cb.ern(p, fq, msg);
+		}
+		num++;
+		qm_mr_next(&p->p);
+		goto mr_loop;
+mr_done:
+		qm_mr_cci_consume(&p->p, num);
+	}
+	/*
+	 * QM_PIRQ_CSCI/CCSCI has already been cleared, as part of its specific
+	 * processing. If that interrupt source has meanwhile been re-asserted,
+	 * we mustn't clear it here (or in the top-level interrupt handler).
+	 */
+	return is & (QM_PIRQ_EQCI | QM_PIRQ_EQRI | QM_PIRQ_MRI);
+}
+
+/* remove some slowish-path stuff from the "fast path" and make sure it isn't
+ * inlined. */
+static noinline void clear_vdqcr(struct qman_portal *p, struct qman_fq *fq)
+{
+	p->vdqcr_owned = NULL;
+	FQLOCK(fq);
+	fq_clear(fq, QMAN_FQ_STATE_VDQCR);
+	FQUNLOCK(fq);
+	wake_up(&affine_queue);
+}
+
+/* Look: no locks, no irq_save()s, no preempt_disable()s! :-) The only states
+ * that would conflict with other things if they ran at the same time on the
+ * same cpu are;
+ *
+ *   (i) setting/clearing vdqcr_owned, and
+ *  (ii) clearing the NE (Not Empty) flag.
+ *
+ * Both are safe. Because;
+ *
+ *   (i) this clearing can only occur after qman_volatile_dequeue() has set the
+ *       vdqcr_owned field (which it does before setting VDQCR), and
+ *       qman_volatile_dequeue() blocks interrupts and preemption while this is
+ *       done so that we can't interfere.
+ *  (ii) the NE flag is only cleared after qman_retire_fq() has set it, and as
+ *       with (i) that API prevents us from interfering until it's safe.
+ *
+ * The good thing is that qman_volatile_dequeue() and qman_retire_fq() run far
+ * less frequently (ie. per-FQ) than __poll_portal_fast() does, so the nett
+ * advantage comes from this function not having to "lock" anything at all.
+ *
+ * Note also that the callbacks are invoked at points which are safe against the
+ * above potential conflicts, but that this function itself is not re-entrant
+ * (this is because the function tracks one end of each FIFO in the portal and
+ * we do *not* want to lock that). So the consequence is that it is safe for
+ * user callbacks to call into any Qman API *except* qman_poll() (as that's the
+ * sole API that could be invoking the callback through this function).
+ */
+static inline unsigned int __poll_portal_fast(struct qman_portal *p,
+					unsigned int poll_limit)
+{
+	const struct qm_dqrr_entry *dq;
+	struct qman_fq *fq;
+	enum qman_cb_dqrr_result res;
+	unsigned int limit = 0;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	struct qm_dqrr_entry *shadow;
+#endif
+loop:
+	qm_dqrr_pvb_update(&p->p);
+	dq = qm_dqrr_current(&p->p);
+	if (!dq)
+		goto done;
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+	/* If running on an LE system the fields of the
+	   dequeue entry must be swapped.  Because the
+	   QMan HW will ignore writes the DQRR entry is
+	   copied and the index stored within the copy */
+	shadow = &p->shadow_dqrr[DQRR_PTR2IDX(dq)];
+	*shadow = *dq;
+	dq = shadow;
+	shadow->fqid = be32_to_cpu(shadow->fqid);
+	shadow->contextB = be32_to_cpu(shadow->contextB);
+	shadow->seqnum = be16_to_cpu(shadow->seqnum);
+	hw_fd_to_cpu(&shadow->fd);
+#endif
+	if (dq->stat & QM_DQRR_STAT_UNSCHEDULED) {
+		/* VDQCR: don't trust contextB as the FQ may have been
+		 * configured for h/w consumption and we're draining it
+		 * post-retirement. */
+		fq = p->vdqcr_owned;
+		/* We only set QMAN_FQ_STATE_NE when retiring, so we only need
+		 * to check for clearing it when doing volatile dequeues. It's
+		 * one less thing to check in the critical path (SDQCR). */
+		if (dq->stat & QM_DQRR_STAT_FQ_EMPTY)
+			fq_clear(fq, QMAN_FQ_STATE_NE);
+		/* this is duplicated from the SDQCR code, but we have stuff to
+		 * do before *and* after this callback, and we don't want
+		 * multiple if()s in the critical path (SDQCR). */
+		res = fq->cb.dqrr(p, fq, dq);
+		if (res == qman_cb_dqrr_stop)
+			goto done;
+		/* Check for VDQCR completion */
+		if (dq->stat & QM_DQRR_STAT_DQCR_EXPIRED)
+			clear_vdqcr(p, fq);
+	} else {
+		/* SDQCR: contextB points to the FQ */
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+		fq = get_fq_table_entry(dq->contextB);
+#else
+		fq = (void *)(uintptr_t)dq->contextB;
+#endif
+		/* Now let the callback do its stuff */
+		res = fq->cb.dqrr(p, fq, dq);
+
+		/* The callback can request that we exit without consuming this
+		 * entry nor advancing; */
+		if (res == qman_cb_dqrr_stop)
+			goto done;
+	}
+	/* Interpret 'dq' from a driver perspective. */
+	/* Parking isn't possible unless HELDACTIVE was set. NB,
+	 * FORCEELIGIBLE implies HELDACTIVE, so we only need to
+	 * check for HELDACTIVE to cover both. */
+	DPA_ASSERT((dq->stat & QM_DQRR_STAT_FQ_HELDACTIVE) ||
+		(res != qman_cb_dqrr_park));
+	/* Defer just means "skip it, I'll consume it myself later on" */
+	if (res != qman_cb_dqrr_defer)
+		qm_dqrr_cdc_consume_1ptr(&p->p, dq, (res == qman_cb_dqrr_park));
+	/* Move forward */
+	qm_dqrr_next(&p->p);
+	/* Entry processed and consumed, increment our counter. The callback can
+	 * request that we exit after consuming the entry, and we also exit if
+	 * we reach our processing limit, so loop back only if neither of these
+	 * conditions is met. */
+	if ((++limit < poll_limit) && (res != qman_cb_dqrr_consume_stop))
+		goto loop;
+done:
+	return limit;
+}
+
+u32 qman_irqsource_get(void)
+{
+	/* "irqsource" and "poll" APIs mustn't redirect when sharing, they
+	 * should shut the user out if they are not the primary CPU hosting the
+	 * portal. That's why we use the "raw" interface. */
+	struct qman_portal *p = get_raw_affine_portal();
+	u32 ret = p->irq_sources & QM_PIRQ_VISIBLE;
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_irqsource_get);
+
+int qman_p_irqsource_add(struct qman_portal *p, u32 bits __maybe_unused)
+{
+	__maybe_unused unsigned long irqflags;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (p->sharing_redirect)
+		return -EINVAL;
+	else
+#endif
+	{
+		PORTAL_IRQ_LOCK(p, irqflags);
+		set_bits(bits & QM_PIRQ_VISIBLE, &p->irq_sources);
+		qm_isr_enable_write(&p->p, p->irq_sources);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_irqsource_add);
+
+int qman_irqsource_add(u32 bits __maybe_unused)
+{
+	struct qman_portal *p = get_raw_affine_portal();
+	int ret;
+	ret = qman_p_irqsource_add(p, bits);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_irqsource_add);
+
+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits)
+{
+	__maybe_unused unsigned long irqflags;
+	u32 ier;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (p->sharing_redirect) {
+		put_affine_portal();
+		return -EINVAL;
+	}
+#endif
+	/* Our interrupt handler only processes+clears status register bits that
+	 * are in p->irq_sources. As we're trimming that mask, if one of them
+	 * were to assert in the status register just before we remove it from
+	 * the enable register, there would be an interrupt-storm when we
+	 * release the IRQ lock. So we wait for the enable register update to
+	 * take effect in h/w (by reading it back) and then clear all other bits
+	 * in the status register. Ie. we clear them from ISR once it's certain
+	 * IER won't allow them to reassert. */
+	PORTAL_IRQ_LOCK(p, irqflags);
+	bits &= QM_PIRQ_VISIBLE;
+	clear_bits(bits, &p->irq_sources);
+	qm_isr_enable_write(&p->p, p->irq_sources);
+
+	ier = qm_isr_enable_read(&p->p);
+	/* Using "~ier" (rather than "bits" or "~p->irq_sources") creates a
+	 * data-dependency, ie. to protect against re-ordering. */
+	qm_isr_status_clear(&p->p, ~ier);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_irqsource_remove);
+
+int qman_irqsource_remove(u32 bits)
+{
+	struct qman_portal *p = get_raw_affine_portal();
+	int ret;
+	ret = qman_p_irqsource_remove(p, bits);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_irqsource_remove);
+
+const cpumask_t *qman_affine_cpus(void)
+{
+	return &affine_mask;
+}
+EXPORT_SYMBOL(qman_affine_cpus);
+
+u16 qman_affine_channel(int cpu)
+{
+	if (cpu < 0) {
+		struct qman_portal *portal = get_raw_affine_portal();
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+		BUG_ON(portal->sharing_redirect);
+#endif
+		cpu = portal->config->public_cfg.cpu;
+		put_affine_portal();
+	}
+	BUG_ON(!cpumask_test_cpu(cpu, &affine_mask));
+	return affine_channels[cpu];
+}
+EXPORT_SYMBOL(qman_affine_channel);
+
+void *qman_get_affine_portal(int cpu)
+{
+	return affine_portals[cpu];
+}
+EXPORT_SYMBOL(qman_get_affine_portal);
+
+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit)
+{
+	int ret;
+
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (unlikely(p->sharing_redirect))
+		ret = -EINVAL;
+	else
+#endif
+	{
+		BUG_ON(p->irq_sources & QM_PIRQ_DQRI);
+		ret = __poll_portal_fast(p, limit);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(qman_p_poll_dqrr);
+
+int qman_poll_dqrr(unsigned int limit)
+{
+	struct qman_portal *p = get_poll_portal();
+	int ret;
+	ret = qman_p_poll_dqrr(p, limit);
+	put_poll_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_poll_dqrr);
+
+u32 qman_p_poll_slow(struct qman_portal *p)
+{
+	u32 ret;
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (unlikely(p->sharing_redirect))
+		ret = (u32)-1;
+	else
+#endif
+	{
+		u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
+		ret = __poll_portal_slow(p, is);
+		qm_isr_status_clear(&p->p, ret);
+	}
+	return ret;
+}
+EXPORT_SYMBOL(qman_p_poll_slow);
+
+u32 qman_poll_slow(void)
+{
+	struct qman_portal *p = get_poll_portal();
+	u32 ret;
+	ret = qman_p_poll_slow(p);
+	put_poll_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_poll_slow);
+
+/* Legacy wrapper */
+void qman_p_poll(struct qman_portal *p)
+{
+#ifdef CONFIG_FSL_DPA_PORTAL_SHARE
+	if (unlikely(p->sharing_redirect))
+		return;
+#endif
+	if ((~p->irq_sources) & QM_PIRQ_SLOW) {
+		if (!(p->slowpoll--)) {
+			u32 is = qm_isr_status_read(&p->p) & ~p->irq_sources;
+			u32 active = __poll_portal_slow(p, is);
+			if (active) {
+				qm_isr_status_clear(&p->p, active);
+				p->slowpoll = SLOW_POLL_BUSY;
+			} else
+				p->slowpoll = SLOW_POLL_IDLE;
+		}
+	}
+	if ((~p->irq_sources) & QM_PIRQ_DQRI)
+		__poll_portal_fast(p, CONFIG_FSL_QMAN_POLL_LIMIT);
+}
+EXPORT_SYMBOL(qman_p_poll);
+
+void qman_poll(void)
+{
+	struct qman_portal *p = get_poll_portal();
+	qman_p_poll(p);
+	put_poll_portal();
+}
+EXPORT_SYMBOL(qman_poll);
+
+void qman_p_stop_dequeues(struct qman_portal *p)
+{
+	qman_stop_dequeues_ex(p);
+}
+EXPORT_SYMBOL(qman_p_stop_dequeues);
+
+void qman_stop_dequeues(void)
+{
+	struct qman_portal *p = get_affine_portal();
+	qman_p_stop_dequeues(p);
+	put_affine_portal();
+}
+EXPORT_SYMBOL(qman_stop_dequeues);
+
+void qman_p_start_dequeues(struct qman_portal *p)
+{
+	unsigned long irqflags __maybe_unused;
+	PORTAL_IRQ_LOCK(p, irqflags);
+	DPA_ASSERT(p->dqrr_disable_ref > 0);
+	if (!(--p->dqrr_disable_ref))
+		qm_dqrr_set_maxfill(&p->p, DQRR_MAXFILL);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+}
+EXPORT_SYMBOL(qman_p_start_dequeues);
+
+void qman_start_dequeues(void)
+{
+	struct qman_portal *p = get_affine_portal();
+	qman_p_start_dequeues(p);
+	put_affine_portal();
+}
+EXPORT_SYMBOL(qman_start_dequeues);
+
+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools)
+{
+	unsigned long irqflags __maybe_unused;
+	PORTAL_IRQ_LOCK(p, irqflags);
+	pools &= p->config->public_cfg.pools;
+	p->sdqcr |= pools;
+	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+}
+EXPORT_SYMBOL(qman_p_static_dequeue_add);
+
+void qman_static_dequeue_add(u32 pools)
+{
+	struct qman_portal *p = get_affine_portal();
+	qman_p_static_dequeue_add(p, pools);
+	put_affine_portal();
+}
+EXPORT_SYMBOL(qman_static_dequeue_add);
+
+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools)
+{
+	unsigned long irqflags __maybe_unused;
+	PORTAL_IRQ_LOCK(p, irqflags);
+	pools &= p->config->public_cfg.pools;
+	p->sdqcr &= ~pools;
+	qm_dqrr_sdqcr_set(&p->p, p->sdqcr);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+}
+EXPORT_SYMBOL(qman_p_static_dequeue_del);
+
+void qman_static_dequeue_del(u32 pools)
+{
+	struct qman_portal *p = get_affine_portal();
+	qman_p_static_dequeue_del(p, pools);
+	put_affine_portal();
+}
+EXPORT_SYMBOL(qman_static_dequeue_del);
+
+u32 qman_p_static_dequeue_get(struct qman_portal *p)
+{
+	return p->sdqcr;
+}
+EXPORT_SYMBOL(qman_p_static_dequeue_get);
+
+u32 qman_static_dequeue_get(void)
+{
+	struct qman_portal *p = get_affine_portal();
+	u32 ret = qman_p_static_dequeue_get(p);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_static_dequeue_get);
+
+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq,
+						int park_request)
+{
+	qm_dqrr_cdc_consume_1ptr(&p->p, dq, park_request);
+}
+EXPORT_SYMBOL(qman_p_dca);
+
+void qman_dca(struct qm_dqrr_entry *dq, int park_request)
+{
+	struct qman_portal *p = get_affine_portal();
+	qman_p_dca(p, dq, park_request);
+	put_affine_portal();
+}
+EXPORT_SYMBOL(qman_dca);
+
+/*******************/
+/* Frame queue API */
+/*******************/
+
+static const char *mcr_result_str(u8 result)
+{
+	switch (result) {
+	case QM_MCR_RESULT_NULL:
+		return "QM_MCR_RESULT_NULL";
+	case QM_MCR_RESULT_OK:
+		return "QM_MCR_RESULT_OK";
+	case QM_MCR_RESULT_ERR_FQID:
+		return "QM_MCR_RESULT_ERR_FQID";
+	case QM_MCR_RESULT_ERR_FQSTATE:
+		return "QM_MCR_RESULT_ERR_FQSTATE";
+	case QM_MCR_RESULT_ERR_NOTEMPTY:
+		return "QM_MCR_RESULT_ERR_NOTEMPTY";
+	case QM_MCR_RESULT_PENDING:
+		return "QM_MCR_RESULT_PENDING";
+	case QM_MCR_RESULT_ERR_BADCOMMAND:
+		return "QM_MCR_RESULT_ERR_BADCOMMAND";
+	}
+	return "<unknown MCR result>";
+}
+
+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq)
+{
+	struct qm_fqd fqd;
+	struct qm_mcr_queryfq_np np;
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+
+	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID) {
+		int ret = qman_alloc_fqid(&fqid);
+		if (ret)
+			return ret;
+	}
+	spin_lock_init(&fq->fqlock);
+	fq->fqid = fqid;
+	fq->flags = flags;
+	fq->state = qman_fq_state_oos;
+	fq->cgr_groupid = 0;
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+	if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
+		return -ENOMEM;
+#endif
+	if (!(flags & QMAN_FQ_FLAG_AS_IS) || (flags & QMAN_FQ_FLAG_NO_MODIFY))
+		return 0;
+	/* Everything else is AS_IS support */
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	mcc->queryfq.fqid = cpu_to_be32(fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ);
+	if (mcr->result != QM_MCR_RESULT_OK) {
+		pr_err("QUERYFQ failed: %s\n", mcr_result_str(mcr->result));
+		goto err;
+	}
+	fqd = mcr->queryfq.fqd;
+	hw_fqd_to_cpu(&fqd);
+	mcc = qm_mc_start(&p->p);
+	mcc->queryfq_np.fqid = cpu_to_be32(fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYFQ_NP);
+	if (mcr->result != QM_MCR_RESULT_OK) {
+		pr_err("QUERYFQ_NP failed: %s\n", mcr_result_str(mcr->result));
+		goto err;
+	}
+	np = mcr->queryfq_np;
+	/* Phew, have queryfq and queryfq_np results, stitch together
+	 * the FQ object from those. */
+	fq->cgr_groupid = fqd.cgid;
+	switch (np.state & QM_MCR_NP_STATE_MASK) {
+	case QM_MCR_NP_STATE_OOS:
+		break;
+	case QM_MCR_NP_STATE_RETIRED:
+		fq->state = qman_fq_state_retired;
+		if (np.frm_cnt)
+			fq_set(fq, QMAN_FQ_STATE_NE);
+		break;
+	case QM_MCR_NP_STATE_TEN_SCHED:
+	case QM_MCR_NP_STATE_TRU_SCHED:
+	case QM_MCR_NP_STATE_ACTIVE:
+		fq->state = qman_fq_state_sched;
+		if (np.state & QM_MCR_NP_STATE_R)
+			fq_set(fq, QMAN_FQ_STATE_CHANGING);
+		break;
+	case QM_MCR_NP_STATE_PARKED:
+		fq->state = qman_fq_state_parked;
+		break;
+	default:
+		DPA_ASSERT(NULL == "invalid FQ state");
+	}
+	if (fqd.fq_ctrl & QM_FQCTRL_CGE)
+		fq->state |= QMAN_FQ_STATE_CGR_EN;
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return 0;
+err:
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (flags & QMAN_FQ_FLAG_DYNAMIC_FQID)
+		qman_release_fqid(fqid);
+	return -EIO;
+}
+EXPORT_SYMBOL(qman_create_fq);
+
+void qman_destroy_fq(struct qman_fq *fq, u32 flags __maybe_unused)
+{
+
+	/* We don't need to lock the FQ as it is a pre-condition that the FQ be
+	 * quiesced. Instead, run some checks. */
+	switch (fq->state) {
+	case qman_fq_state_parked:
+		DPA_ASSERT(flags & QMAN_FQ_DESTROY_PARKED);
+	case qman_fq_state_oos:
+		if (fq_isset(fq, QMAN_FQ_FLAG_DYNAMIC_FQID))
+			qman_release_fqid(fq->fqid);
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+		clear_fq_table_entry(fq->key);
+#endif
+		return;
+	default:
+		break;
+	}
+	DPA_ASSERT(NULL == "qman_free_fq() on unquiesced FQ!");
+}
+EXPORT_SYMBOL(qman_destroy_fq);
+
+u32 qman_fq_fqid(struct qman_fq *fq)
+{
+	return fq->fqid;
+}
+EXPORT_SYMBOL(qman_fq_fqid);
+
+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags)
+{
+	if (state)
+		*state = fq->state;
+	if (flags)
+		*flags = fq->flags;
+}
+EXPORT_SYMBOL(qman_fq_state);
+
+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res, myverb = (flags & QMAN_INITFQ_FLAG_SCHED) ?
+		QM_MCC_VERB_INITFQ_SCHED : QM_MCC_VERB_INITFQ_PARKED;
+
+	if ((fq->state != qman_fq_state_oos) &&
+			(fq->state != qman_fq_state_parked))
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
+		return -EINVAL;
+#endif
+	if (opts && (opts->we_mask & QM_INITFQ_WE_OAC)) {
+		/* And can't be set at the same time as TDTHRESH */
+		if (opts->we_mask & QM_INITFQ_WE_TDTHRESH)
+			return -EINVAL;
+	}
+	/* Issue an INITFQ_[PARKED|SCHED] management command */
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	FQLOCK(fq);
+	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
+			((fq->state != qman_fq_state_oos) &&
+				(fq->state != qman_fq_state_parked)))) {
+		FQUNLOCK(fq);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		put_affine_portal();
+		return -EBUSY;
+	}
+	mcc = qm_mc_start(&p->p);
+	if (opts)
+		mcc->initfq = *opts;
+	mcc->initfq.fqid = cpu_to_be32(fq->fqid);
+	mcc->initfq.count = 0;
+
+	/* If the FQ does *not* have the TO_DCPORTAL flag, contextB is set as a
+	 * demux pointer. Otherwise, the caller-provided value is allowed to
+	 * stand, don't overwrite it. */
+	if (fq_isclear(fq, QMAN_FQ_FLAG_TO_DCPORTAL)) {
+		dma_addr_t phys_fq;
+		mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTB;
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+		mcc->initfq.fqd.context_b = fq->key;
+#else
+		mcc->initfq.fqd.context_b = (u32)(uintptr_t)fq;
+#endif
+		/* and the physical address - NB, if the user wasn't trying to
+		 * set CONTEXTA, clear the stashing settings. */
+		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_CONTEXTA)) {
+			mcc->initfq.we_mask |= QM_INITFQ_WE_CONTEXTA;
+			memset(&mcc->initfq.fqd.context_a, 0,
+				sizeof(mcc->initfq.fqd.context_a));
+		} else {
+			phys_fq = dma_map_single(&p->pdev->dev, fq, sizeof(*fq),
+						DMA_TO_DEVICE);
+			qm_fqd_stashing_set64(&mcc->initfq.fqd, phys_fq);
+		}
+	}
+	if (flags & QMAN_INITFQ_FLAG_LOCAL) {
+		mcc->initfq.fqd.dest.channel = p->config->public_cfg.channel;
+		if (!(mcc->initfq.we_mask & QM_INITFQ_WE_DESTWQ)) {
+			mcc->initfq.we_mask |= QM_INITFQ_WE_DESTWQ;
+			mcc->initfq.fqd.dest.wq = 4;
+		}
+	}
+	mcc->initfq.we_mask = cpu_to_be16(mcc->initfq.we_mask);
+	cpu_to_hw_fqd(&mcc->initfq.fqd);
+	qm_mc_commit(&p->p, myverb);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		FQUNLOCK(fq);
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		put_affine_portal();
+		return -EIO;
+	}
+	if (opts) {
+		if (opts->we_mask & QM_INITFQ_WE_FQCTRL) {
+			if (opts->fqd.fq_ctrl & QM_FQCTRL_CGE)
+				fq_set(fq, QMAN_FQ_STATE_CGR_EN);
+			else
+				fq_clear(fq, QMAN_FQ_STATE_CGR_EN);
+		}
+		if (opts->we_mask & QM_INITFQ_WE_CGID)
+			fq->cgr_groupid = opts->fqd.cgid;
+	}
+	fq->state = (flags & QMAN_INITFQ_FLAG_SCHED) ?
+			qman_fq_state_sched : qman_fq_state_parked;
+	FQUNLOCK(fq);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return 0;
+}
+EXPORT_SYMBOL(qman_init_fq);
+
+int qman_schedule_fq(struct qman_fq *fq)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	int ret = 0;
+	u8 res;
+
+	if (fq->state != qman_fq_state_parked)
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
+		return -EINVAL;
+#endif
+	/* Issue a ALTERFQ_SCHED management command */
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	FQLOCK(fq);
+	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
+			(fq->state != qman_fq_state_parked))) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_SCHED);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_SCHED);
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+	fq->state = qman_fq_state_sched;
+out:
+	FQUNLOCK(fq);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_schedule_fq);
+
+int qman_retire_fq(struct qman_fq *fq, u32 *flags)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	int rval;
+	u8 res;
+
+	if ((fq->state != qman_fq_state_parked) &&
+			(fq->state != qman_fq_state_sched))
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
+		return -EINVAL;
+#endif
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	FQLOCK(fq);
+	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
+			(fq->state == qman_fq_state_retired) ||
+				(fq->state == qman_fq_state_oos))) {
+		rval = -EBUSY;
+		goto out;
+	}
+	rval = table_push_fq(p, fq);
+	if (rval)
+		goto out;
+	mcc = qm_mc_start(&p->p);
+	mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_RETIRE);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_RETIRE);
+	res = mcr->result;
+	/* "Elegant" would be to treat OK/PENDING the same way; set CHANGING,
+	 * and defer the flags until FQRNI or FQRN (respectively) show up. But
+	 * "Friendly" is to process OK immediately, and not set CHANGING. We do
+	 * friendly, otherwise the caller doesn't necessarily have a fully
+	 * "retired" FQ on return even if the retirement was immediate. However
+	 * this does mean some code duplication between here and
+	 * fq_state_change(). */
+	if (likely(res == QM_MCR_RESULT_OK)) {
+		rval = 0;
+		/* Process 'fq' right away, we'll ignore FQRNI */
+		if (mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY)
+			fq_set(fq, QMAN_FQ_STATE_NE);
+		if (mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)
+			fq_set(fq, QMAN_FQ_STATE_ORL);
+		else
+			table_del_fq(p, fq);
+		if (flags)
+			*flags = fq->flags;
+		fq->state = qman_fq_state_retired;
+		if (fq->cb.fqs) {
+			/* Another issue with supporting "immediate" retirement
+			 * is that we're forced to drop FQRNIs, because by the
+			 * time they're seen it may already be "too late" (the
+			 * fq may have been OOS'd and free()'d already). But if
+			 * the upper layer wants a callback whether it's
+			 * immediate or not, we have to fake a "MR" entry to
+			 * look like an FQRNI... */
+			struct qm_mr_entry msg;
+			msg.verb = QM_MR_VERB_FQRNI;
+			msg.fq.fqs = mcr->alterfq.fqs;
+			msg.fq.fqid = fq->fqid;
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+			msg.fq.contextB = fq->key;
+#else
+			msg.fq.contextB = (u32)(uintptr_t)fq;
+#endif
+			fq->cb.fqs(p, fq, &msg);
+		}
+	} else if (res == QM_MCR_RESULT_PENDING) {
+		rval = 1;
+		fq_set(fq, QMAN_FQ_STATE_CHANGING);
+	} else {
+		rval = -EIO;
+		table_del_fq(p, fq);
+	}
+out:
+	FQUNLOCK(fq);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return rval;
+}
+EXPORT_SYMBOL(qman_retire_fq);
+
+int qman_oos_fq(struct qman_fq *fq)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	int ret = 0;
+	u8 res;
+
+	if (fq->state != qman_fq_state_retired)
+		return -EINVAL;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
+		return -EINVAL;
+#endif
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	FQLOCK(fq);
+	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_BLOCKOOS)) ||
+			(fq->state != qman_fq_state_retired))) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	mcc->alterfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_ALTER_OOS);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_ALTER_OOS);
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+	fq->state = qman_fq_state_oos;
+out:
+	FQUNLOCK(fq);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_oos_fq);
+
+int qman_fq_flow_control(struct qman_fq *fq, int xon)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	int ret = 0;
+	u8 res;
+	u8 myverb;
+
+	if ((fq->state == qman_fq_state_oos) ||
+		(fq->state == qman_fq_state_retired) ||
+		(fq->state == qman_fq_state_parked))
+		return -EINVAL;
+
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (unlikely(fq_isset(fq, QMAN_FQ_FLAG_NO_MODIFY)))
+		return -EINVAL;
+#endif
+	/* Issue a ALTER_FQXON or ALTER_FQXOFF management command */
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	FQLOCK(fq);
+	if (unlikely((fq_isset(fq, QMAN_FQ_STATE_CHANGING)) ||
+			(fq->state == qman_fq_state_parked) ||
+			(fq->state == qman_fq_state_oos) ||
+			(fq->state == qman_fq_state_retired))) {
+		ret = -EBUSY;
+		goto out;
+	}
+	mcc = qm_mc_start(&p->p);
+	mcc->alterfq.fqid = fq->fqid;
+	mcc->alterfq.count = 0;
+	myverb = xon ? QM_MCC_VERB_ALTER_FQXON : QM_MCC_VERB_ALTER_FQXOFF;
+
+	qm_mc_commit(&p->p, myverb);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		ret = -EIO;
+		goto out;
+	}
+out:
+	FQUNLOCK(fq);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_fq_flow_control);
+
+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK)
+		memcpy_fromio(fqd, &mcr->queryfq.fqd, sizeof(*fqd));
+	hw_fqd_to_cpu(fqd);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK)
+		return -EIO;
+	return 0;
+}
+EXPORT_SYMBOL(qman_query_fq);
+
+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	mcc->queryfq.fqid = cpu_to_be32(fq->fqid);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYFQ_NP);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK) {
+		memcpy_fromio(np, &mcr->queryfq_np, sizeof(*np));
+		np->fqd_link = be24_to_cpu(np->fqd_link);
+		np->odp_seq = be16_to_cpu(np->odp_seq);
+		np->orp_nesn = be16_to_cpu(np->orp_nesn);
+		np->orp_ea_hseq  = be16_to_cpu(np->orp_ea_hseq);
+		np->orp_ea_tseq  = be16_to_cpu(np->orp_ea_tseq);
+		np->orp_ea_hptr = be24_to_cpu(np->orp_ea_hptr);
+		np->orp_ea_tptr = be24_to_cpu(np->orp_ea_tptr);
+		np->pfdr_hptr = be24_to_cpu(np->pfdr_hptr);
+		np->pfdr_tptr = be24_to_cpu(np->pfdr_tptr);
+		np->ics_surp = be16_to_cpu(np->ics_surp);
+		np->byte_cnt = be32_to_cpu(np->byte_cnt);
+		np->frm_cnt = be24_to_cpu(np->frm_cnt);
+		np->ra1_sfdr = be16_to_cpu(np->ra1_sfdr);
+		np->ra2_sfdr = be16_to_cpu(np->ra2_sfdr);
+		np->od1_sfdr = be16_to_cpu(np->od1_sfdr);
+		np->od2_sfdr = be16_to_cpu(np->od2_sfdr);
+		np->od3_sfdr = be16_to_cpu(np->od3_sfdr);
+
+
+	}
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res == QM_MCR_RESULT_ERR_FQID)
+		return -ERANGE;
+	else if (res != QM_MCR_RESULT_OK)
+		return -EIO;
+	return 0;
+}
+EXPORT_SYMBOL(qman_query_fq_np);
+
+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res, myverb;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	myverb = (query_dedicated) ? QM_MCR_VERB_QUERYWQ_DEDICATED :
+				 QM_MCR_VERB_QUERYWQ;
+	mcc = qm_mc_start(&p->p);
+	mcc->querywq.channel.id = cpu_to_be16(wq->channel.id);
+	qm_mc_commit(&p->p, myverb);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == myverb);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK) {
+		int i, array_len;
+		wq->channel.id = be16_to_cpu(mcr->querywq.channel.id);
+		array_len = ARRAY_SIZE(mcr->querywq.wq_len);
+		for (i = 0; i < array_len; i++)
+			wq->wq_len[i] = be32_to_cpu(mcr->querywq.wq_len[i]);
+	}
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("QUERYWQ failed: %s\n", mcr_result_str(res));
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_query_wq);
+
+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
+			struct qm_mcr_cgrtestwrite *result)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	mcc->cgrtestwrite.cgid = cgr->cgrid;
+	mcc->cgrtestwrite.i_bcnt_hi = (u8)(i_bcnt >> 32);
+	mcc->cgrtestwrite.i_bcnt_lo = (u32)i_bcnt;
+	qm_mc_commit(&p->p, QM_MCC_VERB_CGRTESTWRITE);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_CGRTESTWRITE);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK)
+		memcpy_fromio(result,  &mcr->cgrtestwrite, sizeof(*result));
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CGR TEST WRITE failed: %s\n", mcr_result_str(res));
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_testwrite_cgr);
+
+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *cgrd)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+	int i;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	mcc->querycgr.cgid = cgr->cgrid;
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCGR);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCC_VERB_QUERYCGR);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK)
+		memcpy_fromio(cgrd, &mcr->querycgr, sizeof(*cgrd));
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("QUERY_CGR failed: %s\n", mcr_result_str(res));
+		return -EIO;
+	}
+	cgrd->cgr.wr_parm_g.word =
+		be32_to_cpu(cgrd->cgr.wr_parm_g.word);
+	cgrd->cgr.wr_parm_y.word =
+		be32_to_cpu(cgrd->cgr.wr_parm_y.word);
+	cgrd->cgr.wr_parm_r.word =
+		be32_to_cpu(cgrd->cgr.wr_parm_r.word);
+	cgrd->cgr.cscn_targ =  be32_to_cpu(cgrd->cgr.cscn_targ);
+	cgrd->cgr.__cs_thres = be16_to_cpu(cgrd->cgr.__cs_thres);
+	for (i = 0; i < ARRAY_SIZE(cgrd->cscn_targ_swp); i++)
+			be32_to_cpus(&cgrd->cscn_targ_swp[i]);
+	return 0;
+}
+EXPORT_SYMBOL(qman_query_cgr);
+
+int qman_query_congestion(struct qm_mcr_querycongestion *congestion)
+{
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+	int i;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	qm_mc_start(&p->p);
+	qm_mc_commit(&p->p, QM_MCC_VERB_QUERYCONGESTION);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			QM_MCC_VERB_QUERYCONGESTION);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK)
+		memcpy_fromio(congestion, &mcr->querycongestion,
+			      sizeof(*congestion));
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("QUERY_CONGESTION failed: %s\n", mcr_result_str(res));
+		return -EIO;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(congestion->state.__state); i++)
+			be32_to_cpus(&congestion->state.__state[i]);
+	return 0;
+}
+EXPORT_SYMBOL(qman_query_congestion);
+
+/* internal function used as a wait_event() expression */
+static int set_p_vdqcr(struct qman_portal *p, struct qman_fq *fq, u32 vdqcr)
+{
+	unsigned long irqflags __maybe_unused;
+	int ret = -EBUSY;
+	PORTAL_IRQ_LOCK(p, irqflags);
+	if (!p->vdqcr_owned) {
+		FQLOCK(fq);
+		if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
+			goto escape;
+		fq_set(fq, QMAN_FQ_STATE_VDQCR);
+		FQUNLOCK(fq);
+		p->vdqcr_owned = fq;
+		ret = 0;
+	}
+escape:
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	if (!ret)
+		qm_dqrr_vdqcr_set(&p->p, vdqcr);
+	return ret;
+}
+
+static int set_vdqcr(struct qman_portal **p, struct qman_fq *fq, u32 vdqcr)
+{
+	int ret;
+	*p = get_affine_portal();
+	ret = set_p_vdqcr(*p, fq, vdqcr);
+	put_affine_portal();
+	return ret;
+}
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+static int wait_p_vdqcr_start(struct qman_portal *p, struct qman_fq *fq,
+				u32 vdqcr, u32 flags)
+{
+	int ret = 0;
+	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+		ret = wait_event_interruptible(affine_queue,
+				!(ret = set_p_vdqcr(p, fq, vdqcr)));
+	else
+		wait_event(affine_queue, !(ret = set_p_vdqcr(p, fq, vdqcr)));
+	return ret;
+}
+
+static int wait_vdqcr_start(struct qman_portal **p, struct qman_fq *fq,
+				u32 vdqcr, u32 flags)
+{
+	int ret = 0;
+	if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+		ret = wait_event_interruptible(affine_queue,
+				!(ret = set_vdqcr(p, fq, vdqcr)));
+	else
+		wait_event(affine_queue, !(ret = set_vdqcr(p, fq, vdqcr)));
+	return ret;
+}
+#endif
+
+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq,
+					u32 flags __maybe_unused, u32 vdqcr)
+{
+	int ret;
+
+	if ((fq->state != qman_fq_state_parked) &&
+			(fq->state != qman_fq_state_retired))
+		return -EINVAL;
+	if (vdqcr & QM_VDQCR_FQID_MASK)
+		return -EINVAL;
+	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
+		return -EBUSY;
+	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_VOLATILE_FLAG_WAIT)
+		ret = wait_p_vdqcr_start(p, fq, vdqcr, flags);
+	else
+#endif
+		ret = set_p_vdqcr(p, fq, vdqcr);
+	if (ret)
+		return ret;
+	/* VDQCR is set */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
+		if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+			/* NB: don't propagate any error - the caller wouldn't
+			 * know whether the VDQCR was issued or not. A signal
+			 * could arrive after returning anyway, so the caller
+			 * can check signal_pending() if that's an issue. */
+			wait_event_interruptible(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+		else
+			wait_event(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_volatile_dequeue);
+
+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags __maybe_unused,
+				u32 vdqcr)
+{
+	struct qman_portal *p;
+	int ret;
+
+	if ((fq->state != qman_fq_state_parked) &&
+			(fq->state != qman_fq_state_retired))
+		return -EINVAL;
+	if (vdqcr & QM_VDQCR_FQID_MASK)
+		return -EINVAL;
+	if (fq_isset(fq, QMAN_FQ_STATE_VDQCR))
+		return -EBUSY;
+	vdqcr = (vdqcr & ~QM_VDQCR_FQID_MASK) | fq->fqid;
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_VOLATILE_FLAG_WAIT)
+		ret = wait_vdqcr_start(&p, fq, vdqcr, flags);
+	else
+#endif
+		ret = set_vdqcr(&p, fq, vdqcr);
+	if (ret)
+		return ret;
+	/* VDQCR is set */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_VOLATILE_FLAG_FINISH) {
+		if (flags & QMAN_VOLATILE_FLAG_WAIT_INT)
+			/* NB: don't propagate any error - the caller wouldn't
+			 * know whether the VDQCR was issued or not. A signal
+			 * could arrive after returning anyway, so the caller
+			 * can check signal_pending() if that's an issue. */
+			wait_event_interruptible(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+		else
+			wait_event(affine_queue,
+				!fq_isset(fq, QMAN_FQ_STATE_VDQCR));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_volatile_dequeue);
+
+static noinline void update_eqcr_ci(struct qman_portal *p, u8 avail)
+{
+	if (avail)
+		qm_eqcr_cce_prefetch(&p->p);
+	else
+		qm_eqcr_cce_update(&p->p);
+}
+
+int qman_eqcr_is_empty(void)
+{
+	unsigned long irqflags __maybe_unused;
+	struct qman_portal *p = get_affine_portal();
+	u8 avail;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	update_eqcr_ci(p, 0);
+	avail = qm_eqcr_get_fill(&p->p);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return avail == 0;
+}
+EXPORT_SYMBOL(qman_eqcr_is_empty);
+
+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine)
+{
+	if (affine) {
+		unsigned long irqflags __maybe_unused;
+		struct qman_portal *p = get_affine_portal();
+		PORTAL_IRQ_LOCK(p, irqflags);
+		p->cb_dc_ern = handler;
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		put_affine_portal();
+	} else
+		cb_dc_ern = handler;
+}
+EXPORT_SYMBOL(qman_set_dc_ern);
+
+static inline struct qm_eqcr_entry *try_p_eq_start(struct qman_portal *p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq;
+	u8 avail;
+	PORTAL_IRQ_LOCK(p, (*irqflags));
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (p->eqci_owned) {
+			PORTAL_IRQ_UNLOCK(p, (*irqflags));
+			return NULL;
+		}
+		p->eqci_owned = fq;
+	}
+#endif
+	if (p->use_eqcr_ci_stashing) {
+		/*
+		 * The stashing case is easy, only update if we need to in
+		 * order to try and liberate ring entries.
+		 */
+		eq = qm_eqcr_start_stash(&p->p);
+	} else {
+		/*
+		 * The non-stashing case is harder, need to prefetch ahead of
+		 * time.
+		 */
+		avail = qm_eqcr_get_avail(&p->p);
+		if (avail < 2)
+			update_eqcr_ci(p, avail);
+		eq = qm_eqcr_start_no_stash(&p->p);
+	}
+
+	if (unlikely(!eq)) {
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+		if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+				(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC)))
+			p->eqci_owned = NULL;
+#endif
+		PORTAL_IRQ_UNLOCK(p, (*irqflags));
+		return NULL;
+	}
+	if (flags & QMAN_ENQUEUE_FLAG_DCA)
+		eq->dca = QM_EQCR_DCA_ENABLE |
+			((flags & QMAN_ENQUEUE_FLAG_DCA_PARK) ?
+					QM_EQCR_DCA_PARK : 0) |
+			((flags >> 8) & QM_EQCR_DCA_IDXMASK);
+	eq->fqid = cpu_to_be32(fq->fqid);
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+	eq->tag = cpu_to_be32(fq->key);
+#else
+	eq->tag = cpu_to_be32((u32)(uintptr_t)fq);
+#endif
+	eq->fd = *fd;
+	cpu_to_hw_fd(&eq->fd);
+	return eq;
+}
+
+static inline struct qm_eqcr_entry *try_eq_start(struct qman_portal **p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq;
+	*p = get_affine_portal();
+	eq = try_p_eq_start(*p, irqflags, fq, fd, flags);
+	if (!eq)
+		put_affine_portal();
+	return eq;
+}
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+static noinline struct qm_eqcr_entry *__wait_eq_start(struct qman_portal **p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq = try_eq_start(p, irqflags, fq, fd, flags);
+	if (!eq)
+		qm_eqcr_set_ithresh(&(*p)->p, EQCR_ITHRESH);
+	return eq;
+}
+static noinline struct qm_eqcr_entry *wait_eq_start(struct qman_portal **p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq;
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+		/* NB: return NULL if signal occurs before completion. Signal
+		 * can occur during return. Caller must check for signal */
+		wait_event_interruptible(affine_queue,
+			(eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
+	else
+		wait_event(affine_queue,
+			(eq = __wait_eq_start(p, irqflags, fq, fd, flags)));
+	return eq;
+}
+static noinline struct qm_eqcr_entry *__wait_p_eq_start(struct qman_portal *p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq = try_p_eq_start(p, irqflags, fq, fd, flags);
+	if (!eq)
+		qm_eqcr_set_ithresh(&p->p, EQCR_ITHRESH);
+	return eq;
+}
+static noinline struct qm_eqcr_entry *wait_p_eq_start(struct qman_portal *p,
+					unsigned long *irqflags __maybe_unused,
+					struct qman_fq *fq,
+					const struct qm_fd *fd,
+					u32 flags)
+{
+	struct qm_eqcr_entry *eq;
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+		/* NB: return NULL if signal occurs before completion. Signal
+		 * can occur during return. Caller must check for signal */
+		wait_event_interruptible(affine_queue,
+			(eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
+	else
+		wait_event(affine_queue,
+			(eq = __wait_p_eq_start(p, irqflags, fq, fd, flags)));
+	return eq;
+}
+#endif
+
+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_fd *fd, u32 flags)
+{
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	/* Factor the below out, it's used from qman_enqueue_orp() too */
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_enqueue);
+
+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags)
+{
+	struct qman_portal *p;
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	/* Factor the below out, it's used from qman_enqueue_orp() too */
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_enqueue);
+
+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_fd *fd, u32 flags,
+				struct qman_fq *orp, u16 orp_seqnum)
+{
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* Process ORP-specifics here */
+	if (flags & QMAN_ENQUEUE_FLAG_NLIS)
+		orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
+	else {
+		orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
+		if (flags & QMAN_ENQUEUE_FLAG_NESN)
+			orp_seqnum |= QM_EQCR_SEQNUM_NESN;
+		else
+			/* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
+			orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
+	}
+	eq->seqnum = cpu_to_be16(orp_seqnum);
+	eq->orp = cpu_to_be32(orp->fqid);
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
+		((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
+				0 : QM_EQCR_VERB_CMD_ENQUEUE) |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_enqueue_orp);
+
+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
+			struct qman_fq *orp, u16 orp_seqnum)
+{
+	struct qman_portal *p;
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* Process ORP-specifics here */
+	if (flags & QMAN_ENQUEUE_FLAG_NLIS)
+		orp_seqnum |= QM_EQCR_SEQNUM_NLIS;
+	else {
+		orp_seqnum &= ~QM_EQCR_SEQNUM_NLIS;
+		if (flags & QMAN_ENQUEUE_FLAG_NESN)
+			orp_seqnum |= QM_EQCR_SEQNUM_NESN;
+		else
+			/* No need to check 4 QMAN_ENQUEUE_FLAG_HOLE */
+			orp_seqnum &= ~QM_EQCR_SEQNUM_NESN;
+	}
+	eq->seqnum = cpu_to_be16(orp_seqnum);
+	eq->orp = cpu_to_be32(orp->fqid);
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_ORP |
+		((flags & (QMAN_ENQUEUE_FLAG_HOLE | QMAN_ENQUEUE_FLAG_NESN)) ?
+				0 : QM_EQCR_VERB_CMD_ENQUEUE) |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_enqueue_orp);
+
+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_fd *fd, u32 flags,
+				qman_cb_precommit cb, void *cb_arg)
+{
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_p_eq_start(p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_p_eq_start(p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* invoke user supplied callback function before writing commit verb */
+	if (cb(cb_arg)) {
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		return -EINVAL;
+	}
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	/* Factor the below out, it's used from qman_enqueue_orp() too */
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_p_enqueue_precommit);
+
+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd,
+		u32 flags, qman_cb_precommit cb, void *cb_arg)
+{
+	struct qman_portal *p;
+	struct qm_eqcr_entry *eq;
+	unsigned long irqflags __maybe_unused;
+
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+	if (flags & QMAN_ENQUEUE_FLAG_WAIT)
+		eq = wait_eq_start(&p, &irqflags, fq, fd, flags);
+	else
+#endif
+	eq = try_eq_start(&p, &irqflags, fq, fd, flags);
+	if (!eq)
+		return -EBUSY;
+	/* invoke user supplied callback function before writing commit verb */
+	if (cb(cb_arg)) {
+		PORTAL_IRQ_UNLOCK(p, irqflags);
+		put_affine_portal();
+		return -EINVAL;
+	}
+	/* Note: QM_EQCR_VERB_INTERRUPT == QMAN_ENQUEUE_FLAG_WAIT_SYNC */
+	qm_eqcr_pvb_commit(&p->p, QM_EQCR_VERB_CMD_ENQUEUE |
+		(flags & (QM_EQCR_VERB_COLOUR_MASK | QM_EQCR_VERB_INTERRUPT)));
+	/* Factor the below out, it's used from qman_enqueue_orp() too */
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+	if (unlikely((flags & QMAN_ENQUEUE_FLAG_WAIT) &&
+			(flags & QMAN_ENQUEUE_FLAG_WAIT_SYNC))) {
+		if (flags & QMAN_ENQUEUE_FLAG_WAIT_INT)
+			/* NB: return success even if signal occurs before
+			 * condition is true. pvb_commit guarantees success */
+			wait_event_interruptible(affine_queue,
+					(p->eqci_owned != fq));
+		else
+			wait_event(affine_queue, (p->eqci_owned != fq));
+	}
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_enqueue_precommit);
+
+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
+			struct qm_mcc_initcgr *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+	u8 verb = QM_MCC_VERB_MODIFYCGR;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+	mcc = qm_mc_start(&p->p);
+	if (opts)
+		mcc->initcgr = *opts;
+	mcc->initcgr.we_mask = cpu_to_be16(mcc->initcgr.we_mask);
+	mcc->initcgr.cgr.wr_parm_g.word =
+		cpu_to_be32(mcc->initcgr.cgr.wr_parm_g.word);
+	mcc->initcgr.cgr.wr_parm_y.word =
+		cpu_to_be32(mcc->initcgr.cgr.wr_parm_y.word);
+	mcc->initcgr.cgr.wr_parm_r.word =
+		cpu_to_be32(mcc->initcgr.cgr.wr_parm_r.word);
+	mcc->initcgr.cgr.cscn_targ =  cpu_to_be32(mcc->initcgr.cgr.cscn_targ);
+	mcc->initcgr.cgr.__cs_thres = cpu_to_be16(mcc->initcgr.cgr.__cs_thres);
+
+	mcc->initcgr.cgid = cgr->cgrid;
+	if (flags & QMAN_CGR_FLAG_USE_INIT)
+		verb = QM_MCC_VERB_INITCGR;
+	qm_mc_commit(&p->p, verb);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == verb);
+	res = mcr->result;
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return (res == QM_MCR_RESULT_OK) ? 0 : -EIO;
+}
+EXPORT_SYMBOL(qman_modify_cgr);
+
+#define TARG_MASK(n) (0x80000000 >> (n->config->public_cfg.channel - \
+					QM_CHANNEL_SWPORTAL0))
+#define TARG_DCP_MASK(n) (0x80000000 >> (10 + n))
+#define PORTAL_IDX(n) (n->config->public_cfg.channel - QM_CHANNEL_SWPORTAL0)
+
+static u8 qman_cgr_cpus[__CGR_NUM];
+
+int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
+			struct qm_mcc_initcgr *opts)
+{
+	unsigned long irqflags __maybe_unused;
+	struct qm_mcr_querycgr cgr_state;
+	struct qm_mcc_initcgr local_opts;
+	int ret;
+	struct qman_portal *p;
+
+	/* We have to check that the provided CGRID is within the limits of the
+	 * data-structures, for obvious reasons. However we'll let h/w take
+	 * care of determining whether it's within the limits of what exists on
+	 * the SoC. */
+	if (cgr->cgrid >= __CGR_NUM)
+		return -EINVAL;
+
+	preempt_disable();
+	p = get_affine_portal();
+	qman_cgr_cpus[cgr->cgrid] = smp_processor_id();
+	preempt_enable();
+
+	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
+	cgr->chan = p->config->public_cfg.channel;
+	spin_lock_irqsave(&p->cgr_lock, irqflags);
+
+	/* if no opts specified, just add it to the list */
+	if (!opts)
+		goto add_list;
+
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret)
+		goto release_lock;
+	if (opts)
+		local_opts = *opts;
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		local_opts.cgr.cscn_targ_upd_ctrl =
+			QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p);
+	else
+		/* Overwrite TARG */
+		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
+							TARG_MASK(p);
+	local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
+
+	/* send init if flags indicate so */
+	if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
+		ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT, &local_opts);
+	else
+		ret = qman_modify_cgr(cgr, 0, &local_opts);
+	if (ret)
+		goto release_lock;
+add_list:
+	list_add(&cgr->node, &p->cgr_cbs);
+
+	/* Determine if newly added object requires its callback to be called */
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret) {
+		/* we can't go back, so proceed and return success, but screen
+		 * and wail to the log file */
+		pr_crit("CGR HW state partially modified\n");
+		ret = 0;
+		goto release_lock;
+	}
+	if (cgr->cb && cgr_state.cgr.cscn_en && qman_cgrs_get(&p->cgrs[1],
+							cgr->cgrid))
+		cgr->cb(p, cgr, 1);
+release_lock:
+	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_create_cgr);
+
+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
+					struct qm_mcc_initcgr *opts)
+{
+	unsigned long irqflags __maybe_unused;
+	struct qm_mcc_initcgr local_opts;
+	struct qm_mcr_querycgr cgr_state;
+	int ret;
+
+	/* We have to check that the provided CGRID is within the limits of the
+	 * data-structures, for obvious reasons. However we'll let h/w take
+	 * care of determining whether it's within the limits of what exists on
+	 * the SoC.
+	 */
+	if (cgr->cgrid >= __CGR_NUM)
+		return -EINVAL;
+
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret)
+		return ret;
+
+	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
+	if (opts)
+		local_opts = *opts;
+
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		local_opts.cgr.cscn_targ_upd_ctrl =
+				QM_CGR_TARG_UDP_CTRL_WRITE_BIT |
+				QM_CGR_TARG_UDP_CTRL_DCP | dcp_portal;
+	else
+		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ |
+					TARG_DCP_MASK(dcp_portal);
+	local_opts.we_mask |= QM_CGR_WE_CSCN_TARG;
+
+	/* send init if flags indicate so */
+	if (opts && (flags & QMAN_CGR_FLAG_USE_INIT))
+		ret = qman_modify_cgr(cgr, QMAN_CGR_FLAG_USE_INIT,
+							&local_opts);
+	else
+		ret = qman_modify_cgr(cgr, 0, &local_opts);
+
+	return ret;
+}
+EXPORT_SYMBOL(qman_create_cgr_to_dcp);
+
+int qman_delete_cgr(struct qman_cgr *cgr)
+{
+	unsigned long irqflags __maybe_unused;
+	struct qm_mcr_querycgr cgr_state;
+	struct qm_mcc_initcgr local_opts;
+	int ret = 0;
+	struct qman_cgr *i;
+	struct qman_portal *p = get_affine_portal();
+
+	if (cgr->chan != p->config->public_cfg.channel) {
+		pr_crit("Attempting to delete cgr from different portal "
+			"than it was create: create 0x%x, delete 0x%x\n",
+			cgr->chan, p->config->public_cfg.channel);
+		ret = -EINVAL;
+		goto put_portal;
+	}
+	memset(&local_opts, 0, sizeof(struct qm_mcc_initcgr));
+	spin_lock_irqsave(&p->cgr_lock, irqflags);
+	list_del(&cgr->node);
+	/*
+	 * If there are no other CGR objects for this CGRID in the list, update
+	 * CSCN_TARG accordingly
+	 */
+	list_for_each_entry(i, &p->cgr_cbs, node)
+		if ((i->cgrid == cgr->cgrid) && i->cb)
+			goto release_lock;
+	ret = qman_query_cgr(cgr, &cgr_state);
+	if (ret)  {
+		/* add back to the list */
+		list_add(&cgr->node, &p->cgr_cbs);
+		goto release_lock;
+	}
+	/* Overwrite TARG */
+	local_opts.we_mask = QM_CGR_WE_CSCN_TARG;
+	if ((qman_ip_rev & 0xFF00) >= QMAN_REV30)
+		local_opts.cgr.cscn_targ_upd_ctrl = PORTAL_IDX(p);
+	else
+		local_opts.cgr.cscn_targ = cgr_state.cgr.cscn_targ &
+							 ~(TARG_MASK(p));
+	ret = qman_modify_cgr(cgr, 0, &local_opts);
+	if (ret)
+		/* add back to the list */
+		list_add(&cgr->node, &p->cgr_cbs);
+release_lock:
+	spin_unlock_irqrestore(&p->cgr_lock, irqflags);
+put_portal:
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_delete_cgr);
+
+struct cgr_comp {
+	struct qman_cgr *cgr;
+	struct completion completion;
+};
+
+static int qman_delete_cgr_thread(void *p)
+{
+	struct cgr_comp *cgr_comp = (struct cgr_comp *)p;
+	int res;
+
+	res = qman_delete_cgr((struct qman_cgr *)cgr_comp->cgr);
+	complete(&cgr_comp->completion);
+
+	return res;
+}
+
+void qman_delete_cgr_safe(struct qman_cgr *cgr)
+{
+	struct task_struct *thread;
+	struct cgr_comp cgr_comp;
+
+	preempt_disable();
+	if (qman_cgr_cpus[cgr->cgrid] != smp_processor_id()) {
+		init_completion(&cgr_comp.completion);
+		cgr_comp.cgr = cgr;
+		thread = kthread_create(qman_delete_cgr_thread, &cgr_comp,
+					"cgr_del");
+
+		if (likely(!IS_ERR(thread))) {
+			kthread_bind(thread, qman_cgr_cpus[cgr->cgrid]);
+			wake_up_process(thread);
+			wait_for_completion(&cgr_comp.completion);
+			preempt_enable();
+			return;
+		}
+	}
+	qman_delete_cgr(cgr);
+	preempt_enable();
+}
+EXPORT_SYMBOL(qman_delete_cgr_safe);
+
+int qm_get_clock(u64 *clock_hz)
+{
+	if (!qman_clk) {
+		pr_warn("Qman clock speed is unknown\n");
+		return  -EINVAL;
+	}
+	*clock_hz = (u64)qman_clk;
+	return 0;
+}
+EXPORT_SYMBOL(qm_get_clock);
+
+int qm_set_clock(u64 clock_hz)
+{
+	if (qman_clk)
+		return -1;
+	qman_clk = (u32)clock_hz;
+		return 0;
+}
+EXPORT_SYMBOL(qm_set_clock);
+
+/* CEETM management command */
+static int qman_ceetm_configure_lfqmt(struct qm_mcc_ceetm_lfqmt_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->lfqmt_config = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+					 QM_CEETM_VERB_LFQMT_CONFIG);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE LFQMT failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+int qman_ceetm_query_lfqmt(int lfqid,
+			   struct qm_mcr_ceetm_lfqmt_query *lfqmt_query)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->lfqmt_query.lfqid = lfqid;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_LFQMT_QUERY);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_LFQMT_QUERY);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK)
+		*lfqmt_query = mcr->lfqmt_query;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY LFQMT failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_query_lfqmt);
+
+static int qman_ceetm_configure_cq(struct qm_mcc_ceetm_cq_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->cq_config = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	res = mcr->result;
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_CONFIG);
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE CQ failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid,
+				struct qm_mcr_ceetm_cq_query *cq_query)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->cq_query.cqid = cpu_to_be16(cqid);
+	mcc->cq_query.dcpid = dcpid;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_QUERY);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CQ_QUERY);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK) {
+		*cq_query = mcr->cq_query;
+		hw_cq_query_to_cpu(cq_query);
+	}
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY CQ failed\n");
+		return -EIO;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_query_cq);
+
+static int qman_ceetm_configure_dct(struct qm_mcc_ceetm_dct_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->dct_config = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_CONFIG);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE DCT failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+static int qman_ceetm_query_dct(struct qm_mcc_ceetm_dct_query *opts,
+			 struct qm_mcr_ceetm_dct_query *dct_query)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p = get_affine_portal();
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->dct_query = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_DCT_QUERY);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_DCT_QUERY);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY DCT failed\n");
+		return -EIO;
+	}
+
+	*dct_query = mcr->dct_query;
+	return 0;
+}
+
+static int qman_ceetm_configure_class_scheduler(
+			struct qm_mcc_ceetm_class_scheduler_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->csch_config = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+					QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE CLASS SCHEDULER failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+static int qman_ceetm_query_class_scheduler(struct qm_ceetm_channel *channel,
+			struct qm_mcr_ceetm_class_scheduler_query *query)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->csch_query.cqcid = channel->idx;
+	mcc->csch_query.dcpid = channel->dcp_idx;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+				QM_CEETM_VERB_CLASS_SCHEDULER_QUERY);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY CLASS SCHEDULER failed\n");
+		return -EIO;
+	}
+	*query = mcr->csch_query;
+	return 0;
+}
+
+static int qman_ceetm_configure_mapping_shaper_tcfc(
+		struct qm_mcc_ceetm_mapping_shaper_tcfc_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->mst_config = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+				QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE CHANNEL MAPPING failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+static int qman_ceetm_query_mapping_shaper_tcfc(
+		struct qm_mcc_ceetm_mapping_shaper_tcfc_query *opts,
+		struct qm_mcr_ceetm_mapping_shaper_tcfc_query *response)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->mst_query = *opts;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+				QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY);
+	res = mcr->result;
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY CHANNEL MAPPING failed\n");
+		return -EIO;
+	}
+
+	*response = mcr->mst_query;
+	return 0;
+}
+
+static int qman_ceetm_configure_ccgr(struct qm_mcc_ceetm_ccgr_config *opts)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->ccgr_config = *opts;
+
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_CONFIG);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_CONFIG);
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CONFIGURE CCGR failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+
+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query,
+				struct qm_mcr_ceetm_ccgr_query *response)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->ccgr_query.ccgrid = cpu_to_be16(ccgr_query->ccgrid);
+	mcc->ccgr_query.dcpid = ccgr_query->dcpid;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
+
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_CEETM_VERB_CCGR_QUERY);
+	res = mcr->result;
+	if (res == QM_MCR_RESULT_OK) {
+		*response = mcr->ccgr_query;
+		hw_ccgr_query_to_cpu(response);
+	}
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: QUERY CCGR failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_query_ccgr);
+
+static int qman_ceetm_cq_peek_pop_xsfdrread(struct qm_ceetm_cq *cq,
+			u8 command_type, u16 xsfdr,
+			struct qm_mcr_ceetm_cq_peek_pop_xsfdrread *cq_ppxr)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	switch (command_type) {
+	case 0:
+	case 1:
+		mcc->cq_ppxr.cqid = (cq->parent->idx << 4) | cq->idx;
+		break;
+	case 2:
+		mcc->cq_ppxr.xsfdr = xsfdr;
+		break;
+	default:
+		break;
+	}
+	mcc->cq_ppxr.ct = command_type;
+	mcc->cq_ppxr.dcpid = cq->parent->dcp_idx;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+				QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD);
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: CQ PEEK/POP/XSFDR READ failed\n");
+		return -EIO;
+	}
+	*cq_ppxr = mcr->cq_ppxr;
+	return 0;
+}
+
+static int qman_ceetm_query_statistics(u16 cid,
+			enum qm_dc_portal dcp_idx,
+			u16 command_type,
+			struct qm_mcr_ceetm_statistics_query *query_result)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->stats_query_write.cid = cid;
+	mcc->stats_query_write.dcpid = dcp_idx;
+	mcc->stats_query_write.ct = command_type;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
+
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+					 QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: STATISTICS QUERY failed\n");
+		return -EIO;
+	}
+	*query_result = mcr->stats_query;
+	return 0;
+}
+
+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx,
+				      u16 command_type, u64 frame_count,
+				      u64 byte_count)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	mcc->stats_query_write.cid = cid;
+	mcc->stats_query_write.dcpid = dcp_idx;
+	mcc->stats_query_write.ct = command_type;
+	mcc->stats_query_write.frm_cnt = frame_count;
+	mcc->stats_query_write.byte_cnt = byte_count;
+	qm_mc_commit(&p->p, QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
+
+	while (!(mcr = qm_mc_result(&p->p)))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+					 QM_CEETM_VERB_STATISTICS_QUERY_WRITE);
+
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+
+	res = mcr->result;
+	if (res != QM_MCR_RESULT_OK) {
+		pr_err("CEETM: STATISTICS WRITE failed\n");
+		return -EIO;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_query_write_statistics);
+
+int qman_ceetm_bps2tokenrate(u64 bps, struct qm_ceetm_rate *token_rate,
+							int rounding)
+{
+	u16 pres;
+	u64 temp;
+	u64 qman_freq;
+	int ret;
+
+	/* Read PRES from CEET_CFG_PRES register */
+	ret = qman_ceetm_get_prescaler(&pres);
+	if (ret)
+		return -EINVAL;
+
+	ret = qm_get_clock(&qman_freq);
+	if (ret)
+		return -EINVAL;
+
+	/* token-rate = bytes-per-second * update-reference-period
+	 *
+	 * Where token-rate is N/8192 for a integer N, and
+	 * update-reference-period is (2^22)/(PRES*QHz), where PRES
+	 * is the prescalar value and QHz is the QMan clock frequency.
+	 * So:
+	 *
+	 * token-rate = (byte-per-second*2^22)/PRES*QHZ)
+	 *
+	 * Converting to bits-per-second gives;
+	 *
+	 *	token-rate = (bps*2^19) / (PRES*QHZ)
+	 *	N = (bps*2^32) / (PRES*QHz)
+	 *
+	 * And to avoid 64-bit overflow if 'bps' is larger than 4Gbps
+	 * (yet minimise rounding error if 'bps' is small), we reorganise
+	 * the formula to use two 16-bit shifts rather than 1 32-bit shift.
+	 *      N = (((bps*2^16)/PRES)*2^16)/QHz
+	 */
+	temp = ROUNDING((bps << 16), pres, rounding);
+	temp = ROUNDING((temp << 16), qman_freq, rounding);
+	token_rate->whole = temp >> 13;
+	token_rate->fraction = temp & (((u64)1 << 13) - 1);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_bps2tokenrate);
+
+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate, u64 *bps,
+							int rounding)
+{
+	u16 pres;
+	u64 temp;
+	u64 qman_freq;
+	int ret;
+
+	/* Read PRES from CEET_CFG_PRES register */
+	ret = qman_ceetm_get_prescaler(&pres);
+	if (ret)
+		return -EINVAL;
+
+	ret = qm_get_clock(&qman_freq);
+	if (ret)
+		return -EINVAL;
+
+	/* bytes-per-second = token-rate / update-reference-period
+	 *
+	 * where "token-rate" is N/8192 for an integer N, and
+	 * "update-reference-period" is (2^22)/(PRES*QHz), where PRES is
+	 * the prescalar value and QHz is the QMan clock frequency. So;
+	 *
+	 * bytes-per-second = (N/8192) / (4194304/PRES*QHz)
+	 *                  = N*PRES*QHz / (4194304*8192)
+	 *                  = N*PRES*QHz / (2^35)
+	 *
+	 * Converting to bits-per-second gives;
+	 *
+	 *             bps = N*PRES*QHZ / (2^32)
+	 *
+	 * Note, the numerator has a maximum width of 72 bits! So to
+	 * avoid 64-bit overflow errors, we calculate PRES*QHZ (maximum
+	 * width 48 bits) divided by 2^9 (reducing to maximum 39 bits), before
+	 * multiplying by N (goes to maximum of 63 bits).
+	 *
+	 *             temp = PRES*QHZ / (2^16)
+	 *             kbps = temp*N / (2^16)
+	 */
+	temp = ROUNDING(qman_freq * pres, (u64)1 << 16 , rounding);
+	temp *= ((token_rate->whole << 13) + token_rate->fraction);
+	*bps = ROUNDING(temp, (u64)(1) << 16, rounding);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_tokenrate2bps);
+
+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp, enum qm_dc_portal dcp_idx,
+						unsigned int sp_idx)
+{
+	struct qm_ceetm_sp *p;
+
+	DPA_ASSERT((dcp_idx ==  qm_dc_portal_fman0) ||
+			(dcp_idx == qm_dc_portal_fman1));
+
+	if ((sp_idx < qman_ceetms[dcp_idx].sp_range[0]) ||
+		(sp_idx > (qman_ceetms[dcp_idx].sp_range[0] +
+		qman_ceetms[dcp_idx].sp_range[1]))) {
+		pr_err("Sub-portal index doesn't exist\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &qman_ceetms[dcp_idx].sub_portals, node) {
+		if ((p->idx == sp_idx) && (p->is_claimed == 0)) {
+			p->is_claimed = 1;
+			*sp = p;
+			return 0;
+		}
+	}
+	pr_err("The sub-portal#%d is not available!\n", sp_idx);
+	return -ENODEV;
+}
+EXPORT_SYMBOL(qman_ceetm_sp_claim);
+
+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp)
+{
+	struct qm_ceetm_sp *p;
+
+	if (sp->lni && sp->lni->is_claimed == 1) {
+		pr_err("The dependency of sub-portal has not been released!\n");
+		return -EBUSY;
+	}
+
+	list_for_each_entry(p, &qman_ceetms[sp->dcp_idx].sub_portals, node) {
+		if (p->idx == sp->idx) {
+			p->is_claimed = 0;
+			p->lni = NULL;
+		}
+	}
+	/* Disable CEETM mode of this sub-portal */
+	qman_sp_disable_ceetm_mode(sp->dcp_idx, sp->idx);
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_sp_release);
+
+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni, enum qm_dc_portal dcp_idx,
+							unsigned int lni_idx)
+{
+	struct qm_ceetm_lni *p;
+
+	if ((lni_idx < qman_ceetms[dcp_idx].lni_range[0]) ||
+		(lni_idx > (qman_ceetms[dcp_idx].lni_range[0] +
+		qman_ceetms[dcp_idx].lni_range[1]))) {
+		pr_err("The lni index is out of range\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &qman_ceetms[dcp_idx].lnis, node) {
+		if ((p->idx == lni_idx) && (p->is_claimed == 0)) {
+			*lni = p;
+			p->is_claimed = 1;
+			return 0;
+		}
+	}
+
+	pr_err("The LNI#%d is not available!\n", lni_idx);
+	return -EINVAL;
+}
+EXPORT_SYMBOL(qman_ceetm_lni_claim);
+
+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni)
+{
+	struct qm_ceetm_lni *p;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	if (!list_empty(&lni->channels)) {
+		pr_err("The LNI dependencies are not released!\n");
+		return -EBUSY;
+	}
+
+	list_for_each_entry(p, &qman_ceetms[lni->dcp_idx].lnis, node) {
+		if (p->idx == lni->idx) {
+			p->shaper_enable = 0;
+			p->shaper_couple = 0;
+			p->cr_token_rate.whole = 0;
+			p->cr_token_rate.fraction = 0;
+			p->er_token_rate.whole = 0;
+			p->er_token_rate.fraction = 0;
+			p->cr_token_bucket_limit = 0;
+			p->er_token_bucket_limit = 0;
+			p->is_claimed = 0;
+		}
+	}
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	config_opts.dcpid = lni->dcp_idx;
+	memset(&config_opts.shaper_config, 0,
+				sizeof(config_opts.shaper_config));
+	return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_release);
+
+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp, struct qm_ceetm_lni *lni)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx);
+	config_opts.dcpid = sp->dcp_idx;
+	config_opts.sp_mapping.map_lni_id = lni->idx;
+	sp->lni = lni;
+
+	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts))
+		return -EINVAL;
+
+	/* Enable CEETM mode for this sub-portal */
+	return qman_sp_enable_ceetm_mode(sp->dcp_idx, sp->idx);
+}
+EXPORT_SYMBOL(qman_ceetm_sp_set_lni);
+
+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp, unsigned int *lni_idx)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_SP_MAPPING | sp->idx);
+	query_opts.dcpid = sp->dcp_idx;
+	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
+		pr_err("Can't get SP <-> LNI mapping\n");
+		return -EINVAL;
+	}
+	*lni_idx = query_result.sp_mapping_query.map_lni_id;
+	sp->lni->idx = query_result.sp_mapping_query.map_lni_id;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_sp_get_lni);
+
+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled,
+								int oal)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	lni->shaper_enable = 1;
+	lni->shaper_couple = coupled;
+	lni->oal = oal;
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	config_opts.dcpid = lni->dcp_idx;
+	config_opts.shaper_config.cpl = coupled;
+	config_opts.shaper_config.oal = oal;
+	config_opts.shaper_config.crtcr = cpu_to_be24((lni->cr_token_rate.whole
+					<< 13) | lni->cr_token_rate.fraction);
+	config_opts.shaper_config.ertcr = cpu_to_be24((lni->er_token_rate.whole
+					<< 13) | lni->er_token_rate.fraction);
+	config_opts.shaper_config.crtbl =
+					cpu_to_be16(lni->cr_token_bucket_limit);
+	config_opts.shaper_config.ertbl =
+					cpu_to_be16(lni->er_token_bucket_limit);
+	config_opts.shaper_config.mps = 60;
+
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_enable_shaper);
+
+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	if (!lni->shaper_enable) {
+		pr_err("The shaper has been disabled\n");
+		return -EINVAL;
+	}
+
+	config_opts.cid = CEETM_COMMAND_LNI_SHAPER | lni->idx;
+	config_opts.dcpid = lni->dcp_idx;
+	config_opts.shaper_config.cpl = (lni->shaper_couple << 7) | lni->oal;
+	config_opts.shaper_config.crtbl = lni->cr_token_bucket_limit;
+	config_opts.shaper_config.ertbl = lni->er_token_bucket_limit;
+	/* Set CR/ER rate with all 1's to configure an infinite rate, thus
+	 * disable the shaping.
+	 */
+	config_opts.shaper_config.crtcr = 0xFFFFFF;
+	config_opts.shaper_config.ertcr = 0xFFFFFF;
+	config_opts.shaper_config.mps = 60;
+	lni->shaper_enable = 0;
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_disable_shaper);
+
+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni)
+{
+	return lni->shaper_enable;
+}
+EXPORT_SYMBOL(qman_ceetm_lni_is_shaper_enabled);
+
+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni,
+				const struct qm_ceetm_rate *token_rate,
+				u16 token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	lni->cr_token_rate.whole = token_rate->whole;
+	lni->cr_token_rate.fraction = token_rate->fraction;
+	lni->cr_token_bucket_limit = token_limit;
+	if (!lni->shaper_enable)
+		return 0;
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
+						   &query_result);
+	if (ret) {
+		pr_err("Fail to get current LNI shaper setting\n");
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	config_opts.dcpid = lni->dcp_idx;
+	config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole << 13)
+						      | (token_rate->fraction));
+	config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
+	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
+	config_opts.shaper_config.oal = query_result.shaper_query.oal;
+	config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr;
+	config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl;
+	config_opts.shaper_config.mps = query_result.shaper_query.mps;
+	return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate);
+
+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 bps,
+				       u16 token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
+	if (ret) {
+		pr_err("Can not convert bps to token rate\n");
+		return -EINVAL;
+	}
+
+	return qman_ceetm_lni_set_commit_rate(lni, &token_rate, token_limit);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_set_commit_rate_bps);
+
+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni,
+				struct qm_ceetm_rate *token_rate,
+				u16 *token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret) {
+		pr_err("The LNI CR rate or limit is not set\n");
+		return -EINVAL;
+	}
+	token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13;
+	token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) &
+				 0x1FFF;
+	*token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate);
+
+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 *bps, u16 *token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_lni_get_commit_rate(lni, &token_rate, token_limit);
+	if (ret) {
+		pr_err("The LNI CR rate or limit is not available\n");
+		return -EINVAL;
+	}
+
+	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_get_commit_rate_bps);
+
+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni,
+					const struct qm_ceetm_rate *token_rate,
+					u16 token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	lni->er_token_rate.whole = token_rate->whole;
+	lni->er_token_rate.fraction = token_rate->fraction;
+	lni->er_token_bucket_limit = token_limit;
+	if (!lni->shaper_enable)
+		return 0;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts,
+						   &query_result);
+	if (ret) {
+		pr_err("Fail to get current LNI shaper setting\n");
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	config_opts.dcpid = lni->dcp_idx;
+	config_opts.shaper_config.ertcr = cpu_to_be24(
+		(token_rate->whole << 13) | (token_rate->fraction));
+	config_opts.shaper_config.ertbl = cpu_to_be16(token_limit);
+	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
+	config_opts.shaper_config.oal = query_result.shaper_query.oal;
+	config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr;
+	config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl;
+	config_opts.shaper_config.mps = query_result.shaper_query.mps;
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate);
+
+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 bps,
+				       u16 token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
+	if (ret) {
+		pr_err("Can not convert bps to token rate\n");
+		return -EINVAL;
+	}
+	return qman_ceetm_lni_set_excess_rate(lni, &token_rate, token_limit);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_set_excess_rate_bps);
+
+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni,
+					struct qm_ceetm_rate *token_rate,
+					u16 *token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_LNI_SHAPER | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret) {
+		pr_err("The LNI ER rate or limit is not set\n");
+		return -EINVAL;
+	}
+	token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13;
+	token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) &
+								 0x1FFF;
+	*token_limit = be16_to_cpu(query_result.shaper_query.ertbl);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate);
+
+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 *bps, u16 *token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_lni_get_excess_rate(lni, &token_rate, token_limit);
+	if (ret) {
+		pr_err("The LNI ER rate or limit is not available\n");
+		return -EINVAL;
+	}
+
+	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_get_excess_rate_bps);
+
+#define QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(n) ((15 - n) * 4)
+#define QMAN_CEETM_LNITCFCC_ENABLE 0x8
+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni,
+				unsigned int cq_level,
+				int traffic_class)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	u64 lnitcfcc;
+
+	if ((cq_level > 15) | (traffic_class > 7)) {
+		pr_err("The CQ or traffic class id is out of range\n");
+		return -EINVAL;
+	}
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
+		pr_err("Fail to query tcfcc\n");
+		return -EINVAL;
+	}
+
+	lnitcfcc = be64_to_cpu(query_result.tcfc_query.lnitcfcc);
+	if (traffic_class == -1) {
+		/* disable tcfc for this CQ */
+		lnitcfcc &= ~((u64)QMAN_CEETM_LNITCFCC_ENABLE <<
+				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
+	} else {
+		lnitcfcc &= ~((u64)0xF <<
+				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
+		lnitcfcc |= ((u64)(QMAN_CEETM_LNITCFCC_ENABLE |
+				traffic_class)) <<
+				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level);
+	}
+	config_opts.tcfc_config.lnitcfcc = cpu_to_be64(lnitcfcc);
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
+	config_opts.dcpid = lni->dcp_idx;
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_lni_set_tcfcc);
+
+#define QMAN_CEETM_LNITCFCC_TC_MASK 0x7
+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni, unsigned int cq_level,
+						int *traffic_class)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+	u8 lnitcfcc;
+
+	if (cq_level > 15) {
+		pr_err("the CQ level is out of range\n");
+		return -EINVAL;
+	}
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_TCFC | lni->idx);
+	query_opts.dcpid = lni->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret)
+		return ret;
+	lnitcfcc = (u8)be64_to_cpu((query_result.tcfc_query.lnitcfcc) >>
+				QMAN_CEETM_LNITCFCC_CQ_LEVEL_SHIFT(cq_level));
+	if (lnitcfcc & QMAN_CEETM_LNITCFCC_ENABLE)
+		*traffic_class = lnitcfcc & QMAN_CEETM_LNITCFCC_TC_MASK;
+	else
+		*traffic_class = -1;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lni_get_tcfcc);
+
+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel,
+				struct qm_ceetm_lni *lni)
+{
+	struct qm_ceetm_channel *p;
+	u32 channel_idx;
+	int ret = 0;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	if (lni->dcp_idx == qm_dc_portal_fman0) {
+		ret = qman_alloc_ceetm0_channel(&channel_idx);
+	} else if (lni->dcp_idx == qm_dc_portal_fman1) {
+		ret = qman_alloc_ceetm1_channel(&channel_idx);
+	} else {
+		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
+			lni->dcp_idx);
+		return -EINVAL;
+	}
+
+	if (ret) {
+		pr_err("The is no channel available for LNI#%d\n", lni->idx);
+		return -ENODEV;
+	}
+
+	p = kzalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+	p->idx = channel_idx;
+	p->dcp_idx = lni->dcp_idx;
+	list_add_tail(&p->node, &lni->channels);
+	INIT_LIST_HEAD(&p->class_queues);
+	INIT_LIST_HEAD(&p->ccgs);
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
+						channel_idx);
+	config_opts.dcpid = lni->dcp_idx;
+	config_opts.channel_mapping.map_lni_id = lni->idx;
+	config_opts.channel_mapping.map_shaped = 0;
+	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
+		pr_err("Can't map channel#%d for LNI#%d\n",
+						channel_idx, lni->idx);
+		return -EINVAL;
+	}
+	*channel = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_claim);
+
+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	if (!list_empty(&channel->class_queues)) {
+		pr_err("CEETM channel#%d has class queue unreleased!\n",
+						channel->idx);
+		return -EBUSY;
+	}
+	if (!list_empty(&channel->ccgs)) {
+		pr_err("CEETM channel#%d has ccg unreleased!\n",
+						channel->idx);
+		return -EBUSY;
+	}
+
+	/* channel->dcp_idx corresponds to known fman validation */
+	if ((channel->dcp_idx != qm_dc_portal_fman0) &&
+	    (channel->dcp_idx != qm_dc_portal_fman1)) {
+		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
+			channel->dcp_idx);
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+				      channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	memset(&config_opts.shaper_config, 0,
+				sizeof(config_opts.shaper_config));
+	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
+		pr_err("Can't reset channel shapping parameters\n");
+		return -EINVAL;
+	}
+
+	if (channel->dcp_idx == qm_dc_portal_fman0) {
+		qman_release_ceetm0_channelid(channel->idx);
+	} else if (channel->dcp_idx == qm_dc_portal_fman1) {
+		qman_release_ceetm1_channelid(channel->idx);
+	} else {
+		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
+			channel->dcp_idx);
+		return -EINVAL;
+	}
+	list_del(&channel->node);
+	kfree(channel);
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_release);
+
+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel,
+								int coupled)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	if (channel->shaper_enable == 1) {
+		pr_err("This channel shaper has been enabled!\n");
+		return -EINVAL;
+	}
+
+	channel->shaper_enable = 1;
+	channel->shaper_couple = coupled;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
+						channel->idx);
+	query_opts.dcpid = (u8)channel->dcp_idx;
+
+	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
+		pr_err("Can't query channel mapping\n");
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
+						channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.channel_mapping.map_lni_id =
+				query_result.channel_mapping_query.map_lni_id;
+	config_opts.channel_mapping.map_shaped = 1;
+	if (qman_ceetm_configure_mapping_shaper_tcfc(&config_opts)) {
+		pr_err("Can't enable shaper for channel #%d\n",	channel->idx);
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+							channel->idx);
+	config_opts.shaper_config.cpl = coupled;
+	config_opts.shaper_config.crtcr = cpu_to_be24((channel->cr_token_rate.
+					whole << 13) |
+					channel->cr_token_rate.fraction);
+	config_opts.shaper_config.ertcr = cpu_to_be24((channel->er_token_rate.
+					whole << 13) |
+					channel->er_token_rate.fraction);
+	config_opts.shaper_config.crtbl =
+				cpu_to_be16(channel->cr_token_bucket_limit);
+	config_opts.shaper_config.ertbl =
+				cpu_to_be16(channel->er_token_bucket_limit);
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_enable_shaper);
+
+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
+						channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+
+	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
+		pr_err("Can't query channel mapping\n");
+		return -EINVAL;
+	}
+
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_MAPPING |
+						channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.channel_mapping.map_shaped = 0;
+	config_opts.channel_mapping.map_lni_id =
+				query_result.channel_mapping_query.map_lni_id;
+
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_disable_shaper);
+
+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+
+	query_opts.cid = CEETM_COMMAND_CHANNEL_MAPPING | channel->idx;
+	query_opts.dcpid = channel->dcp_idx;
+
+	if (qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result)) {
+		pr_err("Can't query channel mapping\n");
+		return -EINVAL;
+	}
+
+	return query_result.channel_mapping_query.map_shaped;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_is_shaper_enabled);
+
+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel,
+				const struct qm_ceetm_rate *token_rate,
+				u16 token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+						channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret) {
+		pr_err("Fail to get the current channel shaper setting\n");
+		return -EINVAL;
+	}
+
+	channel->cr_token_rate.whole = token_rate->whole;
+	channel->cr_token_rate.fraction = token_rate->fraction;
+	channel->cr_token_bucket_limit = token_limit;
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+							channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.shaper_config.crtcr = cpu_to_be24((token_rate->whole
+					<< 13) | (token_rate->fraction));
+	config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
+	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
+	config_opts.shaper_config.ertcr = query_result.shaper_query.ertcr;
+	config_opts.shaper_config.ertbl = query_result.shaper_query.ertbl;
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate);
+
+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 bps, u16 token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
+	if (ret) {
+		pr_err("Can not convert bps to token rate\n");
+		return -EINVAL;
+	}
+	return qman_ceetm_channel_set_commit_rate(channel, &token_rate,
+						  token_limit);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_commit_rate_bps);
+
+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel,
+				struct qm_ceetm_rate *token_rate,
+				u16 *token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+						channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret | !query_result.shaper_query.crtcr |
+			 !query_result.shaper_query.crtbl) {
+		pr_err("The channel commit rate or limit is not set\n");
+		return -EINVAL;
+	}
+	token_rate->whole = be24_to_cpu(query_result.shaper_query.crtcr) >> 13;
+	token_rate->fraction = be24_to_cpu(query_result.shaper_query.crtcr) &
+							0x1FFF;
+	*token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate);
+
+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 *bps, u16 *token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_channel_get_commit_rate(channel, &token_rate,
+						 token_limit);
+	if (ret) {
+		pr_err("The channel CR rate or limit is not available\n");
+		return -EINVAL;
+	}
+
+	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_commit_rate_bps);
+
+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel,
+					const struct qm_ceetm_rate *token_rate,
+					u16 token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+							channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret) {
+		pr_err("Fail to get the current channel shaper setting\n");
+		return -EINVAL;
+	}
+
+	channel->er_token_rate.whole = token_rate->whole;
+	channel->er_token_rate.fraction = token_rate->fraction;
+	channel->er_token_bucket_limit = token_limit;
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+							channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.shaper_config.ertcr = cpu_to_be24(
+			 (token_rate->whole << 13) | (token_rate->fraction));
+	config_opts.shaper_config.ertbl = cpu_to_be16(token_limit);
+	config_opts.shaper_config.cpl = query_result.shaper_query.cpl;
+	config_opts.shaper_config.crtcr = query_result.shaper_query.crtcr;
+	config_opts.shaper_config.crtbl = query_result.shaper_query.crtbl;
+	return qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate);
+
+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 bps, u16 token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_bps2tokenrate(bps, &token_rate, 0);
+	if (ret) {
+		pr_err("Can not convert bps to token rate\n");
+		return -EINVAL;
+	}
+	return qman_ceetm_channel_set_excess_rate(channel, &token_rate,
+						  token_limit);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_excess_rate_bps);
+
+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel,
+					struct qm_ceetm_rate *token_rate,
+					u16 *token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+							channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret | !query_result.shaper_query.ertcr |
+			 !query_result.shaper_query.ertbl) {
+		pr_err("The channel excess rate or limit is not set\n");
+		return -EINVAL;
+	}
+	token_rate->whole = be24_to_cpu(query_result.shaper_query.ertcr) >> 13;
+	token_rate->fraction = be24_to_cpu(query_result.shaper_query.ertcr) &
+								0x1FFF;
+	*token_limit = be16_to_cpu(query_result.shaper_query.ertbl);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate);
+
+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 *bps, u16 *token_limit)
+{
+	struct qm_ceetm_rate token_rate;
+	int ret;
+
+	ret = qman_ceetm_channel_get_excess_rate(channel, &token_rate,
+						 token_limit);
+	if (ret) {
+		pr_err("The channel ER rate or limit is not available\n");
+		return -EINVAL;
+	}
+
+	return qman_ceetm_tokenrate2bps(&token_rate, bps, 0);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_excess_rate_bps);
+
+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel,
+						u16 token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_config config_opts;
+
+	if (channel->shaper_enable) {
+		pr_err("This channel is a shaped one\n");
+		return -EINVAL;
+	}
+
+	channel->cr_token_bucket_limit = token_limit;
+	config_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+						channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.shaper_config.crtbl = cpu_to_be16(token_limit);
+	return	qman_ceetm_configure_mapping_shaper_tcfc(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_weight);
+
+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel,
+					u16 *token_limit)
+{
+	struct qm_mcc_ceetm_mapping_shaper_tcfc_query query_opts;
+	struct qm_mcr_ceetm_mapping_shaper_tcfc_query query_result;
+	int ret;
+
+	query_opts.cid = cpu_to_be16(CEETM_COMMAND_CHANNEL_SHAPER |
+						channel->idx);
+	query_opts.dcpid = channel->dcp_idx;
+	ret = qman_ceetm_query_mapping_shaper_tcfc(&query_opts, &query_result);
+	if (ret | !query_result.shaper_query.crtbl) {
+		pr_err("This unshaped channel's uFQ wight is unavailable\n");
+		return -EINVAL;
+	}
+	*token_limit = be16_to_cpu(query_result.shaper_query.crtbl);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_weight);
+
+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel, int group_b,
+				unsigned int prio_a, unsigned int prio_b)
+{
+	struct qm_mcc_ceetm_class_scheduler_config config_opts;
+	struct qm_mcr_ceetm_class_scheduler_query query_result;
+	int i;
+
+	if (prio_a > 7) {
+		pr_err("The priority of group A is out of range\n");
+		return -EINVAL;
+	}
+	if (group_b && (prio_b > 7)) {
+		pr_err("The priority of group B is out of range\n");
+		return -EINVAL;
+	}
+
+	if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
+		pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
+		return -EINVAL;
+	}
+
+	config_opts.cqcid = cpu_to_be16(channel->idx);
+	config_opts.dcpid = channel->dcp_idx;
+	config_opts.gpc_combine_flag = !group_b;
+	config_opts.gpc_prio_a = prio_a;
+	config_opts.gpc_prio_b = prio_b;
+
+	for (i = 0; i < 8; i++)
+		config_opts.w[i] = query_result.w[i];
+	config_opts.crem = query_result.crem;
+	config_opts.erem = query_result.erem;
+
+	return qman_ceetm_configure_class_scheduler(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_group);
+
+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel, int *group_b,
+				unsigned int *prio_a, unsigned int *prio_b)
+{
+	struct qm_mcr_ceetm_class_scheduler_query query_result;
+
+	if (qman_ceetm_query_class_scheduler(channel, &query_result)) {
+		pr_err("Can't query channel#%d's scheduler!\n", channel->idx);
+		return -EINVAL;
+	}
+	*group_b = !query_result.gpc_combine_flag;
+	*prio_a = query_result.gpc_prio_a;
+	*prio_b = query_result.gpc_prio_b;
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_get_group);
+
+#define GROUP_A_ELIGIBILITY_SET		(1 << 8)
+#define GROUP_B_ELIGIBILITY_SET		(1 << 9)
+#define CQ_ELIGIBILITY_SET(n)		(1 << (7 - n))
+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel
+				*channel, int group_b, int cre)
+{
+	struct qm_mcc_ceetm_class_scheduler_config csch_config;
+	struct qm_mcr_ceetm_class_scheduler_query csch_query;
+	int i;
+
+	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
+		pr_err("Cannot get the channel %d scheduler setting.\n",
+						channel->idx);
+		return -EINVAL;
+	}
+	csch_config.cqcid = cpu_to_be16(channel->idx);
+	csch_config.dcpid = channel->dcp_idx;
+	csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
+	csch_config.gpc_prio_a = csch_query.gpc_prio_a;
+	csch_config.gpc_prio_b = csch_query.gpc_prio_b;
+
+	for (i = 0; i < 8; i++)
+		csch_config.w[i] = csch_query.w[i];
+	csch_config.erem = csch_query.erem;
+	if (group_b)
+		csch_config.crem = (be16_to_cpu(csch_query.crem)
+					& ~GROUP_B_ELIGIBILITY_SET)
+					| (cre ? GROUP_B_ELIGIBILITY_SET : 0);
+	else
+		csch_config.crem = (be16_to_cpu(csch_query.crem)
+					& ~GROUP_A_ELIGIBILITY_SET)
+					| (cre ? GROUP_A_ELIGIBILITY_SET : 0);
+
+	csch_config.crem = cpu_to_be16(csch_config.crem);
+
+	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
+		pr_err("Cannot config channel %d's scheduler with "
+			"group_%c's cr eligibility\n", channel->idx,
+			group_b ? 'b' : 'a');
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_group_cr_eligibility);
+
+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel
+				*channel, int group_b, int ere)
+{
+	struct qm_mcc_ceetm_class_scheduler_config csch_config;
+	struct qm_mcr_ceetm_class_scheduler_query csch_query;
+	int i;
+
+	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
+		pr_err("Cannot get the channel %d scheduler setting.\n",
+						channel->idx);
+		return -EINVAL;
+	}
+	csch_config.cqcid = cpu_to_be16(channel->idx);
+	csch_config.dcpid = channel->dcp_idx;
+	csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
+	csch_config.gpc_prio_a = csch_query.gpc_prio_a;
+	csch_config.gpc_prio_b = csch_query.gpc_prio_b;
+
+	for (i = 0; i < 8; i++)
+		csch_config.w[i] = csch_query.w[i];
+	csch_config.crem = csch_query.crem;
+	if (group_b)
+		csch_config.erem = (be16_to_cpu(csch_query.erem)
+					& ~GROUP_B_ELIGIBILITY_SET)
+					| (ere ? GROUP_B_ELIGIBILITY_SET : 0);
+	else
+		csch_config.erem = (be16_to_cpu(csch_query.erem)
+					& ~GROUP_A_ELIGIBILITY_SET)
+					| (ere ? GROUP_A_ELIGIBILITY_SET : 0);
+
+	csch_config.erem = cpu_to_be16(csch_config.erem);
+
+	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
+		pr_err("Cannot config channel %d's scheduler with "
+			"group_%c's er eligibility\n", channel->idx,
+			group_b ? 'b' : 'a');
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_group_er_eligibility);
+
+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel,
+						unsigned int idx, int cre)
+{
+	struct qm_mcc_ceetm_class_scheduler_config csch_config;
+	struct qm_mcr_ceetm_class_scheduler_query csch_query;
+	int i;
+
+	if (idx > 7) {
+		pr_err("CQ index is out of range\n");
+		return -EINVAL;
+	}
+	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
+		pr_err("Cannot get the channel %d scheduler setting.\n",
+						channel->idx);
+		return -EINVAL;
+	}
+	csch_config.cqcid = cpu_to_be16(channel->idx);
+	csch_config.dcpid = channel->dcp_idx;
+	csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
+	csch_config.gpc_prio_a = csch_query.gpc_prio_a;
+	csch_config.gpc_prio_b = csch_query.gpc_prio_b;
+	for (i = 0; i < 8; i++)
+		csch_config.w[i] = csch_query.w[i];
+	csch_config.erem = csch_query.erem;
+	csch_config.crem = (be16_to_cpu(csch_query.crem)
+					& ~CQ_ELIGIBILITY_SET(idx)) |
+					(cre ? CQ_ELIGIBILITY_SET(idx) : 0);
+	csch_config.crem = cpu_to_be16(csch_config.crem);
+	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
+		pr_err("Cannot config channel scheduler to set "
+			"cr eligibility mask for CQ#%d\n", idx);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_cr_eligibility);
+
+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel,
+						unsigned int idx, int ere)
+{
+	struct qm_mcc_ceetm_class_scheduler_config csch_config;
+	struct qm_mcr_ceetm_class_scheduler_query csch_query;
+	int i;
+
+	if (idx > 7) {
+		pr_err("CQ index is out of range\n");
+		return -EINVAL;
+	}
+	if (qman_ceetm_query_class_scheduler(channel, &csch_query)) {
+		pr_err("Cannot get the channel %d scheduler setting.\n",
+						channel->idx);
+		return -EINVAL;
+	}
+	csch_config.cqcid = cpu_to_be16(channel->idx);
+	csch_config.dcpid = channel->dcp_idx;
+	csch_config.gpc_combine_flag = csch_query.gpc_combine_flag;
+	csch_config.gpc_prio_a = csch_query.gpc_prio_a;
+	csch_config.gpc_prio_b = csch_query.gpc_prio_b;
+	for (i = 0; i < 8; i++)
+		csch_config.w[i] = csch_query.w[i];
+	csch_config.crem = csch_query.crem;
+	csch_config.erem = (be16_to_cpu(csch_query.erem)
+					& ~CQ_ELIGIBILITY_SET(idx)) |
+					(ere ? CQ_ELIGIBILITY_SET(idx) : 0);
+	csch_config.erem = cpu_to_be16(csch_config.erem);
+	if (qman_ceetm_configure_class_scheduler(&csch_config)) {
+		pr_err("Cannot config channel scheduler to set "
+			"er eligibility mask for CQ#%d\n", idx);
+		return -EINVAL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_channel_set_cq_er_eligibility);
+
+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq,
+		struct qm_ceetm_channel *channel, unsigned int idx,
+		struct qm_ceetm_ccg *ccg)
+{
+	struct qm_ceetm_cq *p;
+	struct qm_mcc_ceetm_cq_config cq_config;
+
+	if (idx > 7) {
+		pr_err("The independent class queue id is out of range\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &channel->class_queues, node) {
+		if (p->idx == idx) {
+			pr_err("The CQ#%d has been claimed!\n", idx);
+			return -EINVAL;
+		}
+	}
+
+	p = kmalloc(sizeof(*p), GFP_KERNEL);
+	if (!p) {
+		pr_err("Can't allocate memory for CQ#%d!\n", idx);
+		return -ENOMEM;
+	}
+
+	list_add_tail(&p->node, &channel->class_queues);
+	p->idx = idx;
+	p->is_claimed = 1;
+	p->parent = channel;
+	INIT_LIST_HEAD(&p->bound_lfqids);
+
+	if (ccg) {
+		cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
+		cq_config.dcpid = channel->dcp_idx;
+		cq_config.ccgid = cpu_to_be16(ccg->idx);
+		if (qman_ceetm_configure_cq(&cq_config)) {
+			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
+						 idx, ccg->idx);
+			list_del(&p->node);
+			kfree(p);
+			return -EINVAL;
+		}
+	}
+
+	*cq = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cq_claim);
+
+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq,
+		struct qm_ceetm_channel *channel, unsigned int idx,
+		struct qm_ceetm_ccg *ccg)
+{
+	struct qm_ceetm_cq *p;
+	struct qm_mcc_ceetm_cq_config cq_config;
+
+	if ((idx < 8) || (idx > 15)) {
+		pr_err("This grouped class queue id is out of range\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &channel->class_queues, node) {
+		if (p->idx == idx) {
+			pr_err("The CQ#%d has been claimed!\n", idx);
+			return -EINVAL;
+		}
+	}
+
+	p = kmalloc(sizeof(*p), GFP_KERNEL);
+	if (!p) {
+		pr_err("Can't allocate memory for CQ#%d!\n", idx);
+		return -ENOMEM;
+	}
+
+	list_add_tail(&p->node, &channel->class_queues);
+	p->idx = idx;
+	p->is_claimed = 1;
+	p->parent = channel;
+	INIT_LIST_HEAD(&p->bound_lfqids);
+
+	if (ccg) {
+		cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
+		cq_config.dcpid = channel->dcp_idx;
+		cq_config.ccgid = cpu_to_be16(ccg->idx);
+		if (qman_ceetm_configure_cq(&cq_config)) {
+			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
+						 idx, ccg->idx);
+			list_del(&p->node);
+			kfree(p);
+			return -EINVAL;
+		}
+	}
+	*cq = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cq_claim_A);
+
+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq,
+		struct qm_ceetm_channel *channel, unsigned int idx,
+		struct qm_ceetm_ccg *ccg)
+{
+	struct qm_ceetm_cq *p;
+	struct qm_mcc_ceetm_cq_config cq_config;
+
+	if ((idx < 12) || (idx > 15)) {
+		pr_err("This grouped class queue id is out of range\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &channel->class_queues, node) {
+		if (p->idx == idx) {
+			pr_err("The CQ#%d has been claimed!\n", idx);
+			return -EINVAL;
+		}
+	}
+
+	p = kmalloc(sizeof(*p), GFP_KERNEL);
+	if (!p) {
+		pr_err("Can't allocate memory for CQ#%d!\n", idx);
+		return -ENOMEM;
+	}
+
+	list_add_tail(&p->node, &channel->class_queues);
+	p->idx = idx;
+	p->is_claimed = 1;
+	p->parent = channel;
+	INIT_LIST_HEAD(&p->bound_lfqids);
+
+	if (ccg) {
+		cq_config.cqid = cpu_to_be16((channel->idx << 4) | idx);
+		cq_config.dcpid = channel->dcp_idx;
+		cq_config.ccgid = cpu_to_be16(ccg->idx);
+		if (qman_ceetm_configure_cq(&cq_config)) {
+			pr_err("Can't configure the CQ#%d with CCGRID#%d\n",
+					 idx, ccg->idx);
+			list_del(&p->node);
+			kfree(p);
+			return -EINVAL;
+		}
+	}
+	*cq = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cq_claim_B);
+
+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq)
+{
+	if (!list_empty(&cq->bound_lfqids)) {
+		pr_err("The CQ#%d has unreleased LFQID\n", cq->idx);
+		return -EBUSY;
+	}
+	list_del(&cq->node);
+	qman_ceetm_drain_cq(cq);
+	kfree(cq);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cq_release);
+
+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq,
+				struct qm_ceetm_weight_code *weight_code)
+{
+	struct qm_mcc_ceetm_class_scheduler_config config_opts;
+	struct qm_mcr_ceetm_class_scheduler_query query_result;
+	int i;
+
+	if (cq->idx < 8) {
+		pr_err("Can not set weight for ungrouped class queue\n");
+		return -EINVAL;
+	}
+
+	if (qman_ceetm_query_class_scheduler(cq->parent, &query_result)) {
+		pr_err("Can't query channel#%d's scheduler!\n",
+						cq->parent->idx);
+		return -EINVAL;
+	}
+
+	config_opts.cqcid = cpu_to_be16(cq->parent->idx);
+	config_opts.dcpid = cq->parent->dcp_idx;
+	config_opts.crem = query_result.crem;
+	config_opts.erem = query_result.erem;
+	config_opts.gpc_combine_flag = query_result.gpc_combine_flag;
+	config_opts.gpc_prio_a = query_result.gpc_prio_a;
+	config_opts.gpc_prio_b = query_result.gpc_prio_b;
+
+	for (i = 0; i < 8; i++)
+		config_opts.w[i] = query_result.w[i];
+	config_opts.w[cq->idx - 8] = ((weight_code->y << 3) |
+						(weight_code->x & 0x7));
+	return qman_ceetm_configure_class_scheduler(&config_opts);
+}
+EXPORT_SYMBOL(qman_ceetm_set_queue_weight);
+
+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq,
+				struct qm_ceetm_weight_code *weight_code)
+{
+	struct qm_mcr_ceetm_class_scheduler_query query_result;
+
+	if (cq->idx < 8) {
+		pr_err("Can not get weight for ungrouped class queue\n");
+		return -EINVAL;
+	}
+
+	if (qman_ceetm_query_class_scheduler(cq->parent,
+						&query_result)) {
+		pr_err("Can't get the weight code for CQ#%d!\n", cq->idx);
+		return -EINVAL;
+	}
+	weight_code->y = query_result.w[cq->idx - 8] >> 3;
+	weight_code->x = query_result.w[cq->idx - 8] & 0x7;
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_get_queue_weight);
+
+/* The WBFS code is represent as {x,y}, the effect wieght can be calculated as:
+ *	effective weight = 2^x / (1 - (y/64))
+ *			 = 2^(x+6) / (64 - y)
+ */
+static void reduce_fraction(u32 *n, u32 *d)
+{
+	u32 factor = 2;
+	u32 lesser = (*n < *d) ? *n : *d;
+	/* If factor exceeds the square-root of the lesser of *n and *d,
+	 * then there's no point continuing. Proof: if there was a factor
+	 * bigger than the square root, that would imply there exists
+	 * another factor smaller than the square-root with which it
+	 * multiplies to give 'lesser' - but that's a contradiction
+	 * because the other factor would have already been found and
+	 * divided out.
+	 */
+	while ((factor * factor) <= lesser) {
+		/* If 'factor' is a factor of *n and *d, divide them both
+		 * by 'factor' as many times as possible.
+		 */
+		while (!(*n % factor) && !(*d % factor)) {
+			*n /= factor;
+			*d /= factor;
+			lesser /= factor;
+		}
+		if (factor == 2)
+			factor = 3;
+		else
+			factor += 2;
+	}
+}
+
+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code,
+				u32 *numerator,
+				u32 *denominator)
+{
+	*numerator = (u32) 1 << (weight_code->x + 6);
+	*denominator = 64 - weight_code->y;
+	reduce_fraction(numerator, denominator);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_wbfs2ratio);
+
+/* For a given x, the weight is between 2^x (inclusive) and 2^(x+1) (exclusive).
+ * So find 'x' by range, and then estimate 'y' using:
+ *		64 - y	= 2^(x + 6) / weight
+ *			= 2^(x + 6) / (n/d)
+ *			= d * 2^(x+6) / n
+ *		y = 64 - (d * 2^(x+6) / n)
+ */
+int qman_ceetm_ratio2wbfs(u32 numerator,
+				u32 denominator,
+				struct qm_ceetm_weight_code *weight_code,
+				int rounding)
+{
+	unsigned int y, x = 0;
+	/* search incrementing 'x' until:
+	 * weight < 2^(x+1)
+	 *    n/d < 2^(x+1)
+	 *	n < d * 2^(x+1)
+	 */
+	while ((x < 8) && (numerator >= (denominator << (x + 1))))
+		x++;
+	if (x >= 8)
+		return -ERANGE;
+	/* because of the subtraction, use '-rounding' */
+	y = 64 - ROUNDING(denominator << (x + 6), numerator, -rounding);
+	if (y >= 32)
+		return -ERANGE;
+	weight_code->x = x;
+	weight_code->y = y;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_ratio2wbfs);
+
+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio)
+{
+	struct qm_ceetm_weight_code weight_code;
+
+	if (qman_ceetm_ratio2wbfs(ratio, 100, &weight_code, 0)) {
+		pr_err("Cannot get wbfs code for cq %x\n", cq->idx);
+		return -EINVAL;
+	}
+	return qman_ceetm_set_queue_weight(cq, &weight_code);
+}
+EXPORT_SYMBOL(qman_ceetm_set_queue_weight_in_ratio);
+
+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio)
+{
+	struct qm_ceetm_weight_code weight_code;
+	u32 n, d;
+
+	if (qman_ceetm_get_queue_weight(cq, &weight_code)) {
+		pr_err("Cannot query the weight code for cq%x\n", cq->idx);
+		return -EINVAL;
+	}
+
+	if (qman_ceetm_wbfs2ratio(&weight_code, &n, &d)) {
+		pr_err("Cannot get the ratio with wbfs code\n");
+		return -EINVAL;
+	}
+
+	*ratio = (n * (u32)100) / d;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_get_queue_weight_in_ratio);
+
+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags,
+					u64 *frame_count, u64 *byte_count)
+{
+	struct qm_mcr_ceetm_statistics_query result;
+	u16 cid, command_type;
+	enum qm_dc_portal dcp_idx;
+	int ret;
+
+	cid = cpu_to_be16((cq->parent->idx << 4) | cq->idx);
+	dcp_idx = cq->parent->dcp_idx;
+	if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
+		command_type = CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS;
+	else
+		command_type = CEETM_QUERY_DEQUEUE_STATISTICS;
+
+	ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
+	if (ret) {
+		pr_err("Can't query the statistics of CQ#%d!\n", cq->idx);
+		return -EINVAL;
+	}
+
+	*frame_count = be40_to_cpu(result.frm_cnt);
+	*byte_count = be48_to_cpu(result.byte_cnt);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cq_get_dequeue_statistics);
+
+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq)
+{
+	struct qm_mcr_ceetm_cq_peek_pop_xsfdrread ppxr;
+	int ret;
+
+	do {
+		ret = qman_ceetm_cq_peek_pop_xsfdrread(cq, 1, 0, &ppxr);
+		if (ret) {
+			pr_err("Failed to pop frame from CQ\n");
+			return -EINVAL;
+		}
+	} while (!(ppxr.stat & 0x2));
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_drain_cq);
+
+#define CEETM_LFQMT_LFQID_MSB 0xF00000
+#define CEETM_LFQMT_LFQID_LSB 0x000FFF
+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq,
+					struct qm_ceetm_cq *cq)
+{
+	struct qm_ceetm_lfq *p;
+	u32 lfqid;
+	int ret = 0;
+	struct qm_mcc_ceetm_lfqmt_config lfqmt_config;
+
+	if (cq->parent->dcp_idx == qm_dc_portal_fman0) {
+		ret = qman_alloc_ceetm0_lfqid(&lfqid);
+	} else if (cq->parent->dcp_idx == qm_dc_portal_fman1) {
+		ret = qman_alloc_ceetm1_lfqid(&lfqid);
+	} else {
+		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
+			cq->parent->dcp_idx);
+		return -EINVAL;
+	}
+
+	if (ret) {
+		pr_err("There is no lfqid avalaible for CQ#%d!\n", cq->idx);
+		return -ENODEV;
+	}
+	p = kmalloc(sizeof(*p), GFP_KERNEL);
+	if (!p)
+		return -ENOMEM;
+	p->idx = lfqid;
+	p->dctidx = (u16)(lfqid & CEETM_LFQMT_LFQID_LSB);
+	p->parent = cq->parent;
+	list_add_tail(&p->node, &cq->bound_lfqids);
+
+	lfqmt_config.lfqid = cpu_to_be24(CEETM_LFQMT_LFQID_MSB |
+				(cq->parent->dcp_idx << 16) |
+				(lfqid & CEETM_LFQMT_LFQID_LSB));
+	lfqmt_config.cqid = cpu_to_be16((cq->parent->idx << 4) | (cq->idx));
+	lfqmt_config.dctidx = cpu_to_be16(p->dctidx);
+	if (qman_ceetm_configure_lfqmt(&lfqmt_config)) {
+		pr_err("Can't configure LFQMT for LFQID#%d @ CQ#%d\n",
+				lfqid, cq->idx);
+		list_del(&p->node);
+		kfree(p);
+		return -EINVAL;
+	}
+	*lfq = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lfq_claim);
+
+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq)
+{
+	if (lfq->parent->dcp_idx == qm_dc_portal_fman0) {
+		qman_release_ceetm0_lfqid(lfq->idx);
+	} else if (lfq->parent->dcp_idx == qm_dc_portal_fman1) {
+		qman_release_ceetm1_lfqid(lfq->idx);
+	} else {
+		pr_err("dcp_idx %u does not correspond to a known fman in this driver\n",
+			lfq->parent->dcp_idx);
+		return -EINVAL;
+	}
+	list_del(&lfq->node);
+	kfree(lfq);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lfq_release);
+
+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq, u64 context_a,
+							u32 context_b)
+{
+	struct qm_mcc_ceetm_dct_config dct_config;
+	lfq->context_a = context_a;
+	lfq->context_b = context_b;
+	dct_config.dctidx = cpu_to_be16(lfq->dctidx);
+	dct_config.dcpid = lfq->parent->dcp_idx;
+	dct_config.context_b = cpu_to_be32(context_b);
+	dct_config.context_a = cpu_to_be64(context_a);
+	return qman_ceetm_configure_dct(&dct_config);
+}
+EXPORT_SYMBOL(qman_ceetm_lfq_set_context);
+
+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq, u64 *context_a,
+							u32 *context_b)
+{
+	struct qm_mcc_ceetm_dct_query dct_query;
+	struct qm_mcr_ceetm_dct_query query_result;
+
+	dct_query.dctidx = cpu_to_be16(lfq->dctidx);
+	dct_query.dcpid = lfq->parent->dcp_idx;
+	if (qman_ceetm_query_dct(&dct_query, &query_result)) {
+		pr_err("Can't query LFQID#%d's context!\n", lfq->idx);
+		return -EINVAL;
+	}
+	*context_a = be64_to_cpu(query_result.context_a);
+	*context_b = be32_to_cpu(query_result.context_b);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_lfq_get_context);
+
+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq)
+{
+	spin_lock_init(&fq->fqlock);
+	fq->fqid = lfq->idx;
+	fq->flags = QMAN_FQ_FLAG_NO_MODIFY;
+	if (lfq->ern)
+		fq->cb.ern = lfq->ern;
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+	if (unlikely(find_empty_fq_table_entry(&fq->key, fq)))
+		return -ENOMEM;
+#endif
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_create_fq);
+
+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg,
+				struct qm_ceetm_channel *channel,
+				unsigned int idx,
+				void (*cscn)(struct qm_ceetm_ccg *,
+					void *cb_ctx,
+					int congested),
+				void *cb_ctx)
+{
+	struct qm_ceetm_ccg *p;
+
+	if (idx > 15) {
+		pr_err("The given ccg index is out of range\n");
+		return -EINVAL;
+	}
+
+	list_for_each_entry(p, &channel->ccgs, node) {
+		if (p->idx == idx) {
+			pr_err("The CCG#%d has been claimed\n", idx);
+			return -EINVAL;
+		}
+	}
+
+	p = kmalloc(sizeof(*p), GFP_KERNEL);
+	if (!p) {
+		pr_err("Can't allocate memory for CCG#%d!\n", idx);
+		return -ENOMEM;
+	}
+
+	list_add_tail(&p->node, &channel->ccgs);
+
+	p->idx = idx;
+	p->parent = channel;
+	p->cb = cscn;
+	p->cb_ctx = cb_ctx;
+	INIT_LIST_HEAD(&p->cb_node);
+
+	*ccg = p;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_ccg_claim);
+
+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg)
+{
+	unsigned long irqflags __maybe_unused;
+	struct qm_mcc_ceetm_ccgr_config config_opts;
+	int ret = 0;
+	struct qman_portal *p = get_affine_portal();
+
+	memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
+	spin_lock_irqsave(&p->ccgr_lock, irqflags);
+	if (!list_empty(&ccg->cb_node))
+		list_del(&ccg->cb_node);
+	config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
+				(ccg->parent->idx << 4) | ccg->idx);
+	config_opts.dcpid = ccg->parent->dcp_idx;
+	config_opts.we_mask = cpu_to_be16(QM_CCGR_WE_CSCN_TUPD);
+	config_opts.cm_config.cscn_tupd = cpu_to_be16(PORTAL_IDX(p));
+	ret = qman_ceetm_configure_ccgr(&config_opts);
+	spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
+	put_affine_portal();
+
+	list_del(&ccg->node);
+	kfree(ccg);
+	return ret;
+}
+EXPORT_SYMBOL(qman_ceetm_ccg_release);
+
+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg, u16 we_mask,
+				const struct qm_ceetm_ccg_params *params)
+{
+	struct qm_mcc_ceetm_ccgr_config config_opts;
+	unsigned long irqflags __maybe_unused;
+	int ret;
+	struct qman_portal *p;
+
+	if (((ccg->parent->idx << 4) | ccg->idx) >= (2 * __CGR_NUM))
+		return -EINVAL;
+
+	p = get_affine_portal();
+
+	memset(&config_opts, 0, sizeof(struct qm_mcc_ceetm_ccgr_config));
+	spin_lock_irqsave(&p->ccgr_lock, irqflags);
+
+	config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
+				(ccg->parent->idx << 4) | ccg->idx);
+	config_opts.dcpid = ccg->parent->dcp_idx;
+	config_opts.we_mask = we_mask;
+	if (we_mask & QM_CCGR_WE_CSCN_EN) {
+		config_opts.we_mask |= QM_CCGR_WE_CSCN_TUPD;
+		config_opts.cm_config.cscn_tupd = cpu_to_be16(
+			QM_CGR_TARG_UDP_CTRL_WRITE_BIT | PORTAL_IDX(p));
+	}
+	config_opts.we_mask = cpu_to_be16(config_opts.we_mask);
+	config_opts.cm_config.ctl_wr_en_g = params->wr_en_g;
+	config_opts.cm_config.ctl_wr_en_y = params->wr_en_y;
+	config_opts.cm_config.ctl_wr_en_r = params->wr_en_r;
+	config_opts.cm_config.ctl_td_en = params->td_en;
+	config_opts.cm_config.ctl_td_mode = params->td_mode;
+	config_opts.cm_config.ctl_cscn_en = params->cscn_en;
+	config_opts.cm_config.ctl_mode = params->mode;
+	config_opts.cm_config.oal = params->oal;
+	config_opts.cm_config.cs_thres.hword =
+					cpu_to_be16(params->cs_thres_in.hword);
+	config_opts.cm_config.cs_thres_x.hword =
+					cpu_to_be16(params->cs_thres_out.hword);
+	config_opts.cm_config.td_thres.hword =
+					cpu_to_be16(params->td_thres.hword);
+	config_opts.cm_config.wr_parm_g.word =
+					cpu_to_be32(params->wr_parm_g.word);
+	config_opts.cm_config.wr_parm_y.word =
+					cpu_to_be32(params->wr_parm_y.word);
+	config_opts.cm_config.wr_parm_r.word =
+					cpu_to_be32(params->wr_parm_r.word);
+	ret = qman_ceetm_configure_ccgr(&config_opts);
+	if (ret) {
+		pr_err("Configure CCGR CM failed!\n");
+		goto release_lock;
+	}
+
+	if (we_mask & QM_CCGR_WE_CSCN_EN)
+		if (list_empty(&ccg->cb_node))
+			list_add(&ccg->cb_node,
+				&p->ccgr_cbs[ccg->parent->dcp_idx]);
+release_lock:
+	spin_unlock_irqrestore(&p->ccgr_lock, irqflags);
+	put_affine_portal();
+	return ret;
+}
+EXPORT_SYMBOL(qman_ceetm_ccg_set);
+
+#define CEETM_CCGR_CTL_MASK 0x01
+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg,
+				struct qm_ceetm_ccg_params *params)
+{
+	struct qm_mcc_ceetm_ccgr_query query_opts;
+	struct qm_mcr_ceetm_ccgr_query query_result;
+
+	query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
+				(ccg->parent->idx << 4) | ccg->idx);
+	query_opts.dcpid = ccg->parent->dcp_idx;
+
+	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
+		pr_err("Can't query CCGR#%d\n", ccg->idx);
+		return -EINVAL;
+	}
+
+	params->wr_parm_r.word =
+			be32_to_cpu(query_result.cm_query.wr_parm_r.word);
+	params->wr_parm_y.word =
+			be32_to_cpu(query_result.cm_query.wr_parm_y.word);
+	params->wr_parm_g.word =
+		       be32_to_cpu(query_result.cm_query.wr_parm_g.word);
+	params->td_thres.hword =
+			be16_to_cpu(query_result.cm_query.td_thres.hword);
+	params->cs_thres_out.hword =
+			be16_to_cpu(query_result.cm_query.cs_thres_x.hword);
+	params->cs_thres_in.hword =
+			be16_to_cpu(query_result.cm_query.cs_thres.hword);
+	params->oal = query_result.cm_query.oal;
+	params->wr_en_g = query_result.cm_query.ctl_wr_en_g;
+	params->wr_en_y = query_result.cm_query.ctl_wr_en_y;
+	params->wr_en_r = query_result.cm_query.ctl_wr_en_r;
+	params->td_en = query_result.cm_query.ctl_td_en;
+	params->td_mode = query_result.cm_query.ctl_td_mode;
+	params->cscn_en = query_result.cm_query.ctl_cscn_en;
+	params->mode = query_result.cm_query.ctl_mode;
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_ccg_get);
+
+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags,
+					u64 *frame_count, u64 *byte_count)
+{
+	struct qm_mcr_ceetm_statistics_query result;
+	u16 cid, command_type;
+	enum qm_dc_portal dcp_idx;
+	int ret;
+
+	cid = cpu_to_be16((ccg->parent->idx << 4) | ccg->idx);
+	dcp_idx = ccg->parent->dcp_idx;
+	if (flags == QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER)
+		command_type = CEETM_QUERY_REJECT_CLEAR_STATISTICS;
+	else
+		command_type = CEETM_QUERY_REJECT_STATISTICS;
+
+	ret = qman_ceetm_query_statistics(cid, dcp_idx, command_type, &result);
+	if (ret) {
+		pr_err("Can't query the statistics of CCG#%d!\n", ccg->idx);
+		return -EINVAL;
+	}
+
+	*frame_count = be40_to_cpu(result.frm_cnt);
+	*byte_count = be48_to_cpu(result.byte_cnt);
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_ccg_get_reject_statistics);
+
+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg,
+					u16 swp_idx,
+					unsigned int *cscn_enabled)
+{
+	struct qm_mcc_ceetm_ccgr_query query_opts;
+	struct qm_mcr_ceetm_ccgr_query query_result;
+	int i;
+
+	DPA_ASSERT(swp_idx < 127);
+	query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
+				(ccg->parent->idx << 4) | ccg->idx);
+	query_opts.dcpid = ccg->parent->dcp_idx;
+
+	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
+		pr_err("Can't query CCGR#%d\n", ccg->idx);
+		return -EINVAL;
+	}
+
+	i = swp_idx / 32;
+	i = 3 - i;
+	*cscn_enabled = be32_to_cpu(query_result.cm_query.cscn_targ_swp[i]) >>
+							(31 - swp_idx % 32);
+
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cscn_swp_get);
+
+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg,
+				u16 dcp_idx,
+				u8 vcgid,
+				unsigned int cscn_enabled,
+				u16 we_mask,
+				const struct qm_ceetm_ccg_params *params)
+{
+	struct qm_mcc_ceetm_ccgr_config config_opts;
+	int ret;
+
+	config_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_CONFIGURE |
+				(ccg->parent->idx << 4) | ccg->idx);
+	config_opts.dcpid = ccg->parent->dcp_idx;
+	config_opts.we_mask = cpu_to_be16(we_mask | QM_CCGR_WE_CSCN_TUPD |
+						 QM_CCGR_WE_CDV);
+	config_opts.cm_config.cdv = vcgid;
+	config_opts.cm_config.cscn_tupd = cpu_to_be16((cscn_enabled << 15) |
+					QM_CGR_TARG_UDP_CTRL_DCP | dcp_idx);
+	config_opts.cm_config.ctl_wr_en_g = params->wr_en_g;
+	config_opts.cm_config.ctl_wr_en_y = params->wr_en_y;
+	config_opts.cm_config.ctl_wr_en_r = params->wr_en_r;
+	config_opts.cm_config.ctl_td_en = params->td_en;
+	config_opts.cm_config.ctl_td_mode = params->td_mode;
+	config_opts.cm_config.ctl_cscn_en = params->cscn_en;
+	config_opts.cm_config.ctl_mode = params->mode;
+	config_opts.cm_config.cs_thres.hword =
+					cpu_to_be16(params->cs_thres_in.hword);
+	config_opts.cm_config.cs_thres_x.hword =
+					cpu_to_be16(params->cs_thres_out.hword);
+	config_opts.cm_config.td_thres.hword =
+					cpu_to_be16(params->td_thres.hword);
+	config_opts.cm_config.wr_parm_g.word =
+					cpu_to_be32(params->wr_parm_g.word);
+	config_opts.cm_config.wr_parm_y.word =
+					cpu_to_be32(params->wr_parm_y.word);
+	config_opts.cm_config.wr_parm_r.word =
+					cpu_to_be32(params->wr_parm_r.word);
+
+	ret = qman_ceetm_configure_ccgr(&config_opts);
+	if (ret) {
+		pr_err("Configure CSCN_TARG_DCP failed!\n");
+		return -EINVAL;
+	}
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_set);
+
+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg,
+				u16 dcp_idx,
+				u8 *vcgid,
+				unsigned int *cscn_enabled)
+{
+	struct qm_mcc_ceetm_ccgr_query query_opts;
+	struct qm_mcr_ceetm_ccgr_query query_result;
+
+	query_opts.ccgrid = cpu_to_be16(CEETM_CCGR_CM_QUERY |
+				(ccg->parent->idx << 4) | ccg->idx);
+	query_opts.dcpid = ccg->parent->dcp_idx;
+
+	if (qman_ceetm_query_ccgr(&query_opts, &query_result)) {
+		pr_err("Can't query CCGR#%d\n", ccg->idx);
+		return -EINVAL;
+	}
+
+	*vcgid = query_result.cm_query.cdv;
+	*cscn_enabled = (be16_to_cpu(query_result.cm_query.cscn_targ_dcp >>
+				     dcp_idx)) & 0x1;
+	return 0;
+}
+EXPORT_SYMBOL(qman_ceetm_cscn_dcp_get);
+
+int qman_ceetm_querycongestion(struct __qm_mcr_querycongestion *ccg_state,
+							unsigned int dcp_idx)
+{
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	u8 res;
+	int i, j;
+
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+
+	mcc = qm_mc_start(&p->p);
+	for (i = 0; i < 2; i++) {
+		mcc->ccgr_query.ccgrid =
+				cpu_to_be16(CEETM_QUERY_CONGESTION_STATE | i);
+		mcc->ccgr_query.dcpid = dcp_idx;
+		qm_mc_commit(&p->p, QM_CEETM_VERB_CCGR_QUERY);
+
+		while (!(mcr = qm_mc_result(&p->p)))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+						QM_CEETM_VERB_CCGR_QUERY);
+		res = mcr->result;
+		if (res == QM_MCR_RESULT_OK) {
+			for (j = 0; j < 8; j++)
+				mcr->ccgr_query.congestion_state.state.
+				__state[j] =
+					be32_to_cpu(mcr->ccgr_query.
+					congestion_state.state.__state[j]);
+
+			*(ccg_state + i) =
+				mcr->ccgr_query.congestion_state.state;
+		} else {
+			pr_err("QUERY CEETM CONGESTION STATE failed\n");
+			return -EIO;
+		}
+	}
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return 0;
+}
+
+int qman_set_wpm(int wpm_enable)
+{
+	return qm_set_wpm(wpm_enable);
+}
+EXPORT_SYMBOL(qman_set_wpm);
+
+int qman_get_wpm(int *wpm_enable)
+{
+	return qm_get_wpm(wpm_enable);
+}
+EXPORT_SYMBOL(qman_get_wpm);
+
+int qman_shutdown_fq(u32 fqid)
+{
+	struct qman_portal *p;
+	unsigned long irqflags __maybe_unused;
+	int ret;
+	struct qm_portal *low_p;
+	p = get_affine_portal();
+	PORTAL_IRQ_LOCK(p, irqflags);
+	low_p = &p->p;
+	ret = qm_shutdown_fq(&low_p, 1, fqid);
+	PORTAL_IRQ_UNLOCK(p, irqflags);
+	put_affine_portal();
+	return ret;
+}
+
+const struct qm_portal_config *qman_get_qm_portal_config(
+						struct qman_portal *portal)
+{
+	return portal->sharing_redirect ? NULL : portal->config;
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_low.h
@@ -0,0 +1,1407 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_private.h"
+
+/***************************/
+/* Portal register assists */
+/***************************/
+
+/* Cache-inhibited register offsets */
+#if defined(CONFIG_PPC32) || defined(CONFIG_PPC64)
+
+#define QM_REG_EQCR_PI_CINH	0x0000
+#define QM_REG_EQCR_CI_CINH	0x0004
+#define QM_REG_EQCR_ITR		0x0008
+#define QM_REG_DQRR_PI_CINH	0x0040
+#define QM_REG_DQRR_CI_CINH	0x0044
+#define QM_REG_DQRR_ITR		0x0048
+#define QM_REG_DQRR_DCAP	0x0050
+#define QM_REG_DQRR_SDQCR	0x0054
+#define QM_REG_DQRR_VDQCR	0x0058
+#define QM_REG_DQRR_PDQCR	0x005c
+#define QM_REG_MR_PI_CINH	0x0080
+#define QM_REG_MR_CI_CINH	0x0084
+#define QM_REG_MR_ITR		0x0088
+#define QM_REG_CFG		0x0100
+#define QM_REG_ISR		0x0e00
+#define QM_REG_IIR              0x0e0c
+#define QM_REG_ITPR		0x0e14
+
+/* Cache-enabled register offsets */
+#define QM_CL_EQCR		0x0000
+#define QM_CL_DQRR		0x1000
+#define QM_CL_MR		0x2000
+#define QM_CL_EQCR_PI_CENA	0x3000
+#define QM_CL_EQCR_CI_CENA	0x3100
+#define QM_CL_DQRR_PI_CENA	0x3200
+#define QM_CL_DQRR_CI_CENA	0x3300
+#define QM_CL_MR_PI_CENA	0x3400
+#define QM_CL_MR_CI_CENA	0x3500
+#define QM_CL_CR		0x3800
+#define QM_CL_RR0		0x3900
+#define QM_CL_RR1		0x3940
+
+#endif
+
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+
+#define QM_REG_EQCR_PI_CINH	0x3000
+#define QM_REG_EQCR_CI_CINH	0x3040
+#define QM_REG_EQCR_ITR		0x3080
+#define QM_REG_DQRR_PI_CINH	0x3100
+#define QM_REG_DQRR_CI_CINH	0x3140
+#define QM_REG_DQRR_ITR		0x3180
+#define QM_REG_DQRR_DCAP	0x31C0
+#define QM_REG_DQRR_SDQCR	0x3200
+#define QM_REG_DQRR_VDQCR	0x3240
+#define QM_REG_DQRR_PDQCR	0x3280
+#define QM_REG_MR_PI_CINH	0x3300
+#define QM_REG_MR_CI_CINH	0x3340
+#define QM_REG_MR_ITR		0x3380
+#define QM_REG_CFG		0x3500
+#define QM_REG_ISR		0x3600
+#define QM_REG_IIR              0x36C0
+#define QM_REG_ITPR		0x3740
+
+/* Cache-enabled register offsets */
+#define QM_CL_EQCR		0x0000
+#define QM_CL_DQRR		0x1000
+#define QM_CL_MR		0x2000
+#define QM_CL_EQCR_PI_CENA	0x3000
+#define QM_CL_EQCR_CI_CENA	0x3040
+#define QM_CL_DQRR_PI_CENA	0x3100
+#define QM_CL_DQRR_CI_CENA	0x3140
+#define QM_CL_MR_PI_CENA	0x3300
+#define QM_CL_MR_CI_CENA	0x3340
+#define QM_CL_CR		0x3800
+#define QM_CL_RR0		0x3900
+#define QM_CL_RR1		0x3940
+
+#endif
+
+
+/* BTW, the drivers (and h/w programming model) already obtain the required
+ * synchronisation for portal accesses via lwsync(), hwsync(), and
+ * data-dependencies. Use of barrier()s or other order-preserving primitives
+ * simply degrade performance. Hence the use of the __raw_*() interfaces, which
+ * simply ensure that the compiler treats the portal registers as volatile (ie.
+ * non-coherent). */
+
+/* Cache-inhibited register access. */
+#define __qm_in(qm, o)		be32_to_cpu(__raw_readl((qm)->addr_ci  + (o)))
+#define __qm_out(qm, o, val)	__raw_writel((cpu_to_be32(val)), \
+					     (qm)->addr_ci + (o));
+#define qm_in(reg)		__qm_in(&portal->addr, QM_REG_##reg)
+#define qm_out(reg, val)	__qm_out(&portal->addr, QM_REG_##reg, val)
+
+/* Cache-enabled (index) register access */
+#define __qm_cl_touch_ro(qm, o) dcbt_ro((qm)->addr_ce + (o))
+#define __qm_cl_touch_rw(qm, o) dcbt_rw((qm)->addr_ce + (o))
+#define __qm_cl_in(qm, o)	be32_to_cpu(__raw_readl((qm)->addr_ce + (o)))
+#define __qm_cl_out(qm, o, val) \
+	do { \
+		u32 *__tmpclout = (qm)->addr_ce + (o); \
+		__raw_writel(cpu_to_be32(val), __tmpclout); \
+		dcbf(__tmpclout); \
+	} while (0)
+#define __qm_cl_invalidate(qm, o) dcbi((qm)->addr_ce + (o))
+#define qm_cl_touch_ro(reg) __qm_cl_touch_ro(&portal->addr, QM_CL_##reg##_CENA)
+#define qm_cl_touch_rw(reg) __qm_cl_touch_rw(&portal->addr, QM_CL_##reg##_CENA)
+#define qm_cl_in(reg)	    __qm_cl_in(&portal->addr, QM_CL_##reg##_CENA)
+#define qm_cl_out(reg, val) __qm_cl_out(&portal->addr, QM_CL_##reg##_CENA, val)
+#define qm_cl_invalidate(reg)\
+	__qm_cl_invalidate(&portal->addr, QM_CL_##reg##_CENA)
+
+/* Cache-enabled ring access */
+#define qm_cl(base, idx)	((void *)base + ((idx) << 6))
+
+/* Cyclic helper for rings. FIXME: once we are able to do fine-grain perf
+ * analysis, look at using the "extra" bit in the ring index registers to avoid
+ * cyclic issues. */
+static inline u8 qm_cyc_diff(u8 ringsize, u8 first, u8 last)
+{
+	/* 'first' is included, 'last' is excluded */
+	if (first <= last)
+		return last - first;
+	return ringsize + last - first;
+}
+
+/* Portal modes.
+ *   Enum types;
+ *     pmode == production mode
+ *     cmode == consumption mode,
+ *     dmode == h/w dequeue mode.
+ *   Enum values use 3 letter codes. First letter matches the portal mode,
+ *   remaining two letters indicate;
+ *     ci == cache-inhibited portal register
+ *     ce == cache-enabled portal register
+ *     vb == in-band valid-bit (cache-enabled)
+ *     dc == DCA (Discrete Consumption Acknowledgement), DQRR-only
+ *   As for "enum qm_dqrr_dmode", it should be self-explanatory.
+ */
+enum qm_eqcr_pmode {		/* matches QCSP_CFG::EPM */
+	qm_eqcr_pci = 0,	/* PI index, cache-inhibited */
+	qm_eqcr_pce = 1,	/* PI index, cache-enabled */
+	qm_eqcr_pvb = 2		/* valid-bit */
+};
+enum qm_dqrr_dmode {		/* matches QCSP_CFG::DP */
+	qm_dqrr_dpush = 0,	/* SDQCR  + VDQCR */
+	qm_dqrr_dpull = 1	/* PDQCR */
+};
+enum qm_dqrr_pmode {		/* s/w-only */
+	qm_dqrr_pci,		/* reads DQRR_PI_CINH */
+	qm_dqrr_pce,		/* reads DQRR_PI_CENA */
+	qm_dqrr_pvb		/* reads valid-bit */
+};
+enum qm_dqrr_cmode {		/* matches QCSP_CFG::DCM */
+	qm_dqrr_cci = 0,	/* CI index, cache-inhibited */
+	qm_dqrr_cce = 1,	/* CI index, cache-enabled */
+	qm_dqrr_cdc = 2		/* Discrete Consumption Acknowledgement */
+};
+enum qm_mr_pmode {		/* s/w-only */
+	qm_mr_pci,		/* reads MR_PI_CINH */
+	qm_mr_pce,		/* reads MR_PI_CENA */
+	qm_mr_pvb		/* reads valid-bit */
+};
+enum qm_mr_cmode {		/* matches QCSP_CFG::MM */
+	qm_mr_cci = 0,		/* CI index, cache-inhibited */
+	qm_mr_cce = 1		/* CI index, cache-enabled */
+};
+
+
+/* ------------------------- */
+/* --- Portal structures --- */
+
+#define QM_EQCR_SIZE		8
+#define QM_DQRR_SIZE		16
+#define QM_MR_SIZE		8
+
+struct qm_eqcr {
+	struct qm_eqcr_entry *ring, *cursor;
+	u8 ci, available, ithresh, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	u32 busy;
+	enum qm_eqcr_pmode pmode;
+#endif
+};
+
+struct qm_dqrr {
+	const struct qm_dqrr_entry *ring, *cursor;
+	u8 pi, ci, fill, ithresh, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	enum qm_dqrr_dmode dmode;
+	enum qm_dqrr_pmode pmode;
+	enum qm_dqrr_cmode cmode;
+#endif
+};
+
+struct qm_mr {
+	const struct qm_mr_entry *ring, *cursor;
+	u8 pi, ci, fill, ithresh, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	enum qm_mr_pmode pmode;
+	enum qm_mr_cmode cmode;
+#endif
+};
+
+struct qm_mc {
+	struct qm_mc_command *cr;
+	struct qm_mc_result *rr;
+	u8 rridx, vbit;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	enum {
+		/* Can be _mc_start()ed */
+		qman_mc_idle,
+		/* Can be _mc_commit()ed or _mc_abort()ed */
+		qman_mc_user,
+		/* Can only be _mc_retry()ed */
+		qman_mc_hw
+	} state;
+#endif
+};
+
+#define QM_PORTAL_ALIGNMENT ____cacheline_aligned
+
+struct qm_addr {
+	void __iomem *addr_ce;	/* cache-enabled */
+	void __iomem *addr_ci;	/* cache-inhibited */
+};
+
+struct qm_portal {
+	/* In the non-CONFIG_FSL_DPA_CHECKING case, the following stuff up to
+	 * and including 'mc' fits within a cacheline (yay!). The 'config' part
+	 * is setup-only, so isn't a cause for a concern. In other words, don't
+	 * rearrange this structure on a whim, there be dragons ... */
+	struct qm_addr addr;
+	struct qm_eqcr eqcr;
+	struct qm_dqrr dqrr;
+	struct qm_mr mr;
+	struct qm_mc mc;
+} QM_PORTAL_ALIGNMENT;
+
+
+/* ---------------- */
+/* --- EQCR API --- */
+
+/* Bit-wise logic to wrap a ring pointer by clearing the "carry bit" */
+#define EQCR_CARRYCLEAR(p) \
+	(void *)((unsigned long)(p) & (~(unsigned long)(QM_EQCR_SIZE << 6)))
+
+/* Bit-wise logic to convert a ring pointer to a ring index */
+static inline u8 EQCR_PTR2IDX(struct qm_eqcr_entry *e)
+{
+	return ((uintptr_t)e >> 6) & (QM_EQCR_SIZE - 1);
+}
+
+/* Increment the 'cursor' ring pointer, taking 'vbit' into account */
+static inline void EQCR_INC(struct qm_eqcr *eqcr)
+{
+	/* NB: this is odd-looking, but experiments show that it generates fast
+	 * code with essentially no branching overheads. We increment to the
+	 * next EQCR pointer and handle overflow and 'vbit'. */
+	struct qm_eqcr_entry *partial = eqcr->cursor + 1;
+	eqcr->cursor = EQCR_CARRYCLEAR(partial);
+	if (partial != eqcr->cursor)
+		eqcr->vbit ^= QM_EQCR_VERB_VBIT;
+}
+
+static inline int qm_eqcr_init(struct qm_portal *portal,
+				enum qm_eqcr_pmode pmode,
+				unsigned int eq_stash_thresh,
+				int eq_stash_prio)
+{
+	/* This use of 'register', as well as all other occurrences, is because
+	 * it has been observed to generate much faster code with gcc than is
+	 * otherwise the case. */
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	u32 cfg;
+	u8 pi;
+
+	eqcr->ring = portal->addr.addr_ce + QM_CL_EQCR;
+	eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
+	qm_cl_invalidate(EQCR_CI);
+	pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
+	eqcr->cursor = eqcr->ring + pi;
+	eqcr->vbit = (qm_in(EQCR_PI_CINH) & QM_EQCR_SIZE) ?
+			QM_EQCR_VERB_VBIT : 0;
+	eqcr->available = QM_EQCR_SIZE - 1 -
+			qm_cyc_diff(QM_EQCR_SIZE, eqcr->ci, pi);
+	eqcr->ithresh = qm_in(EQCR_ITR);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 0;
+	eqcr->pmode = pmode;
+#endif
+	cfg = (qm_in(CFG) & 0x00ffffff) |
+		(eq_stash_thresh << 28) | /* QCSP_CFG: EST */
+		(eq_stash_prio << 26)	| /* QCSP_CFG: EP */
+		((pmode & 0x3) << 24);	/* QCSP_CFG::EPM */
+	qm_out(CFG, cfg);
+	return 0;
+}
+
+static inline unsigned int qm_eqcr_get_ci_stashing(struct qm_portal *portal)
+{
+	return (qm_in(CFG) >> 28) & 0x7;
+}
+
+static inline void qm_eqcr_finish(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 pi, ci;
+	u32 cfg;
+
+	/*
+	 * Disable EQCI stashing because the QMan only
+	 * presents the value it previously stashed to
+	 * maintain coherency.  Setting the stash threshold
+	 * to 1 then 0 ensures that QMan has resyncronized
+	 * its internal copy so that the portal is clean
+	 * when it is reinitialized in the future
+	 */
+	cfg = (qm_in(CFG) & 0x0fffffff) |
+		(1 << 28); /* QCSP_CFG: EST */
+	qm_out(CFG, cfg);
+	cfg &= 0x0fffffff; /* stash threshold = 0 */
+	qm_out(CFG, cfg);
+
+	pi = qm_in(EQCR_PI_CINH) & (QM_EQCR_SIZE - 1);
+	ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
+
+	/* Refresh EQCR CI cache value */
+	qm_cl_invalidate(EQCR_CI);
+	eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
+
+	DPA_ASSERT(!eqcr->busy);
+	if (pi != EQCR_PTR2IDX(eqcr->cursor))
+		pr_crit("losing uncommited EQCR entries\n");
+	if (ci != eqcr->ci)
+		pr_crit("missing existing EQCR completions\n");
+	if (eqcr->ci != EQCR_PTR2IDX(eqcr->cursor))
+		pr_crit("EQCR destroyed unquiesced\n");
+}
+
+static inline struct qm_eqcr_entry *qm_eqcr_start_no_stash(struct qm_portal
+								 *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	DPA_ASSERT(!eqcr->busy);
+	if (!eqcr->available)
+		return NULL;
+
+
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 1;
+#endif
+	dcbz_64(eqcr->cursor);
+	return eqcr->cursor;
+}
+
+static inline struct qm_eqcr_entry *qm_eqcr_start_stash(struct qm_portal
+								*portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 diff, old_ci;
+
+	DPA_ASSERT(!eqcr->busy);
+	if (!eqcr->available) {
+		old_ci = eqcr->ci;
+		eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
+		diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+		eqcr->available += diff;
+		if (!diff)
+			return NULL;
+	}
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 1;
+#endif
+	dcbz_64(eqcr->cursor);
+	return eqcr->cursor;
+}
+
+static inline void qm_eqcr_abort(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
+	DPA_ASSERT(eqcr->busy);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 0;
+#endif
+}
+
+static inline struct qm_eqcr_entry *qm_eqcr_pend_and_next(
+					struct qm_portal *portal, u8 myverb)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	DPA_ASSERT(eqcr->busy);
+	DPA_ASSERT(eqcr->pmode != qm_eqcr_pvb);
+	if (eqcr->available == 1)
+		return NULL;
+	eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
+	dcbf(eqcr->cursor);
+	EQCR_INC(eqcr);
+	eqcr->available--;
+	dcbz_64(eqcr->cursor);
+	return eqcr->cursor;
+}
+
+#define EQCR_COMMIT_CHECKS(eqcr) \
+do { \
+	DPA_ASSERT(eqcr->busy); \
+	DPA_ASSERT(eqcr->cursor->orp == (eqcr->cursor->orp & 0x00ffffff)); \
+	DPA_ASSERT(eqcr->cursor->fqid == (eqcr->cursor->fqid & 0x00ffffff)); \
+} while (0)
+
+static inline void qm_eqcr_pci_commit(struct qm_portal *portal, u8 myverb)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	EQCR_COMMIT_CHECKS(eqcr);
+	DPA_ASSERT(eqcr->pmode == qm_eqcr_pci);
+	eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
+	EQCR_INC(eqcr);
+	eqcr->available--;
+	dcbf(eqcr->cursor);
+	hwsync();
+	qm_out(EQCR_PI_CINH, EQCR_PTR2IDX(eqcr->cursor));
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 0;
+#endif
+}
+
+static inline void qm_eqcr_pce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
+	DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
+	qm_cl_invalidate(EQCR_PI);
+	qm_cl_touch_rw(EQCR_PI);
+}
+
+static inline void qm_eqcr_pce_commit(struct qm_portal *portal, u8 myverb)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	EQCR_COMMIT_CHECKS(eqcr);
+	DPA_ASSERT(eqcr->pmode == qm_eqcr_pce);
+	eqcr->cursor->__dont_write_directly__verb = myverb | eqcr->vbit;
+	EQCR_INC(eqcr);
+	eqcr->available--;
+	dcbf(eqcr->cursor);
+	lwsync();
+	qm_cl_out(EQCR_PI, EQCR_PTR2IDX(eqcr->cursor));
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 0;
+#endif
+}
+
+static inline void qm_eqcr_pvb_commit(struct qm_portal *portal, u8 myverb)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	struct qm_eqcr_entry *eqcursor;
+	EQCR_COMMIT_CHECKS(eqcr);
+	DPA_ASSERT(eqcr->pmode == qm_eqcr_pvb);
+	lwsync();
+	eqcursor = eqcr->cursor;
+	eqcursor->__dont_write_directly__verb = myverb | eqcr->vbit;
+	dcbf(eqcursor);
+	EQCR_INC(eqcr);
+	eqcr->available--;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	eqcr->busy = 0;
+#endif
+}
+
+static inline u8 qm_eqcr_cci_update(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 diff, old_ci = eqcr->ci;
+	eqcr->ci = qm_in(EQCR_CI_CINH) & (QM_EQCR_SIZE - 1);
+	diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+	eqcr->available += diff;
+	return diff;
+}
+
+static inline void qm_eqcr_cce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_eqcr *eqcr = &portal->eqcr;
+	qm_cl_touch_ro(EQCR_CI);
+}
+
+static inline u8 qm_eqcr_cce_update(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	u8 diff, old_ci = eqcr->ci;
+	eqcr->ci = qm_cl_in(EQCR_CI) & (QM_EQCR_SIZE - 1);
+	qm_cl_invalidate(EQCR_CI);
+	diff = qm_cyc_diff(QM_EQCR_SIZE, old_ci, eqcr->ci);
+	eqcr->available += diff;
+	return diff;
+}
+
+static inline u8 qm_eqcr_get_ithresh(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	return eqcr->ithresh;
+}
+
+static inline void qm_eqcr_set_ithresh(struct qm_portal *portal, u8 ithresh)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	eqcr->ithresh = ithresh;
+	qm_out(EQCR_ITR, ithresh);
+}
+
+static inline u8 qm_eqcr_get_avail(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	return eqcr->available;
+}
+
+static inline u8 qm_eqcr_get_fill(struct qm_portal *portal)
+{
+	register struct qm_eqcr *eqcr = &portal->eqcr;
+	return QM_EQCR_SIZE - 1 - eqcr->available;
+}
+
+
+/* ---------------- */
+/* --- DQRR API --- */
+
+/* FIXME: many possible improvements;
+ * - look at changing the API to use pointer rather than index parameters now
+ *   that 'cursor' is a pointer,
+ * - consider moving other parameters to pointer if it could help (ci)
+ */
+
+#define DQRR_CARRYCLEAR(p) \
+	(void *)((unsigned long)(p) & (~(unsigned long)(QM_DQRR_SIZE << 6)))
+
+static inline u8 DQRR_PTR2IDX(const struct qm_dqrr_entry *e)
+{
+	return ((uintptr_t)e >> 6) & (QM_DQRR_SIZE - 1);
+}
+
+static inline const struct qm_dqrr_entry *DQRR_INC(
+						const struct qm_dqrr_entry *e)
+{
+	return DQRR_CARRYCLEAR(e + 1);
+}
+
+static inline void qm_dqrr_set_maxfill(struct qm_portal *portal, u8 mf)
+{
+	qm_out(CFG, (qm_in(CFG) & 0xff0fffff) |
+		((mf & (QM_DQRR_SIZE - 1)) << 20));
+}
+
+static inline void qm_dqrr_cci_consume(struct qm_portal *portal, u8 num)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
+	dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
+	qm_out(DQRR_CI_CINH, dqrr->ci);
+}
+
+static inline void qm_dqrr_cce_consume(struct qm_portal *portal, u8 num)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+	dqrr->ci = (dqrr->ci + num) & (QM_DQRR_SIZE - 1);
+	qm_cl_out(DQRR_CI, dqrr->ci);
+}
+
+static inline void qm_dqrr_cdc_consume_n(struct qm_portal *portal, u16 bitmask)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	qm_out(DQRR_DCAP, (1 << 8) |		/* DQRR_DCAP::S */
+		((u32)bitmask << 16));		/* DQRR_DCAP::DCAP_CI */
+	dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
+	dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
+}
+
+static inline int qm_dqrr_init(struct qm_portal *portal,
+				const struct qm_portal_config *config,
+				enum qm_dqrr_dmode dmode,
+				__maybe_unused enum qm_dqrr_pmode pmode,
+				enum qm_dqrr_cmode cmode, u8 max_fill)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	u32 cfg;
+
+	/* Make sure the DQRR will be idle when we enable */
+	qm_out(DQRR_SDQCR, 0);
+	qm_out(DQRR_VDQCR, 0);
+	qm_out(DQRR_PDQCR, 0);
+	dqrr->ring = portal->addr.addr_ce + QM_CL_DQRR;
+	dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
+	dqrr->ci = qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
+	dqrr->cursor = dqrr->ring + dqrr->ci;
+	dqrr->fill = qm_cyc_diff(QM_DQRR_SIZE, dqrr->ci, dqrr->pi);
+	dqrr->vbit = (qm_in(DQRR_PI_CINH) & QM_DQRR_SIZE) ?
+			QM_DQRR_VERB_VBIT : 0;
+	dqrr->ithresh = qm_in(DQRR_ITR);
+
+	/* Free up pending DQRR entries if any as per current DCM */
+	if (dqrr->fill) {
+		enum qm_dqrr_cmode dcm = (qm_in(CFG) >> 16) & 3;
+
+#ifdef CONFIG_FSL_DPA_CHECKING
+		dqrr->cmode = dcm;
+#endif
+		switch (dcm) {
+		case qm_dqrr_cci:
+			qm_dqrr_cci_consume(portal, dqrr->fill);
+			break;
+		case qm_dqrr_cce:
+			qm_dqrr_cce_consume(portal, dqrr->fill);
+			break;
+		case qm_dqrr_cdc:
+			qm_dqrr_cdc_consume_n(portal, (QM_DQRR_SIZE - 1));
+			break;
+		default:
+			DPA_ASSERT(0);
+		}
+	}
+
+#ifdef CONFIG_FSL_DPA_CHECKING
+	dqrr->dmode = dmode;
+	dqrr->pmode = pmode;
+	dqrr->cmode = cmode;
+#endif
+	/* Invalidate every ring entry before beginning */
+	for (cfg = 0; cfg < QM_DQRR_SIZE; cfg++)
+		dcbi(qm_cl(dqrr->ring, cfg));
+	cfg = (qm_in(CFG) & 0xff000f00) |
+		((max_fill & (QM_DQRR_SIZE - 1)) << 20) | /* DQRR_MF */
+		((dmode & 1) << 18) |			/* DP */
+		((cmode & 3) << 16) |			/* DCM */
+		0xa0 |					/* RE+SE */
+		(0 ? 0x40 : 0) |			/* Ignore RP */
+		(0 ? 0x10 : 0);				/* Ignore SP */
+	qm_out(CFG, cfg);
+	qm_dqrr_set_maxfill(portal, max_fill);
+	return 0;
+}
+
+static inline void qm_dqrr_finish(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if ((dqrr->cmode != qm_dqrr_cdc) &&
+			(dqrr->ci != DQRR_PTR2IDX(dqrr->cursor)))
+		pr_crit("Ignoring completed DQRR entries\n");
+#endif
+}
+
+static inline const struct qm_dqrr_entry *qm_dqrr_current(
+						struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	if (!dqrr->fill)
+		return NULL;
+	return dqrr->cursor;
+}
+
+static inline u8 qm_dqrr_cursor(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	return DQRR_PTR2IDX(dqrr->cursor);
+}
+
+static inline u8 qm_dqrr_next(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->fill);
+	dqrr->cursor = DQRR_INC(dqrr->cursor);
+	return --dqrr->fill;
+}
+
+static inline u8 qm_dqrr_pci_update(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	u8 diff, old_pi = dqrr->pi;
+	DPA_ASSERT(dqrr->pmode == qm_dqrr_pci);
+	dqrr->pi = qm_in(DQRR_PI_CINH) & (QM_DQRR_SIZE - 1);
+	diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
+	dqrr->fill += diff;
+	return diff;
+}
+
+static inline void qm_dqrr_pce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
+	qm_cl_invalidate(DQRR_PI);
+	qm_cl_touch_ro(DQRR_PI);
+}
+
+static inline u8 qm_dqrr_pce_update(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	u8 diff, old_pi = dqrr->pi;
+	DPA_ASSERT(dqrr->pmode == qm_dqrr_pce);
+	dqrr->pi = qm_cl_in(DQRR_PI) & (QM_DQRR_SIZE - 1);
+	diff = qm_cyc_diff(QM_DQRR_SIZE, old_pi, dqrr->pi);
+	dqrr->fill += diff;
+	return diff;
+}
+
+static inline void qm_dqrr_pvb_update(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	const struct qm_dqrr_entry *res = qm_cl(dqrr->ring, dqrr->pi);
+	DPA_ASSERT(dqrr->pmode == qm_dqrr_pvb);
+#ifndef CONFIG_FSL_PAMU
+        /*
+         * If PAMU is not available we need to invalidate the cache.
+         * When PAMU is available the cache is updated by stash
+         */
+	dcbi(res);
+	dcbt_ro(res);
+#endif
+
+	/* when accessing 'verb', use __raw_readb() to ensure that compiler
+	 * inlining doesn't try to optimise out "excess reads". */
+	if ((__raw_readb(&res->verb) & QM_DQRR_VERB_VBIT) == dqrr->vbit) {
+		dqrr->pi = (dqrr->pi + 1) & (QM_DQRR_SIZE - 1);
+		if (!dqrr->pi)
+			dqrr->vbit ^= QM_DQRR_VERB_VBIT;
+		dqrr->fill++;
+	}
+}
+
+
+static inline void qm_dqrr_cci_consume_to_current(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cci);
+	dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
+	qm_out(DQRR_CI_CINH, dqrr->ci);
+}
+
+static inline void qm_dqrr_cce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+	qm_cl_invalidate(DQRR_CI);
+	qm_cl_touch_rw(DQRR_CI);
+}
+
+static inline void qm_dqrr_cce_consume_to_current(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cce);
+	dqrr->ci = DQRR_PTR2IDX(dqrr->cursor);
+	qm_cl_out(DQRR_CI, dqrr->ci);
+}
+
+static inline void qm_dqrr_cdc_consume_1(struct qm_portal *portal, u8 idx,
+					int park)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	DPA_ASSERT(idx < QM_DQRR_SIZE);
+	qm_out(DQRR_DCAP, (0 << 8) |	/* S */
+		((park ? 1 : 0) << 6) |	/* PK */
+		idx);			/* DCAP_CI */
+}
+
+static inline void qm_dqrr_cdc_consume_1ptr(struct qm_portal *portal,
+					const struct qm_dqrr_entry *dq,
+					int park)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	u8 idx = DQRR_PTR2IDX(dq);
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	DPA_ASSERT((dqrr->ring + idx) == dq);
+	DPA_ASSERT(idx < QM_DQRR_SIZE);
+	qm_out(DQRR_DCAP, (0 << 8) |		/* DQRR_DCAP::S */
+		((park ? 1 : 0) << 6) |		/* DQRR_DCAP::PK */
+		idx);				/* DQRR_DCAP::DCAP_CI */
+}
+
+static inline u8 qm_dqrr_cdc_cci(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	return qm_in(DQRR_CI_CINH) & (QM_DQRR_SIZE - 1);
+}
+
+static inline void qm_dqrr_cdc_cce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	qm_cl_invalidate(DQRR_CI);
+	qm_cl_touch_ro(DQRR_CI);
+}
+
+static inline u8 qm_dqrr_cdc_cce(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode == qm_dqrr_cdc);
+	return qm_cl_in(DQRR_CI) & (QM_DQRR_SIZE - 1);
+}
+
+static inline u8 qm_dqrr_get_ci(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+	return dqrr->ci;
+}
+
+static inline void qm_dqrr_park(struct qm_portal *portal, u8 idx)
+{
+	__maybe_unused register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+	qm_out(DQRR_DCAP, (0 << 8) |		/* S */
+		(1 << 6) |			/* PK */
+		(idx & (QM_DQRR_SIZE - 1)));	/* DCAP_CI */
+}
+
+static inline void qm_dqrr_park_current(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	DPA_ASSERT(dqrr->cmode != qm_dqrr_cdc);
+	qm_out(DQRR_DCAP, (0 << 8) |		/* S */
+		(1 << 6) |			/* PK */
+		DQRR_PTR2IDX(dqrr->cursor));	/* DCAP_CI */
+}
+
+static inline void qm_dqrr_sdqcr_set(struct qm_portal *portal, u32 sdqcr)
+{
+	qm_out(DQRR_SDQCR, sdqcr);
+}
+
+static inline u32 qm_dqrr_sdqcr_get(struct qm_portal *portal)
+{
+	return qm_in(DQRR_SDQCR);
+}
+
+static inline void qm_dqrr_vdqcr_set(struct qm_portal *portal, u32 vdqcr)
+{
+	qm_out(DQRR_VDQCR, vdqcr);
+}
+
+static inline u32 qm_dqrr_vdqcr_get(struct qm_portal *portal)
+{
+	return qm_in(DQRR_VDQCR);
+}
+
+static inline void qm_dqrr_pdqcr_set(struct qm_portal *portal, u32 pdqcr)
+{
+	qm_out(DQRR_PDQCR, pdqcr);
+}
+
+static inline u32 qm_dqrr_pdqcr_get(struct qm_portal *portal)
+{
+	return qm_in(DQRR_PDQCR);
+}
+
+static inline u8 qm_dqrr_get_ithresh(struct qm_portal *portal)
+{
+	register struct qm_dqrr *dqrr = &portal->dqrr;
+	return dqrr->ithresh;
+}
+
+static inline void qm_dqrr_set_ithresh(struct qm_portal *portal, u8 ithresh)
+{
+	qm_out(DQRR_ITR, ithresh);
+}
+
+static inline u8 qm_dqrr_get_maxfill(struct qm_portal *portal)
+{
+	return (qm_in(CFG) & 0x00f00000) >> 20;
+}
+
+
+/* -------------- */
+/* --- MR API --- */
+
+#define MR_CARRYCLEAR(p) \
+	(void *)((unsigned long)(p) & (~(unsigned long)(QM_MR_SIZE << 6)))
+
+static inline u8 MR_PTR2IDX(const struct qm_mr_entry *e)
+{
+	return ((uintptr_t)e >> 6) & (QM_MR_SIZE - 1);
+}
+
+static inline const struct qm_mr_entry *MR_INC(const struct qm_mr_entry *e)
+{
+	return MR_CARRYCLEAR(e + 1);
+}
+
+static inline int qm_mr_init(struct qm_portal *portal, enum qm_mr_pmode pmode,
+		enum qm_mr_cmode cmode)
+{
+	register struct qm_mr *mr = &portal->mr;
+	u32 cfg;
+
+	mr->ring = portal->addr.addr_ce + QM_CL_MR;
+	mr->pi = qm_in(MR_PI_CINH) & (QM_MR_SIZE - 1);
+	mr->ci = qm_in(MR_CI_CINH) & (QM_MR_SIZE - 1);
+	mr->cursor = mr->ring + mr->ci;
+	mr->fill = qm_cyc_diff(QM_MR_SIZE, mr->ci, mr->pi);
+	mr->vbit = (qm_in(MR_PI_CINH) & QM_MR_SIZE) ? QM_MR_VERB_VBIT : 0;
+	mr->ithresh = qm_in(MR_ITR);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mr->pmode = pmode;
+	mr->cmode = cmode;
+#endif
+	cfg = (qm_in(CFG) & 0xfffff0ff) |
+		((cmode & 1) << 8);		/* QCSP_CFG:MM */
+	qm_out(CFG, cfg);
+	return 0;
+}
+
+static inline void qm_mr_finish(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	if (mr->ci != MR_PTR2IDX(mr->cursor))
+		pr_crit("Ignoring completed MR entries\n");
+}
+
+static inline const struct qm_mr_entry *qm_mr_current(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	if (!mr->fill)
+		return NULL;
+	return mr->cursor;
+}
+
+static inline u8 qm_mr_cursor(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	return MR_PTR2IDX(mr->cursor);
+}
+
+static inline u8 qm_mr_next(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->fill);
+	mr->cursor = MR_INC(mr->cursor);
+	return --mr->fill;
+}
+
+static inline u8 qm_mr_pci_update(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	u8 diff, old_pi = mr->pi;
+	DPA_ASSERT(mr->pmode == qm_mr_pci);
+	mr->pi = qm_in(MR_PI_CINH);
+	diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
+	mr->fill += diff;
+	return diff;
+}
+
+static inline void qm_mr_pce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->pmode == qm_mr_pce);
+	qm_cl_invalidate(MR_PI);
+	qm_cl_touch_ro(MR_PI);
+}
+
+static inline u8 qm_mr_pce_update(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	u8 diff, old_pi = mr->pi;
+	DPA_ASSERT(mr->pmode == qm_mr_pce);
+	mr->pi = qm_cl_in(MR_PI) & (QM_MR_SIZE - 1);
+	diff = qm_cyc_diff(QM_MR_SIZE, old_pi, mr->pi);
+	mr->fill += diff;
+	return diff;
+}
+
+static inline void qm_mr_pvb_update(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	const struct qm_mr_entry *res = qm_cl(mr->ring, mr->pi);
+	DPA_ASSERT(mr->pmode == qm_mr_pvb);
+	/* when accessing 'verb', use __raw_readb() to ensure that compiler
+	 * inlining doesn't try to optimise out "excess reads". */
+	if ((__raw_readb(&res->verb) & QM_MR_VERB_VBIT) == mr->vbit) {
+		mr->pi = (mr->pi + 1) & (QM_MR_SIZE - 1);
+		if (!mr->pi)
+			mr->vbit ^= QM_MR_VERB_VBIT;
+		mr->fill++;
+		res = MR_INC(res);
+	}
+	dcbit_ro(res);
+}
+
+static inline void qm_mr_cci_consume(struct qm_portal *portal, u8 num)
+{
+	register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->cmode == qm_mr_cci);
+	mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
+	qm_out(MR_CI_CINH, mr->ci);
+}
+
+static inline void qm_mr_cci_consume_to_current(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->cmode == qm_mr_cci);
+	mr->ci = MR_PTR2IDX(mr->cursor);
+	qm_out(MR_CI_CINH, mr->ci);
+}
+
+static inline void qm_mr_cce_prefetch(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->cmode == qm_mr_cce);
+	qm_cl_invalidate(MR_CI);
+	qm_cl_touch_rw(MR_CI);
+}
+
+static inline void qm_mr_cce_consume(struct qm_portal *portal, u8 num)
+{
+	register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->cmode == qm_mr_cce);
+	mr->ci = (mr->ci + num) & (QM_MR_SIZE - 1);
+	qm_cl_out(MR_CI, mr->ci);
+}
+
+static inline void qm_mr_cce_consume_to_current(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	DPA_ASSERT(mr->cmode == qm_mr_cce);
+	mr->ci = MR_PTR2IDX(mr->cursor);
+	qm_cl_out(MR_CI, mr->ci);
+}
+
+static inline u8 qm_mr_get_ci(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	return mr->ci;
+}
+
+static inline u8 qm_mr_get_ithresh(struct qm_portal *portal)
+{
+	register struct qm_mr *mr = &portal->mr;
+	return mr->ithresh;
+}
+
+static inline void qm_mr_set_ithresh(struct qm_portal *portal, u8 ithresh)
+{
+	qm_out(MR_ITR, ithresh);
+}
+
+
+/* ------------------------------ */
+/* --- Management command API --- */
+
+static inline int qm_mc_init(struct qm_portal *portal)
+{
+	register struct qm_mc *mc = &portal->mc;
+	mc->cr = portal->addr.addr_ce + QM_CL_CR;
+	mc->rr = portal->addr.addr_ce + QM_CL_RR0;
+	mc->rridx = (__raw_readb(&mc->cr->__dont_write_directly__verb) &
+			QM_MCC_VERB_VBIT) ?  0 : 1;
+	mc->vbit = mc->rridx ? QM_MCC_VERB_VBIT : 0;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = qman_mc_idle;
+#endif
+	return 0;
+}
+
+static inline void qm_mc_finish(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == qman_mc_idle);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	if (mc->state != qman_mc_idle)
+		pr_crit("Losing incomplete MC command\n");
+#endif
+}
+
+static inline struct qm_mc_command *qm_mc_start(struct qm_portal *portal)
+{
+	register struct qm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == qman_mc_idle);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = qman_mc_user;
+#endif
+	dcbz_64(mc->cr);
+	return mc->cr;
+}
+
+static inline void qm_mc_abort(struct qm_portal *portal)
+{
+	__maybe_unused register struct qm_mc *mc = &portal->mc;
+	DPA_ASSERT(mc->state == qman_mc_user);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = qman_mc_idle;
+#endif
+}
+
+static inline void qm_mc_commit(struct qm_portal *portal, u8 myverb)
+{
+	register struct qm_mc *mc = &portal->mc;
+	struct qm_mc_result *rr = mc->rr + mc->rridx;
+	DPA_ASSERT(mc->state == qman_mc_user);
+	lwsync();
+	mc->cr->__dont_write_directly__verb = myverb | mc->vbit;
+	dcbf(mc->cr);
+	dcbit_ro(rr);
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = qman_mc_hw;
+#endif
+}
+
+static inline struct qm_mc_result *qm_mc_result(struct qm_portal *portal)
+{
+	register struct qm_mc *mc = &portal->mc;
+	struct qm_mc_result *rr = mc->rr + mc->rridx;
+	DPA_ASSERT(mc->state == qman_mc_hw);
+	/* The inactive response register's verb byte always returns zero until
+	 * its command is submitted and completed. This includes the valid-bit,
+	 * in case you were wondering... */
+	if (!__raw_readb(&rr->verb)) {
+		dcbit_ro(rr);
+		return NULL;
+	}
+	mc->rridx ^= 1;
+	mc->vbit ^= QM_MCC_VERB_VBIT;
+#ifdef CONFIG_FSL_DPA_CHECKING
+	mc->state = qman_mc_idle;
+#endif
+	return rr;
+}
+
+
+/* ------------------------------------- */
+/* --- Portal interrupt register API --- */
+
+static inline int qm_isr_init(__always_unused struct qm_portal *portal)
+{
+	return 0;
+}
+
+static inline void qm_isr_finish(__always_unused struct qm_portal *portal)
+{
+}
+
+static inline void qm_isr_set_iperiod(struct qm_portal *portal, u16 iperiod)
+{
+	qm_out(ITPR, iperiod);
+}
+
+static inline u32 __qm_isr_read(struct qm_portal *portal, enum qm_isr_reg n)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	return __qm_in(&portal->addr, QM_REG_ISR + (n << 6));
+#else
+	return __qm_in(&portal->addr, QM_REG_ISR + (n << 2));
+#endif
+}
+
+static inline void __qm_isr_write(struct qm_portal *portal, enum qm_isr_reg n,
+					u32 val)
+{
+#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
+	__qm_out(&portal->addr, QM_REG_ISR + (n << 6), val);
+#else
+	__qm_out(&portal->addr, QM_REG_ISR + (n << 2), val);
+#endif
+}
+
+/* Cleanup FQs */
+static inline int qm_shutdown_fq(struct qm_portal **portal, int portal_count,
+				 u32 fqid)
+{
+
+	struct qm_mc_command *mcc;
+	struct qm_mc_result *mcr;
+	u8 state;
+	int orl_empty, fq_empty, i, drain = 0;
+	u32 result;
+	u32 channel, wq;
+	u16 dest_wq;
+
+	/* Determine the state of the FQID */
+	mcc = qm_mc_start(portal[0]);
+	mcc->queryfq_np.fqid = cpu_to_be32(fqid);
+	qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ_NP);
+	while (!(mcr = qm_mc_result(portal[0])))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ_NP);
+	state = mcr->queryfq_np.state & QM_MCR_NP_STATE_MASK;
+	if (state == QM_MCR_NP_STATE_OOS)
+		return 0; /* Already OOS, no need to do anymore checks */
+
+	/* Query which channel the FQ is using */
+	mcc = qm_mc_start(portal[0]);
+	mcc->queryfq.fqid = cpu_to_be32(fqid);
+	qm_mc_commit(portal[0], QM_MCC_VERB_QUERYFQ);
+	while (!(mcr = qm_mc_result(portal[0])))
+		cpu_relax();
+	DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) == QM_MCR_VERB_QUERYFQ);
+
+	/* Need to store these since the MCR gets reused */
+	dest_wq = be16_to_cpu(mcr->queryfq.fqd.dest_wq);
+	wq = dest_wq & 0x7;
+	channel = dest_wq>>3;
+
+	switch (state) {
+	case QM_MCR_NP_STATE_TEN_SCHED:
+	case QM_MCR_NP_STATE_TRU_SCHED:
+	case QM_MCR_NP_STATE_ACTIVE:
+	case QM_MCR_NP_STATE_PARKED:
+		orl_empty = 0;
+		mcc = qm_mc_start(portal[0]);
+		mcc->alterfq.fqid = cpu_to_be32(fqid);
+		qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_RETIRE);
+		while (!(mcr = qm_mc_result(portal[0])))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			   QM_MCR_VERB_ALTER_RETIRE);
+		result = mcr->result; /* Make a copy as we reuse MCR below */
+
+		if (result == QM_MCR_RESULT_PENDING) {
+			/* Need to wait for the FQRN in the message ring, which
+			   will only occur once the FQ has been drained.  In
+			   order for the FQ to drain the portal needs to be set
+			   to dequeue from the channel the FQ is scheduled on */
+			const struct qm_mr_entry *msg;
+			const struct qm_dqrr_entry *dqrr = NULL;
+			int found_fqrn = 0;
+			u16 dequeue_wq = 0;
+
+			/* Flag that we need to drain FQ */
+			drain = 1;
+
+			if (channel >= qm_channel_pool1 &&
+			    channel < (qm_channel_pool1 + 15)) {
+				/* Pool channel, enable the bit in the portal */
+				dequeue_wq = (channel -
+					      qm_channel_pool1 + 1)<<4 | wq;
+			} else if (channel < qm_channel_pool1) {
+				/* Dedicated channel */
+				dequeue_wq = wq;
+			} else {
+				pr_info("Cannot recover FQ 0x%x, it is "
+					"scheduled on channel 0x%x",
+					fqid, channel);
+				return -EBUSY;
+			}
+			/* Set the sdqcr to drain this channel */
+			if (channel < qm_channel_pool1)
+				for (i = 0; i < portal_count; i++)
+					qm_dqrr_sdqcr_set(portal[i],
+						  QM_SDQCR_TYPE_ACTIVE |
+						  QM_SDQCR_CHANNELS_DEDICATED);
+			else
+				for (i = 0; i < portal_count; i++)
+					qm_dqrr_sdqcr_set(
+						portal[i],
+						QM_SDQCR_TYPE_ACTIVE |
+						QM_SDQCR_CHANNELS_POOL_CONV
+						(channel));
+			while (!found_fqrn) {
+				/* Keep draining DQRR while checking the MR*/
+				for (i = 0; i < portal_count; i++) {
+					qm_dqrr_pvb_update(portal[i]);
+					dqrr = qm_dqrr_current(portal[i]);
+					while (dqrr) {
+						qm_dqrr_cdc_consume_1ptr(
+							portal[i], dqrr, 0);
+						qm_dqrr_pvb_update(portal[i]);
+						qm_dqrr_next(portal[i]);
+						dqrr = qm_dqrr_current(
+							portal[i]);
+					}
+					/* Process message ring too */
+					qm_mr_pvb_update(portal[i]);
+					msg = qm_mr_current(portal[i]);
+					while (msg) {
+						if ((msg->verb &
+						     QM_MR_VERB_TYPE_MASK)
+						    == QM_MR_VERB_FQRN)
+							found_fqrn = 1;
+						qm_mr_next(portal[i]);
+						qm_mr_cci_consume_to_current(
+							portal[i]);
+						qm_mr_pvb_update(portal[i]);
+						msg = qm_mr_current(portal[i]);
+					}
+					cpu_relax();
+				}
+			}
+		}
+		if (result != QM_MCR_RESULT_OK &&
+		    result !=  QM_MCR_RESULT_PENDING) {
+			/* error */
+			pr_err("qman_retire_fq failed on FQ 0x%x, result=0x%x\n",
+			       fqid, result);
+			return -1;
+		}
+		if (!(mcr->alterfq.fqs & QM_MCR_FQS_ORLPRESENT)) {
+			/* ORL had no entries, no need to wait until the
+			   ERNs come in */
+			orl_empty = 1;
+		}
+		/* Retirement succeeded, check to see if FQ needs
+		   to be drained */
+		if (drain || mcr->alterfq.fqs & QM_MCR_FQS_NOTEMPTY) {
+			/* FQ is Not Empty, drain using volatile DQ commands */
+			fq_empty = 0;
+			do {
+				const struct qm_dqrr_entry *dqrr = NULL;
+				u32 vdqcr = fqid | QM_VDQCR_NUMFRAMES_SET(3);
+				qm_dqrr_vdqcr_set(portal[0], vdqcr);
+
+				/* Wait for a dequeue to occur */
+				while (dqrr == NULL) {
+					qm_dqrr_pvb_update(portal[0]);
+					dqrr = qm_dqrr_current(portal[0]);
+					if (!dqrr)
+						cpu_relax();
+				}
+				/* Process the dequeues, making sure to
+				   empty the ring completely */
+				while (dqrr) {
+					if (be32_to_cpu(dqrr->fqid) == fqid &&
+					    dqrr->stat & QM_DQRR_STAT_FQ_EMPTY)
+						fq_empty = 1;
+					qm_dqrr_cdc_consume_1ptr(portal[0],
+								 dqrr, 0);
+					qm_dqrr_pvb_update(portal[0]);
+					qm_dqrr_next(portal[0]);
+					dqrr = qm_dqrr_current(portal[0]);
+				}
+			} while (fq_empty == 0);
+		}
+		for (i = 0; i < portal_count; i++)
+			qm_dqrr_sdqcr_set(portal[i], 0);
+
+		/* Wait for the ORL to have been completely drained */
+		while (orl_empty == 0) {
+			const struct qm_mr_entry *msg;
+			qm_mr_pvb_update(portal[0]);
+			msg = qm_mr_current(portal[0]);
+			while (msg) {
+				if ((msg->verb & QM_MR_VERB_TYPE_MASK) ==
+				    QM_MR_VERB_FQRL)
+					orl_empty = 1;
+				qm_mr_next(portal[0]);
+				qm_mr_cci_consume_to_current(portal[0]);
+				qm_mr_pvb_update(portal[0]);
+				msg = qm_mr_current(portal[0]);
+			}
+			cpu_relax();
+		}
+		mcc = qm_mc_start(portal[0]);
+		mcc->alterfq.fqid = cpu_to_be32(fqid);
+		qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS);
+		while (!(mcr = qm_mc_result(portal[0])))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			   QM_MCR_VERB_ALTER_OOS);
+		if (mcr->result != QM_MCR_RESULT_OK) {
+			pr_err("OOS after drain Failed on FQID 0x%x, result 0x%x\n",
+			       fqid, mcr->result);
+			return -1;
+		}
+		return 0;
+	case QM_MCR_NP_STATE_RETIRED:
+		/* Send OOS Command */
+		mcc = qm_mc_start(portal[0]);
+		mcc->alterfq.fqid = cpu_to_be32(fqid);
+		qm_mc_commit(portal[0], QM_MCC_VERB_ALTER_OOS);
+		while (!(mcr = qm_mc_result(portal[0])))
+			cpu_relax();
+		DPA_ASSERT((mcr->verb & QM_MCR_VERB_MASK) ==
+			   QM_MCR_VERB_ALTER_OOS);
+		if (mcr->result) {
+			pr_err("OOS Failed on FQID 0x%x\n", fqid);
+			return -1;
+		}
+		return 0;
+	}
+	return -1;
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_private.h
@@ -0,0 +1,398 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "dpa_sys.h"
+#include <linux/fsl_qman.h>
+#include <linux/iommu.h>
+
+#if defined(CONFIG_FSL_PAMU)
+#include <asm/fsl_pamu_stash.h>
+#endif
+
+#if !defined(CONFIG_FSL_QMAN_FQ_LOOKUP) && defined(CONFIG_PPC64)
+#error "_PPC64 requires _FSL_QMAN_FQ_LOOKUP"
+#endif
+
+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
+	/* ----------------- */
+	/* Congestion Groups */
+	/* ----------------- */
+/* This wrapper represents a bit-array for the state of the 256 Qman congestion
+ * groups. Is also used as a *mask* for congestion groups, eg. so we ignore
+ * those that don't concern us. We harness the structure and accessor details
+ * already used in the management command to query congestion groups. */
+struct qman_cgrs {
+	struct __qm_mcr_querycongestion q;
+};
+static inline void qman_cgrs_init(struct qman_cgrs *c)
+{
+	memset(c, 0, sizeof(*c));
+}
+static inline void qman_cgrs_fill(struct qman_cgrs *c)
+{
+	memset(c, 0xff, sizeof(*c));
+}
+static inline int qman_cgrs_get(struct qman_cgrs *c, int num)
+{
+	return QM_MCR_QUERYCONGESTION(&c->q, num);
+}
+static inline void qman_cgrs_set(struct qman_cgrs *c, int num)
+{
+	c->q.__state[__CGR_WORD(num)] |= (0x80000000 >> __CGR_SHIFT(num));
+}
+static inline void qman_cgrs_unset(struct qman_cgrs *c, int num)
+{
+	c->q.__state[__CGR_WORD(num)] &= ~(0x80000000 >> __CGR_SHIFT(num));
+}
+static inline int qman_cgrs_next(struct qman_cgrs *c, int num)
+{
+	while ((++num < __CGR_NUM) && !qman_cgrs_get(c, num))
+		;
+	return num;
+}
+static inline void qman_cgrs_cp(struct qman_cgrs *dest,
+				const struct qman_cgrs *src)
+{
+	*dest = *src;
+}
+static inline void qman_cgrs_and(struct qman_cgrs *dest,
+			const struct qman_cgrs *a, const struct qman_cgrs *b)
+{
+	int ret;
+	u32 *_d = dest->q.__state;
+	const u32 *_a = a->q.__state;
+	const u32 *_b = b->q.__state;
+	for (ret = 0; ret < 8; ret++)
+		*(_d++) = *(_a++) & *(_b++);
+}
+static inline void qman_cgrs_xor(struct qman_cgrs *dest,
+			const struct qman_cgrs *a, const struct qman_cgrs *b)
+{
+	int ret;
+	u32 *_d = dest->q.__state;
+	const u32 *_a = a->q.__state;
+	const u32 *_b = b->q.__state;
+	for (ret = 0; ret < 8; ret++)
+		*(_d++) = *(_a++) ^ *(_b++);
+}
+
+	/* ----------------------- */
+	/* CEETM Congestion Groups */
+	/* ----------------------- */
+/* This wrapper represents a bit-array for the state of the 512 Qman CEETM
+ * congestion groups.
+ */
+struct qman_ccgrs {
+	struct __qm_mcr_querycongestion q[2];
+};
+static inline void qman_ccgrs_init(struct qman_ccgrs *c)
+{
+	memset(c, 0, sizeof(*c));
+}
+static inline void qman_ccgrs_fill(struct qman_ccgrs *c)
+{
+	memset(c, 0xff, sizeof(*c));
+}
+static inline int qman_ccgrs_get(struct qman_ccgrs *c, int num)
+{
+	if (num < __CGR_NUM)
+		return QM_MCR_QUERYCONGESTION(&c->q[0], num);
+	else
+		return QM_MCR_QUERYCONGESTION(&c->q[1], (num - __CGR_NUM));
+}
+static inline int qman_ccgrs_next(struct qman_ccgrs *c, int num)
+{
+	while ((++num < __CGR_NUM) && !qman_ccgrs_get(c, num))
+		;
+	return num;
+}
+static inline void qman_ccgrs_cp(struct qman_ccgrs *dest,
+					const struct qman_ccgrs *src)
+{
+	*dest = *src;
+}
+static inline void qman_ccgrs_and(struct qman_ccgrs *dest,
+			const struct qman_ccgrs *a, const struct qman_ccgrs *b)
+{
+	int ret, i;
+	u32 *_d;
+	const u32 *_a, *_b;
+	for (i = 0; i < 2; i++) {
+		_d = dest->q[i].__state;
+		_a = a->q[i].__state;
+		_b = b->q[i].__state;
+		for (ret = 0; ret < 8; ret++)
+			*(_d++) = *(_a++) & *(_b++);
+	}
+}
+static inline void qman_ccgrs_xor(struct qman_ccgrs *dest,
+			const struct qman_ccgrs *a, const struct qman_ccgrs *b)
+{
+	int ret, i;
+	u32 *_d;
+	const u32 *_a, *_b;
+	for (i = 0; i < 2; i++) {
+		_d = dest->q[i].__state;
+		_a = a->q[i].__state;
+		_b = b->q[i].__state;
+		for (ret = 0; ret < 8; ret++)
+			*(_d++) = *(_a++) ^ *(_b++);
+	}
+}
+
+/* used by CCSR and portal interrupt code */
+enum qm_isr_reg {
+	qm_isr_status = 0,
+	qm_isr_enable = 1,
+	qm_isr_disable = 2,
+	qm_isr_inhibit = 3
+};
+
+struct qm_portal_config {
+	/* Corenet portal addresses;
+	 * [0]==cache-enabled, [1]==cache-inhibited. */
+	__iomem void *addr_virt[2];
+	struct resource addr_phys[2];
+	struct device dev;
+	struct iommu_domain *iommu_domain;
+	/* Allow these to be joined in lists */
+	struct list_head list;
+	/* User-visible portal configuration settings */
+	struct qman_portal_config public_cfg;
+	/* power management saved data */
+	u32 saved_isdr;
+};
+
+/* Revision info (for errata and feature handling) */
+#define QMAN_REV11 0x0101
+#define QMAN_REV12 0x0102
+#define QMAN_REV20 0x0200
+#define QMAN_REV30 0x0300
+#define QMAN_REV31 0x0301
+#define QMAN_REV32 0x0302
+
+/* QMan REV_2 register contains the Cfg option */
+#define QMAN_REV_CFG_0 0x0
+#define QMAN_REV_CFG_1 0x1
+#define QMAN_REV_CFG_2 0x2
+#define QMAN_REV_CFG_3 0x3
+
+extern u16 qman_ip_rev; /* 0 if uninitialised, otherwise QMAN_REVx */
+extern u8 qman_ip_cfg;
+extern u32 qman_clk;
+extern u16 qman_portal_max;
+
+#ifdef CONFIG_FSL_QMAN_CONFIG
+/* Hooks from qman_driver.c to qman_config.c */
+int qman_init_ccsr(struct device_node *node);
+void qman_liodn_fixup(u16 channel);
+int qman_set_sdest(u16 channel, unsigned int cpu_idx);
+size_t get_qman_fqd_size(void);
+#else
+static inline size_t get_qman_fqd_size(void)
+{
+	return (PAGE_SIZE << CONFIG_FSL_QMAN_FQD_SZ);
+}
+#endif
+
+int qm_set_wpm(int wpm);
+int qm_get_wpm(int *wpm);
+
+/* Hooks from qman_driver.c in to qman_high.c */
+struct qman_portal *qman_create_portal(
+			struct qman_portal *portal,
+			const struct qm_portal_config *config,
+			const struct qman_cgrs *cgrs);
+
+struct qman_portal *qman_create_affine_portal(
+			const struct qm_portal_config *config,
+			const struct qman_cgrs *cgrs);
+struct qman_portal *qman_create_affine_slave(struct qman_portal *redirect,
+								int cpu);
+const struct qm_portal_config *qman_destroy_affine_portal(void);
+void qman_destroy_portal(struct qman_portal *qm);
+
+/* Hooks from fsl_usdpaa.c to qman_driver.c */
+struct qm_portal_config *qm_get_unused_portal(void);
+struct qm_portal_config *qm_get_unused_portal_idx(uint32_t idx);
+
+void qm_put_unused_portal(struct qm_portal_config *pcfg);
+void qm_set_liodns(struct qm_portal_config *pcfg);
+
+/* This CGR feature is supported by h/w and required by unit-tests and the
+ * debugfs hooks, so is implemented in the driver. However it allows an explicit
+ * corruption of h/w fields by s/w that are usually incorruptible (because the
+ * counters are usually maintained entirely within h/w). As such, we declare
+ * this API internally. */
+int qman_testwrite_cgr(struct qman_cgr *cgr, u64 i_bcnt,
+	struct qm_mcr_cgrtestwrite *result);
+
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+/* If the fq object pointer is greater than the size of context_b field,
+ * than a lookup table is required. */
+int qman_setup_fq_lookup_table(size_t num_entries);
+#endif
+
+
+/*************************************************/
+/*   QMan s/w corenet portal, low-level i/face   */
+/*************************************************/
+
+/* Note: most functions are only used by the high-level interface, so are
+ * inlined from qman_low.h. The stuff below is for use by other parts of the
+ * driver. */
+
+/* For qm_dqrr_sdqcr_set(); Choose one SOURCE. Choose one COUNT. Choose one
+ * dequeue TYPE. Choose TOKEN (8-bit).
+ * If SOURCE == CHANNELS,
+ *   Choose CHANNELS_DEDICATED and/or CHANNELS_POOL(n).
+ *   You can choose DEDICATED_PRECEDENCE if the portal channel should have
+ *   priority.
+ * If SOURCE == SPECIFICWQ,
+ *     Either select the work-queue ID with SPECIFICWQ_WQ(), or select the
+ *     channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the
+ *     work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the
+ *     same value.
+ */
+#define QM_SDQCR_SOURCE_CHANNELS	0x0
+#define QM_SDQCR_SOURCE_SPECIFICWQ	0x40000000
+#define QM_SDQCR_COUNT_EXACT1		0x0
+#define QM_SDQCR_COUNT_UPTO3		0x20000000
+#define QM_SDQCR_DEDICATED_PRECEDENCE	0x10000000
+#define QM_SDQCR_TYPE_MASK		0x03000000
+#define QM_SDQCR_TYPE_NULL		0x0
+#define QM_SDQCR_TYPE_PRIO_QOS		0x01000000
+#define QM_SDQCR_TYPE_ACTIVE_QOS	0x02000000
+#define QM_SDQCR_TYPE_ACTIVE		0x03000000
+#define QM_SDQCR_TOKEN_MASK		0x00ff0000
+#define QM_SDQCR_TOKEN_SET(v)		(((v) & 0xff) << 16)
+#define QM_SDQCR_TOKEN_GET(v)		(((v) >> 16) & 0xff)
+#define QM_SDQCR_CHANNELS_DEDICATED	0x00008000
+#define QM_SDQCR_SPECIFICWQ_MASK	0x000000f7
+#define QM_SDQCR_SPECIFICWQ_DEDICATED	0x00000000
+#define QM_SDQCR_SPECIFICWQ_POOL(n)	((n) << 4)
+#define QM_SDQCR_SPECIFICWQ_WQ(n)	(n)
+
+/* For qm_dqrr_vdqcr_set(): use FQID(n) to fill in the frame queue ID */
+#define QM_VDQCR_FQID_MASK		0x00ffffff
+#define QM_VDQCR_FQID(n)		((n) & QM_VDQCR_FQID_MASK)
+
+/* For qm_dqrr_pdqcr_set(); Choose one MODE. Choose one COUNT.
+ * If MODE==SCHEDULED
+ *   Choose SCHEDULED_CHANNELS or SCHEDULED_SPECIFICWQ. Choose one dequeue TYPE.
+ *   If CHANNELS,
+ *     Choose CHANNELS_DEDICATED and/or CHANNELS_POOL() channels.
+ *     You can choose DEDICATED_PRECEDENCE if the portal channel should have
+ *     priority.
+ *   If SPECIFICWQ,
+ *     Either select the work-queue ID with SPECIFICWQ_WQ(), or select the
+ *     channel (SPECIFICWQ_DEDICATED or SPECIFICWQ_POOL()) and specify the
+ *     work-queue priority (0-7) with SPECIFICWQ_WQ() - either way, you get the
+ *     same value.
+ * If MODE==UNSCHEDULED
+ *     Choose FQID().
+ */
+#define QM_PDQCR_MODE_SCHEDULED		0x0
+#define QM_PDQCR_MODE_UNSCHEDULED	0x80000000
+#define QM_PDQCR_SCHEDULED_CHANNELS	0x0
+#define QM_PDQCR_SCHEDULED_SPECIFICWQ	0x40000000
+#define QM_PDQCR_COUNT_EXACT1		0x0
+#define QM_PDQCR_COUNT_UPTO3		0x20000000
+#define QM_PDQCR_DEDICATED_PRECEDENCE	0x10000000
+#define QM_PDQCR_TYPE_MASK		0x03000000
+#define QM_PDQCR_TYPE_NULL		0x0
+#define QM_PDQCR_TYPE_PRIO_QOS		0x01000000
+#define QM_PDQCR_TYPE_ACTIVE_QOS	0x02000000
+#define QM_PDQCR_TYPE_ACTIVE		0x03000000
+#define QM_PDQCR_CHANNELS_DEDICATED	0x00008000
+#define QM_PDQCR_CHANNELS_POOL(n)	(0x00008000 >> (n))
+#define QM_PDQCR_SPECIFICWQ_MASK	0x000000f7
+#define QM_PDQCR_SPECIFICWQ_DEDICATED	0x00000000
+#define QM_PDQCR_SPECIFICWQ_POOL(n)	((n) << 4)
+#define QM_PDQCR_SPECIFICWQ_WQ(n)	(n)
+#define QM_PDQCR_FQID(n)		((n) & 0xffffff)
+
+/* Used by all portal interrupt registers except 'inhibit'
+ * Channels with frame availability
+ */
+#define QM_PIRQ_DQAVAIL	0x0000ffff
+
+/* The DQAVAIL interrupt fields break down into these bits; */
+#define QM_DQAVAIL_PORTAL	0x8000		/* Portal channel */
+#define QM_DQAVAIL_POOL(n)	(0x8000 >> (n))	/* Pool channel, n==[1..15] */
+#define QM_DQAVAIL_MASK		0xffff
+/* This mask contains all the "irqsource" bits visible to API users */
+#define QM_PIRQ_VISIBLE	(QM_PIRQ_SLOW | QM_PIRQ_DQRI)
+
+/* These are qm_<reg>_<verb>(). So for example, qm_disable_write() means "write
+ * the disable register" rather than "disable the ability to write". */
+#define qm_isr_status_read(qm)		__qm_isr_read(qm, qm_isr_status)
+#define qm_isr_status_clear(qm, m)	__qm_isr_write(qm, qm_isr_status, m)
+#define qm_isr_enable_read(qm)		__qm_isr_read(qm, qm_isr_enable)
+#define qm_isr_enable_write(qm, v)	__qm_isr_write(qm, qm_isr_enable, v)
+#define qm_isr_disable_read(qm)		__qm_isr_read(qm, qm_isr_disable)
+#define qm_isr_disable_write(qm, v)	__qm_isr_write(qm, qm_isr_disable, v)
+/* TODO: unfortunate name-clash here, reword? */
+#define qm_isr_inhibit(qm)		__qm_isr_write(qm, qm_isr_inhibit, 1)
+#define qm_isr_uninhibit(qm)		__qm_isr_write(qm, qm_isr_inhibit, 0)
+
+#ifdef CONFIG_FSL_QMAN_CONFIG
+int qman_have_ccsr(void);
+#else
+#define qman_have_ccsr	0
+#endif
+
+__init int qman_init(void);
+__init int qman_resource_init(void);
+
+/* CEETM related */
+#define QMAN_CEETM_MAX	2
+extern u8 num_ceetms;
+extern struct qm_ceetm qman_ceetms[QMAN_CEETM_MAX];
+int qman_sp_enable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal);
+int qman_sp_disable_ceetm_mode(enum qm_dc_portal portal, u16 sub_portal);
+int qman_ceetm_set_prescaler(enum qm_dc_portal portal);
+int qman_ceetm_get_prescaler(u16 *pres);
+int qman_ceetm_query_cq(unsigned int cqid, unsigned int dcpid,
+			struct qm_mcr_ceetm_cq_query *cq_query);
+int qman_ceetm_query_ccgr(struct qm_mcc_ceetm_ccgr_query *ccgr_query,
+				struct qm_mcr_ceetm_ccgr_query *response);
+int qman_ceetm_get_xsfdr(enum qm_dc_portal portal, unsigned int *num);
+
+extern void *affine_portals[NR_CPUS];
+const struct qm_portal_config *qman_get_qm_portal_config(
+						struct qman_portal *portal);
+
+/* power management */
+#ifdef CONFIG_SUSPEND
+void suspend_unused_qportal(void);
+void resume_unused_qportal(void);
+#endif
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_test.c
@@ -0,0 +1,57 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_test.h"
+
+MODULE_AUTHOR("Geoff Thorpe");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_DESCRIPTION("Qman testing");
+
+static int test_init(void)
+{
+	int loop = 1;
+	while (loop--) {
+#ifdef CONFIG_FSL_QMAN_TEST_STASH_POTATO
+		qman_test_hotpotato();
+#endif
+#ifdef CONFIG_FSL_QMAN_TEST_HIGH
+		qman_test_high();
+#endif
+	}
+	return 0;
+}
+
+static void test_exit(void)
+{
+}
+
+module_init(test_init);
+module_exit(test_exit);
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_test.h
@@ -0,0 +1,45 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/io.h>
+#include <linux/slab.h>
+#include <linux/module.h>
+#include <linux/interrupt.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+
+#include <linux/fsl_qman.h>
+
+void qman_test_hotpotato(void);
+void qman_test_high(void);
+
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_test_high.c
@@ -0,0 +1,216 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_test.h"
+
+/*************/
+/* constants */
+/*************/
+
+#define CGR_ID		27
+#define POOL_ID		2
+#define FQ_FLAGS	QMAN_FQ_FLAG_DYNAMIC_FQID
+#define NUM_ENQUEUES	10
+#define NUM_PARTIAL	4
+#define PORTAL_SDQCR	(QM_SDQCR_SOURCE_CHANNELS | \
+			QM_SDQCR_TYPE_PRIO_QOS | \
+			QM_SDQCR_TOKEN_SET(0x98) | \
+			QM_SDQCR_CHANNELS_DEDICATED | \
+			QM_SDQCR_CHANNELS_POOL(POOL_ID))
+#define PORTAL_OPAQUE	((void *)0xf00dbeef)
+#define VDQCR_FLAGS	(QMAN_VOLATILE_FLAG_WAIT | QMAN_VOLATILE_FLAG_FINISH)
+
+/*************************************/
+/* Predeclarations (eg. for fq_base) */
+/*************************************/
+
+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *,
+					struct qman_fq *,
+					const struct qm_dqrr_entry *);
+static void cb_ern(struct qman_portal *, struct qman_fq *,
+			const struct qm_mr_entry *);
+static void cb_fqs(struct qman_portal *, struct qman_fq *,
+			const struct qm_mr_entry *);
+
+/***************/
+/* global vars */
+/***************/
+
+static struct qm_fd fd, fd_dq;
+static struct qman_fq fq_base = {
+	.cb.dqrr = cb_dqrr,
+	.cb.ern = cb_ern,
+	.cb.fqs = cb_fqs
+};
+static DECLARE_WAIT_QUEUE_HEAD(waitqueue);
+static int retire_complete, sdqcr_complete;
+
+/**********************/
+/* internal functions */
+/**********************/
+
+/* Helpers for initialising and "incrementing" a frame descriptor */
+static void fd_init(struct qm_fd *__fd)
+{
+	qm_fd_addr_set64(__fd, 0xabdeadbeefLLU);
+	__fd->format = qm_fd_contig_big;
+	__fd->length29 = 0x0000ffff;
+	__fd->cmd = 0xfeedf00d;
+}
+
+static void fd_inc(struct qm_fd *__fd)
+{
+	u64 t = qm_fd_addr_get64(__fd);
+	int z = t >> 40;
+	t <<= 1;
+	if (z)
+		t |= 1;
+	qm_fd_addr_set64(__fd, t);
+	__fd->length29--;
+	__fd->cmd++;
+}
+
+/* The only part of the 'fd' we can't memcmp() is the ppid */
+static int fd_cmp(const struct qm_fd *a, const struct qm_fd *b)
+{
+	int r = (qm_fd_addr_get64(a) == qm_fd_addr_get64(b)) ? 0 : -1;
+	if (!r)
+		r = a->format - b->format;
+	if (!r)
+		r = a->opaque - b->opaque;
+	if (!r)
+		r = a->cmd - b->cmd;
+	return r;
+}
+
+/********/
+/* test */
+/********/
+
+static void do_enqueues(struct qman_fq *fq)
+{
+	unsigned int loop;
+	for (loop = 0; loop < NUM_ENQUEUES; loop++) {
+		if (qman_enqueue(fq, &fd, QMAN_ENQUEUE_FLAG_WAIT |
+				(((loop + 1) == NUM_ENQUEUES) ?
+				QMAN_ENQUEUE_FLAG_WAIT_SYNC : 0)))
+			panic("qman_enqueue() failed\n");
+		fd_inc(&fd);
+	}
+}
+
+void qman_test_high(void)
+{
+	unsigned int flags;
+	int res;
+	struct qman_fq *fq = &fq_base;
+
+	pr_info("qman_test_high starting\n");
+	fd_init(&fd);
+	fd_init(&fd_dq);
+
+	/* Initialise (parked) FQ */
+	if (qman_create_fq(0, FQ_FLAGS, fq))
+		panic("qman_create_fq() failed\n");
+	if (qman_init_fq(fq, QMAN_INITFQ_FLAG_LOCAL, NULL))
+		panic("qman_init_fq() failed\n");
+
+	/* Do enqueues + VDQCR, twice. (Parked FQ) */
+	do_enqueues(fq);
+	pr_info("VDQCR (till-empty);\n");
+	if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
+			QM_VDQCR_NUMFRAMES_TILLEMPTY))
+		panic("qman_volatile_dequeue() failed\n");
+	do_enqueues(fq);
+	pr_info("VDQCR (%d of %d);\n", NUM_PARTIAL, NUM_ENQUEUES);
+	if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
+			QM_VDQCR_NUMFRAMES_SET(NUM_PARTIAL)))
+		panic("qman_volatile_dequeue() failed\n");
+	pr_info("VDQCR (%d of %d);\n", NUM_ENQUEUES - NUM_PARTIAL,
+					NUM_ENQUEUES);
+	if (qman_volatile_dequeue(fq, VDQCR_FLAGS,
+			QM_VDQCR_NUMFRAMES_SET(NUM_ENQUEUES - NUM_PARTIAL)))
+		panic("qman_volatile_dequeue() failed\n");
+
+	do_enqueues(fq);
+	pr_info("scheduled dequeue (till-empty)\n");
+	if (qman_schedule_fq(fq))
+		panic("qman_schedule_fq() failed\n");
+	wait_event(waitqueue, sdqcr_complete);
+
+	/* Retire and OOS the FQ */
+	res = qman_retire_fq(fq, &flags);
+	if (res < 0)
+		panic("qman_retire_fq() failed\n");
+	wait_event(waitqueue, retire_complete);
+	if (flags & QMAN_FQ_STATE_BLOCKOOS)
+		panic("leaking frames\n");
+	if (qman_oos_fq(fq))
+		panic("qman_oos_fq() failed\n");
+	qman_destroy_fq(fq, 0);
+	pr_info("qman_test_high finished\n");
+}
+
+static enum qman_cb_dqrr_result cb_dqrr(struct qman_portal *p,
+					struct qman_fq *fq,
+					const struct qm_dqrr_entry *dq)
+{
+	if (fd_cmp(&fd_dq, &dq->fd)) {
+		pr_err("BADNESS: dequeued frame doesn't match;\n");
+		pr_err("Expected 0x%llx, got 0x%llx\n",
+		       (unsigned long long)fd_dq.length29,
+		       (unsigned long long)dq->fd.length29);
+		BUG();
+	}
+	fd_inc(&fd_dq);
+	if (!(dq->stat & QM_DQRR_STAT_UNSCHEDULED) && !fd_cmp(&fd_dq, &fd)) {
+		sdqcr_complete = 1;
+		wake_up(&waitqueue);
+	}
+	return qman_cb_dqrr_consume;
+}
+
+static void cb_ern(struct qman_portal *p, struct qman_fq *fq,
+			const struct qm_mr_entry *msg)
+{
+	panic("cb_ern() unimplemented");
+}
+
+static void cb_fqs(struct qman_portal *p, struct qman_fq *fq,
+			const struct qm_mr_entry *msg)
+{
+	u8 verb = (msg->verb & QM_MR_VERB_TYPE_MASK);
+	if ((verb != QM_MR_VERB_FQRN) && (verb != QM_MR_VERB_FQRNI))
+		panic("unexpected FQS message");
+	pr_info("Retirement message received\n");
+	retire_complete = 1;
+	wake_up(&waitqueue);
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_test_hotpotato.c
@@ -0,0 +1,499 @@
+/* Copyright 2009-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/kthread.h>
+#include <linux/platform_device.h>
+#include <linux/dma-mapping.h>
+#include "qman_test.h"
+
+/* Algorithm:
+ *
+ * Each cpu will have HP_PER_CPU "handlers" set up, each of which incorporates
+ * an rx/tx pair of FQ objects (both of which are stashed on dequeue). The
+ * organisation of FQIDs is such that the HP_PER_CPU*NUM_CPUS handlers will
+ * shuttle a "hot potato" frame around them such that every forwarding action
+ * moves it from one cpu to another. (The use of more than one handler per cpu
+ * is to allow enough handlers/FQs to truly test the significance of caching -
+ * ie. when cache-expiries are occurring.)
+ *
+ * The "hot potato" frame content will be HP_NUM_WORDS*4 bytes in size, and the
+ * first and last words of the frame data will undergo a transformation step on
+ * each forwarding action. To achieve this, each handler will be assigned a
+ * 32-bit "mixer", that is produced using a 32-bit LFSR. When a frame is
+ * received by a handler, the mixer of the expected sender is XOR'd into all
+ * words of the entire frame, which is then validated against the original
+ * values. Then, before forwarding, the entire frame is XOR'd with the mixer of
+ * the current handler. Apart from validating that the frame is taking the
+ * expected path, this also provides some quasi-realistic overheads to each
+ * forwarding action - dereferencing *all* the frame data, computation, and
+ * conditional branching. There is a "special" handler designated to act as the
+ * instigator of the test by creating an enqueuing the "hot potato" frame, and
+ * to determine when the test has completed by counting HP_LOOPS iterations.
+ *
+ * Init phases:
+ *
+ * 1. prepare each cpu's 'hp_cpu' struct using on_each_cpu(,,1) and link them
+ *    into 'hp_cpu_list'. Specifically, set processor_id, allocate HP_PER_CPU
+ *    handlers and link-list them (but do no other handler setup).
+ *
+ * 2. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each
+ *    hp_cpu's 'iterator' to point to its first handler. With each loop,
+ *    allocate rx/tx FQIDs and mixer values to the hp_cpu's iterator handler
+ *    and advance the iterator for the next loop. This includes a final fixup,
+ *    which connects the last handler to the first (and which is why phase 2
+ *    and 3 are separate).
+ *
+ * 3. scan over 'hp_cpu_list' HP_PER_CPU times, the first time sets each
+ *    hp_cpu's 'iterator' to point to its first handler. With each loop,
+ *    initialise FQ objects and advance the iterator for the next loop.
+ *    Moreover, do this initialisation on the cpu it applies to so that Rx FQ
+ *    initialisation targets the correct cpu.
+ */
+
+/* helper to run something on all cpus (can't use on_each_cpu(), as that invokes
+ * the fn from irq context, which is too restrictive). */
+struct bstrap {
+	void (*fn)(void);
+	atomic_t started;
+};
+static int bstrap_fn(void *__bstrap)
+{
+	struct bstrap *bstrap = __bstrap;
+	atomic_inc(&bstrap->started);
+	bstrap->fn();
+	while (!kthread_should_stop())
+		msleep(1);
+	return 0;
+}
+static int on_all_cpus(void (*fn)(void))
+{
+	int cpu;
+	for_each_cpu(cpu, cpu_online_mask) {
+		struct bstrap bstrap = {
+			.fn = fn,
+			.started = ATOMIC_INIT(0)
+		};
+		struct task_struct *k = kthread_create(bstrap_fn, &bstrap,
+			"hotpotato%d", cpu);
+		int ret;
+		if (IS_ERR(k))
+			return -ENOMEM;
+		kthread_bind(k, cpu);
+		wake_up_process(k);
+		/* If we call kthread_stop() before the "wake up" has had an
+		 * effect, then the thread may exit with -EINTR without ever
+		 * running the function. So poll until it's started before
+		 * requesting it to stop. */
+		while (!atomic_read(&bstrap.started))
+			msleep(10);
+		ret = kthread_stop(k);
+		if (ret)
+			return ret;
+	}
+	return 0;
+}
+
+struct hp_handler {
+
+	/* The following data is stashed when 'rx' is dequeued; */
+	/* -------------- */
+	/* The Rx FQ, dequeues of which will stash the entire hp_handler */
+	struct qman_fq rx;
+	/* The Tx FQ we should forward to */
+	struct qman_fq tx;
+	/* The value we XOR post-dequeue, prior to validating */
+	u32 rx_mixer;
+	/* The value we XOR pre-enqueue, after validating */
+	u32 tx_mixer;
+	/* what the hotpotato address should be on dequeue */
+	dma_addr_t addr;
+	u32 *frame_ptr;
+
+	/* The following data isn't (necessarily) stashed on dequeue; */
+	/* -------------- */
+	u32 fqid_rx, fqid_tx;
+	/* list node for linking us into 'hp_cpu' */
+	struct list_head node;
+	/* Just to check ... */
+	unsigned int processor_id;
+} ____cacheline_aligned;
+
+struct hp_cpu {
+	/* identify the cpu we run on; */
+	unsigned int processor_id;
+	/* root node for the per-cpu list of handlers */
+	struct list_head handlers;
+	/* list node for linking us into 'hp_cpu_list' */
+	struct list_head node;
+	/* when repeatedly scanning 'hp_list', each time linking the n'th
+	 * handlers together, this is used as per-cpu iterator state */
+	struct hp_handler *iterator;
+};
+
+/* Each cpu has one of these */
+static DEFINE_PER_CPU(struct hp_cpu, hp_cpus);
+
+/* links together the hp_cpu structs, in first-come first-serve order. */
+static LIST_HEAD(hp_cpu_list);
+static spinlock_t hp_lock = __SPIN_LOCK_UNLOCKED(hp_lock);
+
+static unsigned int hp_cpu_list_length;
+
+/* the "special" handler, that starts and terminates the test. */
+static struct hp_handler *special_handler;
+static int loop_counter;
+
+/* handlers are allocated out of this, so they're properly aligned. */
+static struct kmem_cache *hp_handler_slab;
+
+/* this is the frame data */
+static void *__frame_ptr;
+static u32 *frame_ptr;
+static dma_addr_t frame_dma;
+
+/* the main function waits on this */
+static DECLARE_WAIT_QUEUE_HEAD(queue);
+
+#define HP_PER_CPU	2
+#define HP_LOOPS	8
+/* 80 bytes, like a small ethernet frame, and bleeds into a second cacheline */
+#define HP_NUM_WORDS	80
+/* First word of the LFSR-based frame data */
+#define HP_FIRST_WORD	0xabbaf00d
+
+static inline u32 do_lfsr(u32 prev)
+{
+	return (prev >> 1) ^ (-(prev & 1u) & 0xd0000001u);
+}
+
+static void allocate_frame_data(void)
+{
+	u32 lfsr = HP_FIRST_WORD;
+	int loop;
+	struct platform_device *pdev = platform_device_alloc("foobar", -1);
+	if (!pdev)
+		panic("platform_device_alloc() failed");
+	if (platform_device_add(pdev))
+		panic("platform_device_add() failed");
+	__frame_ptr = kmalloc(4 * HP_NUM_WORDS, GFP_KERNEL);
+	if (!__frame_ptr)
+		panic("kmalloc() failed");
+	frame_ptr = (void *)(((unsigned long)__frame_ptr + 63) &
+				~(unsigned long)63);
+	for (loop = 0; loop < HP_NUM_WORDS; loop++) {
+		frame_ptr[loop] = lfsr;
+		lfsr = do_lfsr(lfsr);
+	}
+	frame_dma = dma_map_single(&pdev->dev, frame_ptr, 4 * HP_NUM_WORDS,
+					DMA_BIDIRECTIONAL);
+	platform_device_del(pdev);
+	platform_device_put(pdev);
+}
+
+static void deallocate_frame_data(void)
+{
+	kfree(__frame_ptr);
+}
+
+static inline void process_frame_data(struct hp_handler *handler,
+				const struct qm_fd *fd)
+{
+	u32 *p = handler->frame_ptr;
+	u32 lfsr = HP_FIRST_WORD;
+	int loop;
+	if (qm_fd_addr_get64(fd) != handler->addr)
+		panic("bad frame address");
+	for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) {
+		*p ^= handler->rx_mixer;
+		if (*p != lfsr)
+			panic("corrupt frame data");
+		*p ^= handler->tx_mixer;
+		lfsr = do_lfsr(lfsr);
+	}
+}
+
+static enum qman_cb_dqrr_result normal_dqrr(struct qman_portal *portal,
+					struct qman_fq *fq,
+					const struct qm_dqrr_entry *dqrr)
+{
+	struct hp_handler *handler = (struct hp_handler *)fq;
+
+	process_frame_data(handler, &dqrr->fd);
+	if (qman_enqueue(&handler->tx, &dqrr->fd, 0))
+		panic("qman_enqueue() failed");
+	return qman_cb_dqrr_consume;
+}
+
+static enum qman_cb_dqrr_result special_dqrr(struct qman_portal *portal,
+					struct qman_fq *fq,
+					const struct qm_dqrr_entry *dqrr)
+{
+	struct hp_handler *handler = (struct hp_handler *)fq;
+
+	process_frame_data(handler, &dqrr->fd);
+	if (++loop_counter < HP_LOOPS) {
+		if (qman_enqueue(&handler->tx, &dqrr->fd, 0))
+			panic("qman_enqueue() failed");
+	} else {
+		pr_info("Received final (%dth) frame\n", loop_counter);
+		wake_up(&queue);
+	}
+	return qman_cb_dqrr_consume;
+}
+
+static void create_per_cpu_handlers(void)
+{
+	struct hp_handler *handler;
+	int loop;
+	struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus);
+
+	hp_cpu->processor_id = smp_processor_id();
+	spin_lock(&hp_lock);
+	list_add_tail(&hp_cpu->node, &hp_cpu_list);
+	hp_cpu_list_length++;
+	spin_unlock(&hp_lock);
+	INIT_LIST_HEAD(&hp_cpu->handlers);
+	for (loop = 0; loop < HP_PER_CPU; loop++) {
+		handler = kmem_cache_alloc(hp_handler_slab, GFP_KERNEL);
+		if (!handler)
+			panic("kmem_cache_alloc() failed");
+		handler->processor_id = hp_cpu->processor_id;
+		handler->addr = frame_dma;
+		handler->frame_ptr = frame_ptr;
+		list_add_tail(&handler->node, &hp_cpu->handlers);
+	}
+	put_cpu_var(hp_cpus);
+}
+
+static void destroy_per_cpu_handlers(void)
+{
+	struct list_head *loop, *tmp;
+	struct hp_cpu *hp_cpu = &get_cpu_var(hp_cpus);
+
+	spin_lock(&hp_lock);
+	list_del(&hp_cpu->node);
+	spin_unlock(&hp_lock);
+	list_for_each_safe(loop, tmp, &hp_cpu->handlers) {
+		u32 flags;
+		struct hp_handler *handler = list_entry(loop, struct hp_handler,
+							node);
+		if (qman_retire_fq(&handler->rx, &flags))
+			panic("qman_retire_fq(rx) failed");
+		BUG_ON(flags & QMAN_FQ_STATE_BLOCKOOS);
+		if (qman_oos_fq(&handler->rx))
+			panic("qman_oos_fq(rx) failed");
+		qman_destroy_fq(&handler->rx, 0);
+		qman_destroy_fq(&handler->tx, 0);
+		qman_release_fqid(handler->fqid_rx);
+		list_del(&handler->node);
+		kmem_cache_free(hp_handler_slab, handler);
+	}
+	put_cpu_var(hp_cpus);
+}
+
+static inline u8 num_cachelines(u32 offset)
+{
+	u8 res = (offset + (L1_CACHE_BYTES - 1))
+			 / (L1_CACHE_BYTES);
+	if (res > 3)
+		return 3;
+	return res;
+}
+#define STASH_DATA_CL \
+	num_cachelines(HP_NUM_WORDS * 4)
+#define STASH_CTX_CL \
+	num_cachelines(offsetof(struct hp_handler, fqid_rx))
+
+static void init_handler(void *__handler)
+{
+	struct qm_mcc_initfq opts;
+	struct hp_handler *handler = __handler;
+	BUG_ON(handler->processor_id != smp_processor_id());
+	/* Set up rx */
+	memset(&handler->rx, 0, sizeof(handler->rx));
+	if (handler == special_handler)
+		handler->rx.cb.dqrr = special_dqrr;
+	else
+		handler->rx.cb.dqrr = normal_dqrr;
+	if (qman_create_fq(handler->fqid_rx, 0, &handler->rx))
+		panic("qman_create_fq(rx) failed");
+	memset(&opts, 0, sizeof(opts));
+	opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA;
+	opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING;
+	opts.fqd.context_a.stashing.data_cl = STASH_DATA_CL;
+	opts.fqd.context_a.stashing.context_cl = STASH_CTX_CL;
+	if (qman_init_fq(&handler->rx, QMAN_INITFQ_FLAG_SCHED |
+				QMAN_INITFQ_FLAG_LOCAL, &opts))
+		panic("qman_init_fq(rx) failed");
+	/* Set up tx */
+	memset(&handler->tx, 0, sizeof(handler->tx));
+	if (qman_create_fq(handler->fqid_tx, QMAN_FQ_FLAG_NO_MODIFY,
+				&handler->tx))
+		panic("qman_create_fq(tx) failed");
+}
+
+static void init_phase2(void)
+{
+	int loop;
+	u32 fqid = 0;
+	u32 lfsr = 0xdeadbeef;
+	struct hp_cpu *hp_cpu;
+	struct hp_handler *handler;
+
+	for (loop = 0; loop < HP_PER_CPU; loop++) {
+		list_for_each_entry(hp_cpu, &hp_cpu_list, node) {
+			int ret;
+			if (!loop)
+				hp_cpu->iterator = list_first_entry(
+						&hp_cpu->handlers,
+						struct hp_handler, node);
+			else
+				hp_cpu->iterator = list_entry(
+						hp_cpu->iterator->node.next,
+						struct hp_handler, node);
+			/* Rx FQID is the previous handler's Tx FQID */
+			hp_cpu->iterator->fqid_rx = fqid;
+			/* Allocate new FQID for Tx */
+			ret = qman_alloc_fqid(&fqid);
+			if (ret)
+				panic("qman_alloc_fqid() failed");
+			hp_cpu->iterator->fqid_tx = fqid;
+			/* Rx mixer is the previous handler's Tx mixer */
+			hp_cpu->iterator->rx_mixer = lfsr;
+			/* Get new mixer for Tx */
+			lfsr = do_lfsr(lfsr);
+			hp_cpu->iterator->tx_mixer = lfsr;
+		}
+	}
+	/* Fix up the first handler (fqid_rx==0, rx_mixer=0xdeadbeef) */
+	hp_cpu = list_first_entry(&hp_cpu_list, struct hp_cpu, node);
+	handler = list_first_entry(&hp_cpu->handlers, struct hp_handler, node);
+	BUG_ON((handler->fqid_rx != 0) || (handler->rx_mixer != 0xdeadbeef));
+	handler->fqid_rx = fqid;
+	handler->rx_mixer = lfsr;
+	/* and tag it as our "special" handler */
+	special_handler = handler;
+}
+
+static void init_phase3(void)
+{
+	int loop;
+	struct hp_cpu *hp_cpu;
+
+	for (loop = 0; loop < HP_PER_CPU; loop++) {
+		list_for_each_entry(hp_cpu, &hp_cpu_list, node) {
+			if (!loop)
+				hp_cpu->iterator = list_first_entry(
+						&hp_cpu->handlers,
+						struct hp_handler, node);
+			else
+				hp_cpu->iterator = list_entry(
+						hp_cpu->iterator->node.next,
+						struct hp_handler, node);
+			preempt_disable();
+			if (hp_cpu->processor_id == smp_processor_id())
+				init_handler(hp_cpu->iterator);
+			else
+				smp_call_function_single(hp_cpu->processor_id,
+					init_handler, hp_cpu->iterator, 1);
+			preempt_enable();
+		}
+	}
+}
+
+static void send_first_frame(void *ignore)
+{
+	u32 *p = special_handler->frame_ptr;
+	u32 lfsr = HP_FIRST_WORD;
+	int loop;
+	struct qm_fd fd;
+
+	BUG_ON(special_handler->processor_id != smp_processor_id());
+	memset(&fd, 0, sizeof(fd));
+	qm_fd_addr_set64(&fd, special_handler->addr);
+	fd.format = qm_fd_contig_big;
+	fd.length29 = HP_NUM_WORDS * 4;
+	for (loop = 0; loop < HP_NUM_WORDS; loop++, p++) {
+		if (*p != lfsr)
+			panic("corrupt frame data");
+		*p ^= special_handler->tx_mixer;
+		lfsr = do_lfsr(lfsr);
+	}
+	pr_info("Sending first frame\n");
+	if (qman_enqueue(&special_handler->tx, &fd, 0))
+		panic("qman_enqueue() failed");
+}
+
+void qman_test_hotpotato(void)
+{
+	if (cpumask_weight(cpu_online_mask) < 2) {
+		pr_info("qman_test_hotpotato, skip - only 1 CPU\n");
+		return;
+	}
+
+	pr_info("qman_test_hotpotato starting\n");
+
+	hp_cpu_list_length = 0;
+	loop_counter = 0;
+	hp_handler_slab = kmem_cache_create("hp_handler_slab",
+			sizeof(struct hp_handler), L1_CACHE_BYTES,
+			SLAB_HWCACHE_ALIGN, NULL);
+	if (!hp_handler_slab)
+		panic("kmem_cache_create() failed");
+
+	allocate_frame_data();
+
+	/* Init phase 1 */
+	pr_info("Creating %d handlers per cpu...\n", HP_PER_CPU);
+	if (on_all_cpus(create_per_cpu_handlers))
+		panic("on_each_cpu() failed");
+	pr_info("Number of cpus: %d, total of %d handlers\n",
+		hp_cpu_list_length, hp_cpu_list_length * HP_PER_CPU);
+
+	init_phase2();
+
+	init_phase3();
+
+	preempt_disable();
+	if (special_handler->processor_id == smp_processor_id())
+		send_first_frame(NULL);
+	else
+		smp_call_function_single(special_handler->processor_id,
+			send_first_frame, NULL, 1);
+	preempt_enable();
+
+	wait_event(queue, loop_counter == HP_LOOPS);
+	deallocate_frame_data();
+	if (on_all_cpus(destroy_per_cpu_handlers))
+		panic("on_each_cpu() failed");
+	kmem_cache_destroy(hp_handler_slab);
+	pr_info("qman_test_hotpotato finished\n");
+}
--- /dev/null
+++ b/drivers/staging/fsl_qbman/qman_utility.c
@@ -0,0 +1,129 @@
+/* Copyright 2008-2011 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "qman_private.h"
+
+/* ----------------- */
+/* --- FQID Pool --- */
+
+struct qman_fqid_pool {
+	/* Base and size of the FQID range */
+	u32 fqid_base;
+	u32 total;
+	/* Number of FQIDs currently "allocated" */
+	u32 used;
+	/* Allocation optimisation. When 'used<total', it is the index of an
+	 * available FQID. Otherwise there are no available FQIDs, and this
+	 * will be set when the next deallocation occurs. */
+	u32 next;
+	/* A bit-field representation of the FQID range. */
+	unsigned long *bits;
+};
+
+#define QLONG_BYTES	sizeof(unsigned long)
+#define QLONG_BITS	(QLONG_BYTES * 8)
+/* Number of 'longs' required for the given number of bits */
+#define QNUM_LONGS(b)	(((b) + QLONG_BITS - 1) / QLONG_BITS)
+/* Shorthand for the number of bytes of same (kmalloc, memset, etc) */
+#define QNUM_BYTES(b)	(QNUM_LONGS(b) * QLONG_BYTES)
+/* And in bits */
+#define QNUM_BITS(b)	(QNUM_LONGS(b) * QLONG_BITS)
+
+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num)
+{
+	struct qman_fqid_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL);
+	unsigned int i;
+
+	BUG_ON(!num);
+	if (!pool)
+		return NULL;
+	pool->fqid_base = fqid_start;
+	pool->total = num;
+	pool->used = 0;
+	pool->next = 0;
+	pool->bits = kzalloc(QNUM_BYTES(num), GFP_KERNEL);
+	if (!pool->bits) {
+		kfree(pool);
+		return NULL;
+	}
+	/* If num is not an even multiple of QLONG_BITS (or even 8, for
+	 * byte-oriented searching) then we fill the trailing bits with 1, to
+	 * make them look allocated (permanently). */
+	for (i = num + 1; i < QNUM_BITS(num); i++)
+		set_bit(i, pool->bits);
+	return pool;
+}
+EXPORT_SYMBOL(qman_fqid_pool_create);
+
+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool)
+{
+	int ret = pool->used;
+	kfree(pool->bits);
+	kfree(pool);
+	return ret;
+}
+EXPORT_SYMBOL(qman_fqid_pool_destroy);
+
+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid)
+{
+	int ret;
+	if (pool->used == pool->total)
+		return -ENOMEM;
+	*fqid = pool->fqid_base + pool->next;
+	ret = test_and_set_bit(pool->next, pool->bits);
+	BUG_ON(ret);
+	if (++pool->used == pool->total)
+		return 0;
+	pool->next = find_next_zero_bit(pool->bits, pool->total, pool->next);
+	if (pool->next >= pool->total)
+		pool->next = find_first_zero_bit(pool->bits, pool->total);
+	BUG_ON(pool->next >= pool->total);
+	return 0;
+}
+EXPORT_SYMBOL(qman_fqid_pool_alloc);
+
+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid)
+{
+	int ret;
+
+	fqid -= pool->fqid_base;
+	ret = test_and_clear_bit(fqid, pool->bits);
+	BUG_ON(!ret);
+	if (pool->used-- == pool->total)
+		pool->next = fqid;
+}
+EXPORT_SYMBOL(qman_fqid_pool_free);
+
+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool)
+{
+	return pool->used;
+}
+EXPORT_SYMBOL(qman_fqid_pool_used);
--- /dev/null
+++ b/include/linux/fsl_bman.h
@@ -0,0 +1,532 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FSL_BMAN_H
+#define FSL_BMAN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Last updated for v00.79 of the BG */
+
+/* Portal processing (interrupt) sources */
+#define BM_PIRQ_RCRI	0x00000002	/* RCR Ring (below threshold) */
+#define BM_PIRQ_BSCN	0x00000001	/* Buffer depletion State Change */
+
+/* This wrapper represents a bit-array for the depletion state of the 64 Bman
+ * buffer pools. */
+struct bman_depletion {
+	u32 __state[2];
+};
+#define BMAN_DEPLETION_EMPTY { { 0x00000000, 0x00000000 } }
+#define BMAN_DEPLETION_FULL { { 0xffffffff, 0xffffffff } }
+#define __bmdep_word(x) ((x) >> 5)
+#define __bmdep_shift(x) ((x) & 0x1f)
+#define __bmdep_bit(x) (0x80000000 >> __bmdep_shift(x))
+static inline void bman_depletion_init(struct bman_depletion *c)
+{
+	c->__state[0] = c->__state[1] = 0;
+}
+static inline void bman_depletion_fill(struct bman_depletion *c)
+{
+	c->__state[0] = c->__state[1] = ~0;
+}
+static inline int bman_depletion_get(const struct bman_depletion *c, u8 bpid)
+{
+	return c->__state[__bmdep_word(bpid)] & __bmdep_bit(bpid);
+}
+static inline void bman_depletion_set(struct bman_depletion *c, u8 bpid)
+{
+	c->__state[__bmdep_word(bpid)] |= __bmdep_bit(bpid);
+}
+static inline void bman_depletion_unset(struct bman_depletion *c, u8 bpid)
+{
+	c->__state[__bmdep_word(bpid)] &= ~__bmdep_bit(bpid);
+}
+
+/* ------------------------------------------------------- */
+/* --- Bman data structures (and associated constants) --- */
+
+/* Represents s/w corenet portal mapped data structures */
+struct bm_rcr_entry;	/* RCR (Release Command Ring) entries */
+struct bm_mc_command;	/* MC (Management Command) command */
+struct bm_mc_result;	/* MC result */
+
+/* Code-reduction, define a wrapper for 48-bit buffers. In cases where a buffer
+ * pool id specific to this buffer is needed (BM_RCR_VERB_CMD_BPID_MULTI,
+ * BM_MCC_VERB_ACQUIRE), the 'bpid' field is used. */
+struct bm_buffer {
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 __reserved1;
+			u8 bpid;
+			u16 hi; /* High 16-bits of 48-bit address */
+			u32 lo; /* Low 32-bits of 48-bit address */
+#else
+			u32 lo;
+			u16 hi;
+			u8 bpid;
+			u8 __reserved;
+#endif
+		};
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u64 __notaddress:16;
+			u64 addr:48;
+#else
+			u64 addr:48;
+			u64 __notaddress:16;
+#endif
+		};
+		u64 opaque;
+	};
+} __aligned(8);
+static inline u64 bm_buffer_get64(const struct bm_buffer *buf)
+{
+	return buf->addr;
+}
+static inline dma_addr_t bm_buf_addr(const struct bm_buffer *buf)
+{
+	return (dma_addr_t)buf->addr;
+}
+/* Macro, so we compile better if 'v' isn't always 64-bit */
+#define bm_buffer_set64(buf, v) \
+	do { \
+		struct bm_buffer *__buf931 = (buf); \
+		__buf931->hi = upper_32_bits(v); \
+		__buf931->lo = lower_32_bits(v); \
+	} while (0)
+
+/* See 1.5.3.5.4: "Release Command" */
+struct bm_rcr_entry {
+	union {
+		struct {
+			u8 __dont_write_directly__verb;
+			u8 bpid; /* used with BM_RCR_VERB_CMD_BPID_SINGLE */
+			u8 __reserved1[62];
+		};
+		struct bm_buffer bufs[8];
+	};
+} __packed;
+#define BM_RCR_VERB_VBIT		0x80
+#define BM_RCR_VERB_CMD_MASK		0x70	/* one of two values; */
+#define BM_RCR_VERB_CMD_BPID_SINGLE	0x20
+#define BM_RCR_VERB_CMD_BPID_MULTI	0x30
+#define BM_RCR_VERB_BUFCOUNT_MASK	0x0f	/* values 1..8 */
+
+/* See 1.5.3.1: "Acquire Command" */
+/* See 1.5.3.2: "Query Command" */
+struct bm_mcc_acquire {
+	u8 bpid;
+	u8 __reserved1[62];
+} __packed;
+struct bm_mcc_query {
+	u8 __reserved2[63];
+} __packed;
+struct bm_mc_command {
+	u8 __dont_write_directly__verb;
+	union {
+		struct bm_mcc_acquire acquire;
+		struct bm_mcc_query query;
+	};
+} __packed;
+#define BM_MCC_VERB_VBIT		0x80
+#define BM_MCC_VERB_CMD_MASK		0x70	/* where the verb contains; */
+#define BM_MCC_VERB_CMD_ACQUIRE		0x10
+#define BM_MCC_VERB_CMD_QUERY		0x40
+#define BM_MCC_VERB_ACQUIRE_BUFCOUNT	0x0f	/* values 1..8 go here */
+
+/* See 1.5.3.3: "Acquire Response" */
+/* See 1.5.3.4: "Query Response" */
+struct bm_pool_state {
+	u8 __reserved1[32];
+	/* "availability state" and "depletion state" */
+	struct {
+		u8 __reserved1[8];
+		/* Access using bman_depletion_***() */
+		struct bman_depletion state;
+	} as, ds;
+};
+struct bm_mc_result {
+	union {
+		struct {
+			u8 verb;
+			u8 __reserved1[63];
+		};
+		union {
+			struct {
+				u8 __reserved1;
+				u8 bpid;
+				u8 __reserved2[62];
+			};
+			struct bm_buffer bufs[8];
+		} acquire;
+		struct bm_pool_state query;
+	};
+} __packed;
+#define BM_MCR_VERB_VBIT		0x80
+#define BM_MCR_VERB_CMD_MASK		BM_MCC_VERB_CMD_MASK
+#define BM_MCR_VERB_CMD_ACQUIRE		BM_MCC_VERB_CMD_ACQUIRE
+#define BM_MCR_VERB_CMD_QUERY		BM_MCC_VERB_CMD_QUERY
+#define BM_MCR_VERB_CMD_ERR_INVALID	0x60
+#define BM_MCR_VERB_CMD_ERR_ECC		0x70
+#define BM_MCR_VERB_ACQUIRE_BUFCOUNT	BM_MCC_VERB_ACQUIRE_BUFCOUNT /* 0..8 */
+/* Determine the "availability state" of pool 'p' from a query result 'r' */
+#define BM_MCR_QUERY_AVAILABILITY(r, p)	\
+		bman_depletion_get(&r->query.as.state, p)
+/* Determine the "depletion state" of pool 'p' from a query result 'r' */
+#define BM_MCR_QUERY_DEPLETION(r, p)	\
+		bman_depletion_get(&r->query.ds.state, p)
+
+/*******************************************************************/
+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */
+/*******************************************************************/
+
+	/* Portal and Buffer Pools */
+	/* ----------------------- */
+/* Represents a managed portal */
+struct bman_portal;
+
+/* This object type represents Bman buffer pools. */
+struct bman_pool;
+
+struct bman_portal_config {
+	/* This is used for any "core-affine" portals, ie. default portals
+	 * associated to the corresponding cpu. -1 implies that there is no core
+	 * affinity configured. */
+	int cpu;
+	/* portal interrupt line */
+	int irq;
+	/* the unique index of this portal */
+	u32 index;
+	/* Is this portal shared? (If so, it has coarser locking and demuxes
+	 * processing on behalf of other CPUs.) */
+	int is_shared;
+	/* These are the buffer pool IDs that may be used via this portal. */
+	struct bman_depletion mask;
+};
+
+/* This callback type is used when handling pool depletion entry/exit. The
+ * 'cb_ctx' value is the opaque value associated with the pool object in
+ * bman_new_pool(). 'depleted' is non-zero on depletion-entry, and zero on
+ * depletion-exit. */
+typedef void (*bman_cb_depletion)(struct bman_portal *bm,
+			struct bman_pool *pool, void *cb_ctx, int depleted);
+
+/* This struct specifies parameters for a bman_pool object. */
+struct bman_pool_params {
+	/* index of the buffer pool to encapsulate (0-63), ignored if
+	 * BMAN_POOL_FLAG_DYNAMIC_BPID is set. */
+	u32 bpid;
+	/* bit-mask of BMAN_POOL_FLAG_*** options */
+	u32 flags;
+	/* depletion-entry/exit callback, if BMAN_POOL_FLAG_DEPLETION is set */
+	bman_cb_depletion cb;
+	/* opaque user value passed as a parameter to 'cb' */
+	void *cb_ctx;
+	/* depletion-entry/exit thresholds, if BMAN_POOL_FLAG_THRESH is set. NB:
+	 * this is only allowed if BMAN_POOL_FLAG_DYNAMIC_BPID is used *and*
+	 * when run in the control plane (which controls Bman CCSR). This array
+	 * matches the definition of bm_pool_set(). */
+	u32 thresholds[4];
+};
+
+/* Flags to bman_new_pool() */
+#define BMAN_POOL_FLAG_NO_RELEASE    0x00000001 /* can't release to pool */
+#define BMAN_POOL_FLAG_ONLY_RELEASE  0x00000002 /* can only release to pool */
+#define BMAN_POOL_FLAG_DEPLETION     0x00000004 /* track depletion entry/exit */
+#define BMAN_POOL_FLAG_DYNAMIC_BPID  0x00000008 /* (de)allocate bpid */
+#define BMAN_POOL_FLAG_THRESH        0x00000010 /* set depletion thresholds */
+#define BMAN_POOL_FLAG_STOCKPILE     0x00000020 /* stockpile to reduce hw ops */
+
+/* Flags to bman_release() */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+#define BMAN_RELEASE_FLAG_WAIT       0x00000001 /* wait if RCR is full */
+#define BMAN_RELEASE_FLAG_WAIT_INT   0x00000002 /* if we wait, interruptible? */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+#define BMAN_RELEASE_FLAG_WAIT_SYNC  0x00000004 /* if wait, until consumed? */
+#endif
+#endif
+#define BMAN_RELEASE_FLAG_NOW        0x00000008 /* issue immediate release */
+
+/* Flags to bman_acquire() */
+#define BMAN_ACQUIRE_FLAG_STOCKPILE  0x00000001 /* no hw op, stockpile only */
+
+	/* Portal Management */
+	/* ----------------- */
+/**
+ * bman_get_portal_config - get portal configuration settings
+ *
+ * This returns a read-only view of the current cpu's affine portal settings.
+ */
+const struct bman_portal_config *bman_get_portal_config(void);
+
+/**
+ * bman_irqsource_get - return the portal work that is interrupt-driven
+ *
+ * Returns a bitmask of BM_PIRQ_**I processing sources that are currently
+ * enabled for interrupt handling on the current cpu's affine portal. These
+ * sources will trigger the portal interrupt and the interrupt handler (or a
+ * tasklet/bottom-half it defers to) will perform the corresponding processing
+ * work. The bman_poll_***() functions will only process sources that are not in
+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU,
+ * this always returns zero.
+ */
+u32 bman_irqsource_get(void);
+
+/**
+ * bman_irqsource_add - add processing sources to be interrupt-driven
+ * @bits: bitmask of BM_PIRQ_**I processing sources
+ *
+ * Adds processing sources that should be interrupt-driven (rather than
+ * processed via bman_poll_***() functions). Returns zero for success, or
+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU. */
+int bman_irqsource_add(u32 bits);
+
+/**
+ * bman_irqsource_remove - remove processing sources from being interrupt-driven
+ * @bits: bitmask of BM_PIRQ_**I processing sources
+ *
+ * Removes processing sources from being interrupt-driven, so that they will
+ * instead be processed via bman_poll_***() functions. Returns zero for success,
+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU. */
+int bman_irqsource_remove(u32 bits);
+
+/**
+ * bman_affine_cpus - return a mask of cpus that have affine portals
+ */
+const cpumask_t *bman_affine_cpus(void);
+
+/**
+ * bman_poll_slow - process anything that isn't interrupt-driven.
+ *
+ * This function does any portal processing that isn't interrupt-driven. If the
+ * current CPU is sharing a portal hosted on another CPU, this function will
+ * return -EINVAL, otherwise the return value is a bitmask of BM_PIRQ_* sources
+ * indicating what interrupt sources were actually processed by the call.
+ *
+ * NB, unlike the legacy wrapper bman_poll(), this function will
+ * deterministically check for the presence of portal processing work and do it,
+ * which implies some latency even if there's nothing to do. The bman_poll()
+ * wrapper on the other hand (like the qman_poll() wrapper) attenuates this by
+ * checking for (and doing) portal processing infrequently. Ie. such that
+ * qman_poll() and bman_poll() can be called from core-processing loops. Use
+ * bman_poll_slow() when you yourself are deciding when to incur the overhead of
+ * processing.
+ */
+u32 bman_poll_slow(void);
+
+/**
+ * bman_poll - process anything that isn't interrupt-driven.
+ *
+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the
+ * affine portal. This function does whatever processing is not triggered by
+ * interrupts. This is a legacy wrapper that can be used in core-processing
+ * loops but mitigates the performance overhead of portal processing by
+ * adaptively bypassing true portal processing most of the time. (Processing is
+ * done once every 10 calls if the previous processing revealed that work needed
+ * to be done, or once very 1000 calls if the previous processing revealed no
+ * work needed doing.) If you wish to control this yourself, call
+ * bman_poll_slow() instead, which always checks for portal processing work.
+ */
+void bman_poll(void);
+
+/**
+ * bman_rcr_is_empty - Determine if portal's RCR is empty
+ *
+ * For use in situations where a cpu-affine caller needs to determine when all
+ * releases for the local portal have been processed by Bman but can't use the
+ * BMAN_RELEASE_FLAG_WAIT_SYNC flag to do this from the final bman_release().
+ * The function forces tracking of RCR consumption (which normally doesn't
+ * happen until release processing needs to find space to put new release
+ * commands), and returns zero if the ring still has unprocessed entries,
+ * non-zero if it is empty.
+ */
+int bman_rcr_is_empty(void);
+
+/**
+ * bman_alloc_bpid_range - Allocate a contiguous range of BPIDs
+ * @result: is set by the API to the base BPID of the allocated range
+ * @count: the number of BPIDs required
+ * @align: required alignment of the allocated range
+ * @partial: non-zero if the API can return fewer than @count BPIDs
+ *
+ * Returns the number of buffer pools allocated, or a negative error code. If
+ * @partial is non zero, the allocation request may return a smaller range of
+ * BPs than requested (though alignment will be as requested). If @partial is
+ * zero, the return value will either be 'count' or negative.
+ */
+int bman_alloc_bpid_range(u32 *result, u32 count, u32 align, int partial);
+static inline int bman_alloc_bpid(u32 *result)
+{
+	int ret = bman_alloc_bpid_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+
+/**
+ * bman_release_bpid_range - Release the specified range of buffer pool IDs
+ * @bpid: the base BPID of the range to deallocate
+ * @count: the number of BPIDs in the range
+ *
+ * This function can also be used to seed the allocator with ranges of BPIDs
+ * that it can subsequently allocate from.
+ */
+void bman_release_bpid_range(u32 bpid, unsigned int count);
+static inline void bman_release_bpid(u32 bpid)
+{
+	bman_release_bpid_range(bpid, 1);
+}
+
+int bman_reserve_bpid_range(u32 bpid, unsigned int count);
+static inline int bman_reserve_bpid(u32 bpid)
+{
+	return bman_reserve_bpid_range(bpid, 1);
+}
+
+void bman_seed_bpid_range(u32 bpid, unsigned int count);
+
+
+int bman_shutdown_pool(u32 bpid);
+
+	/* Pool management */
+	/* --------------- */
+/**
+ * bman_new_pool - Allocates a Buffer Pool object
+ * @params: parameters specifying the buffer pool ID and behaviour
+ *
+ * Creates a pool object for the given @params. A portal and the depletion
+ * callback field of @params are only used if the BMAN_POOL_FLAG_DEPLETION flag
+ * is set. NB, the fields from @params are copied into the new pool object, so
+ * the structure provided by the caller can be released or reused after the
+ * function returns.
+ */
+struct bman_pool *bman_new_pool(const struct bman_pool_params *params);
+
+/**
+ * bman_free_pool - Deallocates a Buffer Pool object
+ * @pool: the pool object to release
+ *
+ */
+void bman_free_pool(struct bman_pool *pool);
+
+/**
+ * bman_get_params - Returns a pool object's parameters.
+ * @pool: the pool object
+ *
+ * The returned pointer refers to state within the pool object so must not be
+ * modified and can no longer be read once the pool object is destroyed.
+ */
+const struct bman_pool_params *bman_get_params(const struct bman_pool *pool);
+
+/**
+ * bman_release - Release buffer(s) to the buffer pool
+ * @pool: the buffer pool object to release to
+ * @bufs: an array of buffers to release
+ * @num: the number of buffers in @bufs (1-8)
+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options
+ *
+ * Adds the given buffers to RCR entries. If the portal @p was created with the
+ * "COMPACT" flag, then it will be using a compaction algorithm to improve
+ * utilisation of RCR. As such, these buffers may join an existing ring entry
+ * and/or it may not be issued right away so as to allow future releases to join
+ * the same ring entry. Use the BMAN_RELEASE_FLAG_NOW flag to override this
+ * behaviour by committing the RCR entry (or entries) right away. If the RCR
+ * ring is full, the function will return -EBUSY unless BMAN_RELEASE_FLAG_WAIT
+ * is selected, in which case it will sleep waiting for space to become
+ * available in RCR. If the function receives a signal before such time (and
+ * BMAN_RELEASE_FLAG_WAIT_INT is set), the function returns -EINTR. Otherwise,
+ * it returns zero.
+ */
+int bman_release(struct bman_pool *pool, const struct bm_buffer *bufs, u8 num,
+			u32 flags);
+
+/**
+ * bman_acquire - Acquire buffer(s) from a buffer pool
+ * @pool: the buffer pool object to acquire from
+ * @bufs: array for storing the acquired buffers
+ * @num: the number of buffers desired (@bufs is at least this big)
+ *
+ * Issues an "Acquire" command via the portal's management command interface.
+ * The return value will be the number of buffers obtained from the pool, or a
+ * negative error code if a h/w error or pool starvation was encountered. In
+ * the latter case, the content of @bufs is undefined.
+ */
+int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num,
+			u32 flags);
+
+/**
+ * bman_flush_stockpile - Flush stockpile buffer(s) to the buffer pool
+ * @pool: the buffer pool object the stockpile belongs
+ * @flags: bit-mask of BMAN_RELEASE_FLAG_*** options
+ *
+ * Adds stockpile buffers to RCR entries until the stockpile is empty.
+ * The return value will be a negative error code if a h/w error occurred.
+ * If BMAN_RELEASE_FLAG_NOW flag is passed and RCR ring is full,
+ * -EAGAIN will be returned.
+ */
+int bman_flush_stockpile(struct bman_pool *pool, u32 flags);
+
+/**
+ * bman_query_pools - Query all buffer pool states
+ * @state: storage for the queried availability and depletion states
+ */
+int bman_query_pools(struct bm_pool_state *state);
+
+#ifdef CONFIG_FSL_BMAN_CONFIG
+/**
+ * bman_query_free_buffers - Query how many free buffers are in buffer pool
+ * @pool: the buffer pool object to query
+ *
+ * Return the number of the free buffers
+ */
+u32 bman_query_free_buffers(struct bman_pool *pool);
+
+/**
+ * bman_update_pool_thresholds - Change the buffer pool's depletion thresholds
+ * @pool: the buffer pool object to which the thresholds will be set
+ * @thresholds: the new thresholds
+ */
+int bman_update_pool_thresholds(struct bman_pool *pool, const u32 *thresholds);
+#endif
+
+/**
+ * The below bman_p_***() variant might be called in a situation that the cpu
+ * which the portal affine to is not online yet.
+ * @bman_portal specifies which portal the API will use.
+*/
+int bman_p_irqsource_add(struct bman_portal *p, __maybe_unused u32 bits);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FSL_BMAN_H */
--- /dev/null
+++ b/include/linux/fsl_qman.h
@@ -0,0 +1,3889 @@
+/* Copyright 2008-2012 Freescale Semiconductor, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *     * Redistributions of source code must retain the above copyright
+ *       notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright
+ *       notice, this list of conditions and the following disclaimer in the
+ *       documentation and/or other materials provided with the distribution.
+ *     * Neither the name of Freescale Semiconductor nor the
+ *       names of its contributors may be used to endorse or promote products
+ *       derived from this software without specific prior written permission.
+ *
+ *
+ * ALTERNATIVELY, this software may be distributed under the terms of the
+ * GNU General Public License ("GPL") as published by the Free Software
+ * Foundation, either version 2 of that License or (at your option) any
+ * later version.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef FSL_QMAN_H
+#define FSL_QMAN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Last updated for v00.800 of the BG */
+
+/* Hardware constants */
+#define QM_CHANNEL_SWPORTAL0 0
+#define QMAN_CHANNEL_POOL1 0x21
+#define QMAN_CHANNEL_CAAM 0x80
+#define QMAN_CHANNEL_PME 0xa0
+#define QMAN_CHANNEL_POOL1_REV3 0x401
+#define QMAN_CHANNEL_CAAM_REV3 0x840
+#define QMAN_CHANNEL_PME_REV3 0x860
+#define QMAN_CHANNEL_DCE 0x8a0
+#define QMAN_CHANNEL_DCE_QMANREV312 0x880
+extern u16 qm_channel_pool1;
+extern u16 qm_channel_caam;
+extern u16 qm_channel_pme;
+extern u16 qm_channel_dce;
+enum qm_dc_portal {
+	qm_dc_portal_fman0 = 0,
+	qm_dc_portal_fman1 = 1,
+	qm_dc_portal_caam = 2,
+	qm_dc_portal_pme = 3,
+	qm_dc_portal_rman = 4,
+	qm_dc_portal_dce = 5
+};
+
+/* Portal processing (interrupt) sources */
+#define QM_PIRQ_CCSCI	0x00200000	/* CEETM Congestion State Change */
+#define QM_PIRQ_CSCI	0x00100000	/* Congestion State Change */
+#define QM_PIRQ_EQCI	0x00080000	/* Enqueue Command Committed */
+#define QM_PIRQ_EQRI	0x00040000	/* EQCR Ring (below threshold) */
+#define QM_PIRQ_DQRI	0x00020000	/* DQRR Ring (non-empty) */
+#define QM_PIRQ_MRI	0x00010000	/* MR Ring (non-empty) */
+/* This mask contains all the interrupt sources that need handling except DQRI,
+ * ie. that if present should trigger slow-path processing. */
+#define QM_PIRQ_SLOW	(QM_PIRQ_CSCI | QM_PIRQ_EQCI | QM_PIRQ_EQRI | \
+			QM_PIRQ_MRI | QM_PIRQ_CCSCI)
+
+/* --- Clock speed --- */
+/* A qman driver instance may or may not know the current qman clock speed.
+ * However, certain CEETM calculations may not be possible if this is not known.
+ * The 'set' function will only succeed (return zero) if the driver did not
+ * already know the clock speed. Likewise, the 'get' function will only succeed
+ * if the driver does know the clock speed (either because it knew when booting,
+ * or was told via 'set'). In cases where software is running on a driver
+ * instance that does not know the clock speed (eg. on a hypervised data-plane),
+ * and the user can obtain the current qman clock speed by other means (eg. from
+ * a message sent from the control-plane), then the 'set' function can be used
+ * to enable rate-calculations in a driver where it would otherwise not be
+ * possible. */
+int qm_get_clock(u64 *clock_hz);
+int qm_set_clock(u64 clock_hz);
+
+/* For qman_static_dequeue_*** APIs */
+#define QM_SDQCR_CHANNELS_POOL_MASK	0x00007fff
+/* for n in [1,15] */
+#define QM_SDQCR_CHANNELS_POOL(n)	(0x00008000 >> (n))
+/* for conversion from n of qm_channel */
+static inline u32 QM_SDQCR_CHANNELS_POOL_CONV(u16 channel)
+{
+	return QM_SDQCR_CHANNELS_POOL(channel + 1 - qm_channel_pool1);
+}
+
+/* For qman_volatile_dequeue(); Choose one PRECEDENCE. EXACT is optional. Use
+ * NUMFRAMES(n) (6-bit) or NUMFRAMES_TILLEMPTY to fill in the frame-count. Use
+ * FQID(n) to fill in the frame queue ID. */
+#define QM_VDQCR_PRECEDENCE_VDQCR	0x0
+#define QM_VDQCR_PRECEDENCE_SDQCR	0x80000000
+#define QM_VDQCR_EXACT			0x40000000
+#define QM_VDQCR_NUMFRAMES_MASK		0x3f000000
+#define QM_VDQCR_NUMFRAMES_SET(n)	(((n) & 0x3f) << 24)
+#define QM_VDQCR_NUMFRAMES_GET(n)	(((n) >> 24) & 0x3f)
+#define QM_VDQCR_NUMFRAMES_TILLEMPTY	QM_VDQCR_NUMFRAMES_SET(0)
+
+
+/* ------------------------------------------------------- */
+/* --- Qman data structures (and associated constants) --- */
+
+/* Represents s/w corenet portal mapped data structures */
+struct qm_eqcr_entry;	/* EQCR (EnQueue Command Ring) entries */
+struct qm_dqrr_entry;	/* DQRR (DeQueue Response Ring) entries */
+struct qm_mr_entry;	/* MR (Message Ring) entries */
+struct qm_mc_command;	/* MC (Management Command) command */
+struct qm_mc_result;	/* MC result */
+
+/* See David Lapp's "Frame formats" document, "dpateam", Jan 07, 2008 */
+#define QM_FD_FORMAT_SG		0x4
+#define QM_FD_FORMAT_LONG	0x2
+#define QM_FD_FORMAT_COMPOUND	0x1
+enum qm_fd_format {
+	/* 'contig' implies a contiguous buffer, whereas 'sg' implies a
+	 * scatter-gather table. 'big' implies a 29-bit length with no offset
+	 * field, otherwise length is 20-bit and offset is 9-bit. 'compound'
+	 * implies a s/g-like table, where each entry itself represents a frame
+	 * (contiguous or scatter-gather) and the 29-bit "length" is
+	 * interpreted purely for congestion calculations, ie. a "congestion
+	 * weight". */
+	qm_fd_contig = 0,
+	qm_fd_contig_big = QM_FD_FORMAT_LONG,
+	qm_fd_sg = QM_FD_FORMAT_SG,
+	qm_fd_sg_big = QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG,
+	qm_fd_compound = QM_FD_FORMAT_COMPOUND
+};
+
+/* Capitalised versions are un-typed but can be used in static expressions */
+#define QM_FD_CONTIG	0
+#define QM_FD_CONTIG_BIG QM_FD_FORMAT_LONG
+#define QM_FD_SG	QM_FD_FORMAT_SG
+#define QM_FD_SG_BIG	(QM_FD_FORMAT_SG | QM_FD_FORMAT_LONG)
+#define QM_FD_COMPOUND	QM_FD_FORMAT_COMPOUND
+
+/* See 1.5.1.1: "Frame Descriptor (FD)" */
+struct qm_fd {
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 dd:2;	/* dynamic debug */
+			u8 liodn_offset:6;
+			u8 bpid:8;	/* Buffer Pool ID */
+			u8 eliodn_offset:4;
+			u8 __reserved:4;
+			u8 addr_hi;	/* high 8-bits of 40-bit address */
+			u32 addr_lo;	/* low 32-bits of 40-bit address */
+#else
+			u8 liodn_offset:6;
+			u8 dd:2;	/* dynamic debug */
+			u8 bpid:8;	/* Buffer Pool ID */
+			u8 __reserved:4;
+			u8 eliodn_offset:4;
+			u8 addr_hi;	/* high 8-bits of 40-bit address */
+			u32 addr_lo;	/* low 32-bits of 40-bit address */
+#endif
+		};
+		struct {
+			u64 __notaddress:24;
+			/* More efficient address accessor */
+			u64 addr:40;
+		};
+		u64 opaque_addr;
+	};
+	/* The 'format' field indicates the interpretation of the remaining 29
+	 * bits of the 32-bit word. For packing reasons, it is duplicated in the
+	 * other union elements. Note, union'd structs are difficult to use with
+	 * static initialisation under gcc, in which case use the "opaque" form
+	 * with one of the macros. */
+	union {
+		/* For easier/faster copying of this part of the fd (eg. from a
+		 * DQRR entry to an EQCR entry) copy 'opaque' */
+		u32 opaque;
+		/* If 'format' is _contig or _sg, 20b length and 9b offset */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			enum qm_fd_format format:3;
+			u16 offset:9;
+			u32 length20:20;
+#else
+			u32 length20:20;
+			u16 offset:9;
+			enum qm_fd_format format:3;
+#endif
+		};
+		/* If 'format' is _contig_big or _sg_big, 29b length */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			enum qm_fd_format _format1:3;
+			u32 length29:29;
+#else
+			u32 length29:29;
+			enum qm_fd_format _format1:3;
+#endif
+		};
+		/* If 'format' is _compound, 29b "congestion weight" */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			enum qm_fd_format _format2:3;
+			u32 cong_weight:29;
+#else
+			u32 cong_weight:29;
+			enum qm_fd_format _format2:3;
+#endif
+		};
+	};
+	union {
+		u32 cmd;
+		u32 status;
+	};
+} __aligned(8);
+#define QM_FD_DD_NULL		0x00
+#define QM_FD_PID_MASK		0x3f
+static inline u64 qm_fd_addr_get64(const struct qm_fd *fd)
+{
+	return fd->addr;
+}
+
+static inline dma_addr_t qm_fd_addr(const struct qm_fd *fd)
+{
+	return (dma_addr_t)fd->addr;
+}
+/* Macro, so we compile better if 'v' isn't always 64-bit */
+#define qm_fd_addr_set64(fd, v) \
+	do { \
+		struct qm_fd *__fd931 = (fd); \
+		__fd931->addr = v; \
+	} while (0)
+
+/* For static initialisation of FDs (which is complicated by the use of unions
+ * in "struct qm_fd"), use the following macros. Note that;
+ * - 'dd', 'pid' and 'bpid' are ignored because there's no static initialisation
+ *   use-case),
+ * - use capitalised QM_FD_*** formats for static initialisation.
+ */
+#define QM_FD_FMT_20(cmd, addr_hi, addr_lo, fmt, off, len) \
+	{ 0, 0, 0, 0, 0, addr_hi, addr_lo, \
+	{ (((fmt)&0x7) << 29) | (((off)&0x1ff) << 20) | ((len)&0xfffff) }, \
+	{ cmd } }
+#define QM_FD_FMT_29(cmd, addr_hi, addr_lo, fmt, len) \
+	{ 0, 0, 0, 0, 0, addr_hi, addr_lo, \
+	{ (((fmt)&0x7) << 29) | ((len)&0x1fffffff) }, \
+	{ cmd } }
+
+/* See 2.2.1.3 Multi-Core Datapath Acceleration Architecture */
+#define QM_SG_OFFSET_MASK 0x1FFF
+struct qm_sg_entry {
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 __reserved1[3];
+			u8 addr_hi;	/* high 8-bits of 40-bit address */
+			u32 addr_lo;	/* low 32-bits of 40-bit address */
+#else
+			u32 addr_lo;	/* low 32-bits of 40-bit address */
+			u8 addr_hi;	/* high 8-bits of 40-bit address */
+			u8 __reserved1[3];
+#endif
+		};
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u64 __notaddress:24;
+			u64 addr:40;
+#else
+			u64 addr:40;
+			u64 __notaddress:24;
+#endif
+		};
+		u64 opaque;
+	};
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u32 extension:1;	/* Extension bit */
+			u32 final:1;		/* Final bit */
+			u32 length:30;
+#else
+			u32 length:30;
+			u32 final:1;            /* Final bit */
+			u32 extension:1;        /* Extension bit */
+#endif
+		};
+		u32 sgt_efl;
+	};
+	u8 __reserved2;
+	u8 bpid;
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 __reserved3:3;
+			u16 offset:13;
+#else
+			u16 offset:13;
+			u16 __reserved3:3;
+#endif
+		};
+		u16 opaque_offset;
+	};
+} __packed;
+union qm_sg_efl {
+	struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+		u32 extension:1;	/* Extension bit */
+		u32 final:1;		/* Final bit */
+		u32 length:30;
+#else
+		u32 length:30;
+		u32 final:1;            /* Final bit */
+		u32 extension:1;        /* Extension bit */
+#endif
+	};
+	u32 efl;
+};
+static inline u64 qm_sg_entry_get64(const struct qm_sg_entry *sg)
+{
+	return be64_to_cpu(sg->opaque);
+}
+static inline dma_addr_t qm_sg_addr(const struct qm_sg_entry *sg)
+{
+	return (dma_addr_t)be64_to_cpu(sg->opaque);
+}
+static inline u8 qm_sg_entry_get_ext(const struct qm_sg_entry *sg)
+{
+	union qm_sg_efl u;
+
+	u.efl = be32_to_cpu(sg->sgt_efl);
+	return u.extension;
+}
+static inline u8 qm_sg_entry_get_final(const struct qm_sg_entry *sg)
+{
+	union qm_sg_efl u;
+
+	u.efl = be32_to_cpu(sg->sgt_efl);
+	return u.final;
+}
+static inline u32 qm_sg_entry_get_len(const struct qm_sg_entry *sg)
+{
+	union qm_sg_efl u;
+
+	u.efl = be32_to_cpu(sg->sgt_efl);
+	return u.length;
+}
+static inline u8 qm_sg_entry_get_bpid(const struct qm_sg_entry *sg)
+{
+	return sg->bpid;
+}
+static inline u16 qm_sg_entry_get_offset(const struct qm_sg_entry *sg)
+{
+	u32 opaque_offset = be16_to_cpu(sg->opaque_offset);
+
+	return opaque_offset & 0x1fff;
+}
+
+/* Macro, so we compile better if 'v' isn't always 64-bit */
+#define qm_sg_entry_set64(sg, v) \
+	do { \
+		struct qm_sg_entry *__sg931 = (sg); \
+		__sg931->opaque = cpu_to_be64(v); \
+	} while (0)
+#define qm_sg_entry_set_ext(sg, v) \
+	do { \
+		union qm_sg_efl __u932; \
+		__u932.efl = be32_to_cpu((sg)->sgt_efl); \
+		__u932.extension = v; \
+		(sg)->sgt_efl = cpu_to_be32(__u932.efl); \
+	} while (0)
+#define qm_sg_entry_set_final(sg, v) \
+	do { \
+		union qm_sg_efl __u933; \
+		__u933.efl = be32_to_cpu((sg)->sgt_efl); \
+		__u933.final = v; \
+		(sg)->sgt_efl = cpu_to_be32(__u933.efl); \
+	} while (0)
+#define qm_sg_entry_set_len(sg, v) \
+	do { \
+		union qm_sg_efl __u934; \
+		__u934.efl = be32_to_cpu((sg)->sgt_efl); \
+		__u934.length = v; \
+		(sg)->sgt_efl = cpu_to_be32(__u934.efl); \
+	} while (0)
+#define qm_sg_entry_set_bpid(sg, v) \
+	do { \
+		struct qm_sg_entry *__u935 = (sg); \
+		__u935->bpid = v; \
+	} while (0)
+#define qm_sg_entry_set_offset(sg, v) \
+	do { \
+		struct qm_sg_entry *__u936 = (sg); \
+		__u936->opaque_offset = cpu_to_be16(v); \
+	} while (0)
+
+/* See 1.5.8.1: "Enqueue Command" */
+struct qm_eqcr_entry {
+	u8 __dont_write_directly__verb;
+	u8 dca;
+	u16 seqnum;
+	u32 orp;	/* 24-bit */
+	u32 fqid;	/* 24-bit */
+	u32 tag;
+	struct qm_fd fd;
+	u8 __reserved3[32];
+} __packed;
+#define QM_EQCR_VERB_VBIT		0x80
+#define QM_EQCR_VERB_CMD_MASK		0x61	/* but only one value; */
+#define QM_EQCR_VERB_CMD_ENQUEUE	0x01
+#define QM_EQCR_VERB_COLOUR_MASK	0x18	/* 4 possible values; */
+#define QM_EQCR_VERB_COLOUR_GREEN	0x00
+#define QM_EQCR_VERB_COLOUR_YELLOW	0x08
+#define QM_EQCR_VERB_COLOUR_RED		0x10
+#define QM_EQCR_VERB_COLOUR_OVERRIDE	0x18
+#define QM_EQCR_VERB_INTERRUPT		0x04	/* on command consumption */
+#define QM_EQCR_VERB_ORP		0x02	/* enable order restoration */
+#define QM_EQCR_DCA_ENABLE		0x80
+#define QM_EQCR_DCA_PARK		0x40
+#define QM_EQCR_DCA_IDXMASK		0x0f	/* "DQRR::idx" goes here */
+#define QM_EQCR_SEQNUM_NESN		0x8000	/* Advance NESN */
+#define QM_EQCR_SEQNUM_NLIS		0x4000	/* More fragments to come */
+#define QM_EQCR_SEQNUM_SEQMASK		0x3fff	/* sequence number goes here */
+#define QM_EQCR_FQID_NULL		0	/* eg. for an ORP seqnum hole */
+
+/* See 1.5.8.2: "Frame Dequeue Response" */
+struct qm_dqrr_entry {
+	u8 verb;
+	u8 stat;
+	u16 seqnum;	/* 15-bit */
+	u8 tok;
+	u8 __reserved2[3];
+	u32 fqid;	/* 24-bit */
+	u32 contextB;
+	struct qm_fd fd;
+	u8 __reserved4[32];
+};
+#define QM_DQRR_VERB_VBIT		0x80
+#define QM_DQRR_VERB_MASK		0x7f	/* where the verb contains; */
+#define QM_DQRR_VERB_FRAME_DEQUEUE	0x60	/* "this format" */
+#define QM_DQRR_STAT_FQ_EMPTY		0x80	/* FQ empty */
+#define QM_DQRR_STAT_FQ_HELDACTIVE	0x40	/* FQ held active */
+#define QM_DQRR_STAT_FQ_FORCEELIGIBLE	0x20	/* FQ was force-eligible'd */
+#define QM_DQRR_STAT_FD_VALID		0x10	/* has a non-NULL FD */
+#define QM_DQRR_STAT_UNSCHEDULED	0x02	/* Unscheduled dequeue */
+#define QM_DQRR_STAT_DQCR_EXPIRED	0x01	/* VDQCR or PDQCR expired*/
+
+/* See 1.5.8.3: "ERN Message Response" */
+/* See 1.5.8.4: "FQ State Change Notification" */
+struct qm_mr_entry {
+	u8 verb;
+	union {
+		struct {
+			u8 dca;
+			u16 seqnum;
+			u8 rc;		/* Rejection Code */
+			u32 orp:24;
+			u32 fqid;	/* 24-bit */
+			u32 tag;
+			struct qm_fd fd;
+		} __packed ern;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 colour:2;	/* See QM_MR_DCERN_COLOUR_* */
+			u8 __reserved1:3;
+			enum qm_dc_portal portal:3;
+#else
+			enum qm_dc_portal portal:3;
+			u8 __reserved1:3;
+			u8 colour:2;	/* See QM_MR_DCERN_COLOUR_* */
+#endif
+			u16 __reserved2;
+			u8 rc;		/* Rejection Code */
+			u32 __reserved3:24;
+			u32 fqid;	/* 24-bit */
+			u32 tag;
+			struct qm_fd fd;
+		} __packed dcern;
+		struct {
+			u8 fqs;		/* Frame Queue Status */
+			u8 __reserved1[6];
+			u32 fqid;	/* 24-bit */
+			u32 contextB;
+			u8 __reserved2[16];
+		} __packed fq;		/* FQRN/FQRNI/FQRL/FQPN */
+	};
+	u8 __reserved2[32];
+} __packed;
+#define QM_MR_VERB_VBIT			0x80
+/* The "ern" VERB bits match QM_EQCR_VERB_*** so aren't reproduced here. ERNs
+ * originating from direct-connect portals ("dcern") use 0x20 as a verb which
+ * would be invalid as a s/w enqueue verb. A s/w ERN can be distinguished from
+ * the other MR types by noting if the 0x20 bit is unset. */
+#define QM_MR_VERB_TYPE_MASK		0x27
+#define QM_MR_VERB_DC_ERN		0x20
+#define QM_MR_VERB_FQRN			0x21
+#define QM_MR_VERB_FQRNI		0x22
+#define QM_MR_VERB_FQRL			0x23
+#define QM_MR_VERB_FQPN			0x24
+#define QM_MR_RC_MASK			0xf0	/* contains one of; */
+#define QM_MR_RC_CGR_TAILDROP		0x00
+#define QM_MR_RC_WRED			0x10
+#define QM_MR_RC_ERROR			0x20
+#define QM_MR_RC_ORPWINDOW_EARLY	0x30
+#define QM_MR_RC_ORPWINDOW_LATE		0x40
+#define QM_MR_RC_FQ_TAILDROP		0x50
+#define QM_MR_RC_ORPWINDOW_RETIRED	0x60
+#define QM_MR_RC_ORP_ZERO		0x70
+#define QM_MR_FQS_ORLPRESENT		0x02	/* ORL fragments to come */
+#define QM_MR_FQS_NOTEMPTY		0x01	/* FQ has enqueued frames */
+#define QM_MR_DCERN_COLOUR_GREEN	0x00
+#define QM_MR_DCERN_COLOUR_YELLOW	0x01
+#define QM_MR_DCERN_COLOUR_RED		0x02
+#define QM_MR_DCERN_COLOUR_OVERRIDE	0x03
+
+/* An identical structure of FQD fields is present in the "Init FQ" command and
+ * the "Query FQ" result, it's suctioned out into the "struct qm_fqd" type.
+ * Within that, the 'stashing' and 'taildrop' pieces are also factored out, the
+ * latter has two inlines to assist with converting to/from the mant+exp
+ * representation. */
+struct qm_fqd_stashing {
+	/* See QM_STASHING_EXCL_<...> */
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u8 exclusive;
+	u8 __reserved1:2;
+	/* Numbers of cachelines */
+	u8 annotation_cl:2;
+	u8 data_cl:2;
+	u8 context_cl:2;
+#else
+	u8 context_cl:2;
+	u8 data_cl:2;
+	u8 annotation_cl:2;
+	u8 __reserved1:2;
+	u8 exclusive;
+#endif
+} __packed;
+struct qm_fqd_taildrop {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u16 __reserved1:3;
+	u16 mant:8;
+	u16 exp:5;
+#else
+	u16 exp:5;
+	u16 mant:8;
+	u16 __reserved1:3;
+#endif
+} __packed;
+struct qm_fqd_oac {
+	/* See QM_OAC_<...> */
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u8 oac:2; /* "Overhead Accounting Control" */
+	u8 __reserved1:6;
+#else
+	u8 __reserved1:6;
+	u8 oac:2; /* "Overhead Accounting Control" */
+#endif
+	/* Two's-complement value (-128 to +127) */
+	signed char oal; /* "Overhead Accounting Length" */
+} __packed;
+struct qm_fqd {
+	union {
+		u8 orpc;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 __reserved1:2;
+			u8 orprws:3;
+			u8 oa:1;
+			u8 olws:2;
+#else
+			u8 olws:2;
+			u8 oa:1;
+			u8 orprws:3;
+			u8 __reserved1:2;
+#endif
+		} __packed;
+	};
+	u8 cgid;
+	u16 fq_ctrl;	/* See QM_FQCTRL_<...> */
+	union {
+		u16 dest_wq;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 channel:13; /* qm_channel */
+			u16 wq:3;
+#else
+			u16 wq:3;
+			u16 channel:13; /* qm_channel */
+#endif
+		} __packed dest;
+	};
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u16 __reserved2:1;
+	u16 ics_cred:15;
+#else
+	u16 __reserved2:1;
+	u16 ics_cred:15;
+#endif
+	/* For "Initialize Frame Queue" commands, the write-enable mask
+	 * determines whether 'td' or 'oac_init' is observed. For query
+	 * commands, this field is always 'td', and 'oac_query' (below) reflects
+	 * the Overhead ACcounting values. */
+	union {
+		struct qm_fqd_taildrop td;
+		struct qm_fqd_oac oac_init;
+	};
+	u32 context_b;
+	union {
+		/* Treat it as 64-bit opaque */
+		u64 opaque;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u32 hi;
+			u32 lo;
+#else
+			u32 lo;
+			u32 hi;
+#endif
+		};
+		/* Treat it as s/w portal stashing config */
+		/* See 1.5.6.7.1: "FQD Context_A field used for [...] */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			struct qm_fqd_stashing stashing;
+			/* 48-bit address of FQ context to
+			 * stash, must be cacheline-aligned */
+			u16 context_hi;
+			u32 context_lo;
+#else
+			u32 context_lo;
+			u16 context_hi;
+			struct qm_fqd_stashing stashing;
+#endif
+		} __packed;
+	} context_a;
+	struct qm_fqd_oac oac_query;
+} __packed;
+/* 64-bit converters for context_hi/lo */
+static inline u64 qm_fqd_stashing_get64(const struct qm_fqd *fqd)
+{
+	return ((u64)fqd->context_a.context_hi << 32) |
+		(u64)fqd->context_a.context_lo;
+}
+static inline dma_addr_t qm_fqd_stashing_addr(const struct qm_fqd *fqd)
+{
+	return (dma_addr_t)qm_fqd_stashing_get64(fqd);
+}
+static inline u64 qm_fqd_context_a_get64(const struct qm_fqd *fqd)
+{
+	return ((u64)fqd->context_a.hi << 32) |
+		(u64)fqd->context_a.lo;
+}
+/* Macro, so we compile better when 'v' isn't necessarily 64-bit */
+#define qm_fqd_stashing_set64(fqd, v) \
+	do { \
+		struct qm_fqd *__fqd931 = (fqd); \
+		__fqd931->context_a.context_hi = upper_32_bits(v); \
+		__fqd931->context_a.context_lo = lower_32_bits(v); \
+	} while (0)
+#define qm_fqd_context_a_set64(fqd, v) \
+	do { \
+		struct qm_fqd *__fqd931 = (fqd); \
+		__fqd931->context_a.hi = upper_32_bits(v); \
+		__fqd931->context_a.lo = lower_32_bits(v); \
+	} while (0)
+/* convert a threshold value into mant+exp representation */
+static inline int qm_fqd_taildrop_set(struct qm_fqd_taildrop *td, u32 val,
+					int roundup)
+{
+	u32 e = 0;
+	int oddbit = 0;
+	if (val > 0xe0000000)
+		return -ERANGE;
+	while (val > 0xff) {
+		oddbit = val & 1;
+		val >>= 1;
+		e++;
+		if (roundup && oddbit)
+			val++;
+	}
+	td->exp = e;
+	td->mant = val;
+	return 0;
+}
+/* and the other direction */
+static inline u32 qm_fqd_taildrop_get(const struct qm_fqd_taildrop *td)
+{
+	return (u32)td->mant << td->exp;
+}
+
+/* See 1.5.2.2: "Frame Queue Descriptor (FQD)" */
+/* Frame Queue Descriptor (FQD) field 'fq_ctrl' uses these constants */
+#define QM_FQCTRL_MASK		0x07ff	/* 'fq_ctrl' flags; */
+#define QM_FQCTRL_CGE		0x0400	/* Congestion Group Enable */
+#define QM_FQCTRL_TDE		0x0200	/* Tail-Drop Enable */
+#define QM_FQCTRL_ORP		0x0100	/* ORP Enable */
+#define QM_FQCTRL_CTXASTASHING	0x0080	/* Context-A stashing */
+#define QM_FQCTRL_CPCSTASH	0x0040	/* CPC Stash Enable */
+#define QM_FQCTRL_FORCESFDR	0x0008	/* High-priority SFDRs */
+#define QM_FQCTRL_AVOIDBLOCK	0x0004	/* Don't block active */
+#define QM_FQCTRL_HOLDACTIVE	0x0002	/* Hold active in portal */
+#define QM_FQCTRL_PREFERINCACHE	0x0001	/* Aggressively cache FQD */
+#define QM_FQCTRL_LOCKINCACHE	QM_FQCTRL_PREFERINCACHE /* older naming */
+
+/* See 1.5.6.7.1: "FQD Context_A field used for [...] */
+/* Frame Queue Descriptor (FQD) field 'CONTEXT_A' uses these constants */
+#define QM_STASHING_EXCL_ANNOTATION	0x04
+#define QM_STASHING_EXCL_DATA		0x02
+#define QM_STASHING_EXCL_CTX		0x01
+
+/* See 1.5.5.3: "Intra Class Scheduling" */
+/* FQD field 'OAC' (Overhead ACcounting) uses these constants */
+#define QM_OAC_ICS		0x2 /* Accounting for Intra-Class Scheduling */
+#define QM_OAC_CG		0x1 /* Accounting for Congestion Groups */
+
+/* See 1.5.8.4: "FQ State Change Notification" */
+/* This struct represents the 32-bit "WR_PARM_[GYR]" parameters in CGR fields
+ * and associated commands/responses. The WRED parameters are calculated from
+ * these fields as follows;
+ *   MaxTH = MA * (2 ^ Mn)
+ *   Slope = SA / (2 ^ Sn)
+ *    MaxP = 4 * (Pn + 1)
+ */
+struct qm_cgr_wr_parm {
+	union {
+		u32 word;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u32 MA:8;
+			u32 Mn:5;
+			u32 SA:7; /* must be between 64-127 */
+			u32 Sn:6;
+			u32 Pn:6;
+#else
+			u32 Pn:6;
+			u32 Sn:6;
+			u32 SA:7; /* must be between 64-127 */
+			u32 Mn:5;
+			u32 MA:8;
+#endif
+		} __packed;
+	};
+} __packed;
+/* This struct represents the 13-bit "CS_THRES" CGR field. In the corresponding
+ * management commands, this is padded to a 16-bit structure field, so that's
+ * how we represent it here. The congestion state threshold is calculated from
+ * these fields as follows;
+ *   CS threshold = TA * (2 ^ Tn)
+ */
+struct qm_cgr_cs_thres {
+	union {
+		u16 hword;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 __reserved:3;
+			u16 TA:8;
+			u16 Tn:5;
+#else
+			u16 Tn:5;
+			u16 TA:8;
+			u16 __reserved:3;
+#endif
+		} __packed;
+	};
+} __packed;
+/* This identical structure of CGR fields is present in the "Init/Modify CGR"
+ * commands and the "Query CGR" result. It's suctioned out here into its own
+ * struct. */
+struct __qm_mc_cgr {
+	struct qm_cgr_wr_parm wr_parm_g;
+	struct qm_cgr_wr_parm wr_parm_y;
+	struct qm_cgr_wr_parm wr_parm_r;
+	u8 wr_en_g;	/* boolean, use QM_CGR_EN */
+	u8 wr_en_y;	/* boolean, use QM_CGR_EN */
+	u8 wr_en_r;	/* boolean, use QM_CGR_EN */
+	u8 cscn_en;	/* boolean, use QM_CGR_EN */
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */
+			u16 cscn_targ_dcp_low;  /* CSCN_TARG_DCP low-16bits */
+#else
+			u16 cscn_targ_dcp_low;  /* CSCN_TARG_DCP low-16bits */
+			u16 cscn_targ_upd_ctrl; /* use QM_CSCN_TARG_UDP_ */
+#endif
+		};
+		u32 cscn_targ;	/* use QM_CGR_TARG_* */
+	};
+	u8 cstd_en;	/* boolean, use QM_CGR_EN */
+	u8 cs;		/* boolean, only used in query response */
+	union {
+		/* use qm_cgr_cs_thres_set64() */
+		struct qm_cgr_cs_thres cs_thres;
+		u16 __cs_thres;
+	};
+	u8 mode;	/* QMAN_CGR_MODE_FRAME not supported in rev1.0 */
+} __packed;
+#define QM_CGR_EN		0x01 /* For wr_en_*, cscn_en, cstd_en */
+#define QM_CGR_TARG_UDP_CTRL_WRITE_BIT	0x8000 /* value written to portal bit*/
+#define QM_CGR_TARG_UDP_CTRL_DCP	0x4000 /* 0: SWP, 1: DCP */
+#define QM_CGR_TARG_PORTAL(n)	(0x80000000 >> (n)) /* s/w portal, 0-9 */
+#define QM_CGR_TARG_FMAN0	0x00200000 /* direct-connect portal: fman0 */
+#define QM_CGR_TARG_FMAN1	0x00100000 /*                      : fman1 */
+/* Convert CGR thresholds to/from "cs_thres" format */
+static inline u64 qm_cgr_cs_thres_get64(const struct qm_cgr_cs_thres *th)
+{
+	return (u64)th->TA << th->Tn;
+}
+static inline int qm_cgr_cs_thres_set64(struct qm_cgr_cs_thres *th, u64 val,
+					int roundup)
+{
+	u32 e = 0;
+	int oddbit = 0;
+	while (val > 0xff) {
+		oddbit = val & 1;
+		val >>= 1;
+		e++;
+		if (roundup && oddbit)
+			val++;
+	}
+	th->Tn = e;
+	th->TA = val;
+	return 0;
+}
+
+/* See 1.5.8.5.1: "Initialize FQ" */
+/* See 1.5.8.5.2: "Query FQ" */
+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */
+/* See 1.5.8.5.4: "Alter FQ State Commands " */
+/* See 1.5.8.6.1: "Initialize/Modify CGR" */
+/* See 1.5.8.6.2: "CGR Test Write" */
+/* See 1.5.8.6.3: "Query CGR" */
+/* See 1.5.8.6.4: "Query Congestion Group State" */
+struct qm_mcc_initfq {
+	u8 __reserved1;
+	u16 we_mask;	/* Write Enable Mask */
+	u32 fqid;	/* 24-bit */
+	u16 count;	/* Initialises 'count+1' FQDs */
+	struct qm_fqd fqd; /* the FQD fields go here */
+	u8 __reserved3[30];
+} __packed;
+struct qm_mcc_queryfq {
+	u8 __reserved1[3];
+	u32 fqid;	/* 24-bit */
+	u8 __reserved2[56];
+} __packed;
+struct qm_mcc_queryfq_np {
+	u8 __reserved1[3];
+	u32 fqid;	/* 24-bit */
+	u8 __reserved2[56];
+} __packed;
+struct qm_mcc_alterfq {
+	u8 __reserved1[3];
+	u32 fqid;	/* 24-bit */
+	u8 __reserved2;
+	u8 count;	/* number of consecutive FQID */
+	u8 __reserved3[10];
+	u32 context_b;	/* frame queue context b */
+	u8 __reserved4[40];
+} __packed;
+struct qm_mcc_initcgr {
+	u8 __reserved1;
+	u16 we_mask;	/* Write Enable Mask */
+	struct __qm_mc_cgr cgr;	/* CGR fields */
+	u8 __reserved2[2];
+	u8 cgid;
+	u8 __reserved4[32];
+} __packed;
+struct qm_mcc_cgrtestwrite {
+	u8 __reserved1[2];
+	u8 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
+	u32 i_bcnt_lo;	/* low 32-bits of 40-bit */
+	u8 __reserved2[23];
+	u8 cgid;
+	u8 __reserved3[32];
+} __packed;
+struct qm_mcc_querycgr {
+	u8 __reserved1[30];
+	u8 cgid;
+	u8 __reserved2[32];
+} __packed;
+struct qm_mcc_querycongestion {
+	u8 __reserved[63];
+} __packed;
+struct qm_mcc_querywq {
+	u8 __reserved;
+	/* select channel if verb != QUERYWQ_DEDICATED */
+	union {
+		u16 channel_wq; /* ignores wq (3 lsbits) */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 id:13; /* qm_channel */
+			u16 __reserved1:3;
+#else
+			u16 __reserved1:3;
+			u16 id:13; /* qm_channel */
+#endif
+		} __packed channel;
+	};
+	u8 __reserved2[60];
+} __packed;
+
+struct qm_mcc_ceetm_lfqmt_config {
+	u8 __reserved1[4];
+	u32 lfqid:24;
+	u8 __reserved2[2];
+	u16 cqid;
+	u8 __reserved3[2];
+	u16 dctidx;
+	u8 __reserved4[48];
+} __packed;
+
+struct qm_mcc_ceetm_lfqmt_query {
+	u8 __reserved1[4];
+	u32 lfqid:24;
+	u8 __reserved2[56];
+} __packed;
+
+struct qm_mcc_ceetm_cq_config {
+	u8 __reserved1;
+	u16 cqid;
+	u8 dcpid;
+	u8 __reserved2;
+	u16 ccgid;
+	u8 __reserved3[56];
+} __packed;
+
+struct qm_mcc_ceetm_cq_query {
+	u8 __reserved1;
+	u16 cqid;
+	u8 dcpid;
+	u8 __reserved2[59];
+} __packed;
+
+struct qm_mcc_ceetm_dct_config {
+	u8 __reserved1;
+	u16 dctidx;
+	u8 dcpid;
+	u8 __reserved2[15];
+	u32 context_b;
+	u64 context_a;
+	u8 __reserved3[32];
+} __packed;
+
+struct qm_mcc_ceetm_dct_query {
+	u8 __reserved1;
+	u16 dctidx;
+	u8 dcpid;
+	u8 __reserved2[59];
+} __packed;
+
+struct qm_mcc_ceetm_class_scheduler_config {
+	u8 __reserved1;
+	u16 cqcid;
+	u8 dcpid;
+	u8 __reserved2[6];
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u8 gpc_reserved:1;
+	u8 gpc_combine_flag:1;
+	u8 gpc_prio_b:3;
+	u8 gpc_prio_a:3;
+#else
+	u8 gpc_prio_a:3;
+	u8 gpc_prio_b:3;
+	u8 gpc_combine_flag:1;
+	u8 gpc_reserved:1;
+#endif
+	u16 crem;
+	u16 erem;
+	u8 w[8];
+	u8 __reserved3[40];
+} __packed;
+
+struct qm_mcc_ceetm_class_scheduler_query {
+	u8 __reserved1;
+	u16 cqcid;
+	u8 dcpid;
+	u8 __reserved2[59];
+} __packed;
+
+#define CEETM_COMMAND_CHANNEL_MAPPING	(0 << 12)
+#define CEETM_COMMAND_SP_MAPPING	(1 << 12)
+#define CEETM_COMMAND_CHANNEL_SHAPER	(2 << 12)
+#define CEETM_COMMAND_LNI_SHAPER	(3 << 12)
+#define CEETM_COMMAND_TCFC		(4 << 12)
+
+#define CEETM_CCGRID_MASK	0x01FF
+#define CEETM_CCGR_CM_CONFIGURE	(0 << 14)
+#define CEETM_CCGR_DN_CONFIGURE	(1 << 14)
+#define CEETM_CCGR_TEST_WRITE	(2 << 14)
+#define CEETM_CCGR_CM_QUERY	(0 << 14)
+#define CEETM_CCGR_DN_QUERY	(1 << 14)
+#define CEETM_CCGR_DN_QUERY_FLUSH	(2 << 14)
+#define CEETM_QUERY_CONGESTION_STATE (3 << 14)
+
+struct qm_mcc_ceetm_mapping_shaper_tcfc_config {
+	u8 __reserved1;
+	u16 cid;
+	u8 dcpid;
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 map_shaped:1;
+			u8 map_reserved:4;
+			u8 map_lni_id:3;
+#else
+			u8 map_lni_id:3;
+			u8 map_reserved:4;
+			u8 map_shaped:1;
+#endif
+			u8 __reserved2[58];
+		} __packed channel_mapping;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 map_reserved:5;
+			u8 map_lni_id:3;
+#else
+			u8 map_lni_id:3;
+			u8 map_reserved:5;
+#endif
+			u8 __reserved2[58];
+		} __packed sp_mapping;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 cpl:1;
+			u8 cpl_reserved:2;
+			u8 oal:5;
+#else
+			u8 oal:5;
+			u8 cpl_reserved:2;
+			u8 cpl:1;
+#endif
+			u32 crtcr:24;
+			u32 ertcr:24;
+			u16 crtbl;
+			u16 ertbl;
+			u8 mps;	/* This will be hardcoded by driver with 60 */
+			u8 __reserved2[47];
+		} __packed shaper_config;
+		struct {
+			u8 __reserved2[11];
+			u64 lnitcfcc;
+			u8 __reserved3[40];
+		} __packed tcfc_config;
+	};
+} __packed;
+
+struct qm_mcc_ceetm_mapping_shaper_tcfc_query {
+	u8 __reserved1;
+	u16 cid;
+	u8 dcpid;
+	u8 __reserved2[59];
+} __packed;
+
+struct qm_mcc_ceetm_ccgr_config {
+	u8 __reserved1;
+	u16 ccgrid;
+	u8 dcpid;
+	u8 __reserved2;
+	u16 we_mask;
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 ctl_reserved:1;
+			u8 ctl_wr_en_g:1;
+			u8 ctl_wr_en_y:1;
+			u8 ctl_wr_en_r:1;
+			u8 ctl_td_en:1;
+			u8 ctl_td_mode:1;
+			u8 ctl_cscn_en:1;
+			u8 ctl_mode:1;
+#else
+			u8 ctl_mode:1;
+			u8 ctl_cscn_en:1;
+			u8 ctl_td_mode:1;
+			u8 ctl_td_en:1;
+			u8 ctl_wr_en_r:1;
+			u8 ctl_wr_en_y:1;
+			u8 ctl_wr_en_g:1;
+			u8 ctl_reserved:1;
+#endif
+			u8 cdv;
+			u16 cscn_tupd;
+			u8 oal;
+			u8 __reserved3;
+			struct qm_cgr_cs_thres cs_thres;
+			struct qm_cgr_cs_thres cs_thres_x;
+			struct qm_cgr_cs_thres td_thres;
+			struct qm_cgr_wr_parm wr_parm_g;
+			struct qm_cgr_wr_parm wr_parm_y;
+			struct qm_cgr_wr_parm wr_parm_r;
+		} __packed cm_config;
+		struct {
+			u8 dnc;
+			u8 dn0;
+			u8 dn1;
+			u64 dnba:40;
+			u8 __reserved3[2];
+			u16 dnth_0;
+			u8 __reserved4[2];
+			u16 dnth_1;
+			u8 __reserved5[8];
+		} __packed dn_config;
+		struct {
+			u8 __reserved3[3];
+			u64 i_cnt:40;
+			u8 __reserved4[16];
+		} __packed test_write;
+	};
+	u8 __reserved5[32];
+} __packed;
+
+struct qm_mcc_ceetm_ccgr_query {
+	u8 __reserved1;
+	u16 ccgrid;
+	u8 dcpid;
+	u8 __reserved2[59];
+} __packed;
+
+struct qm_mcc_ceetm_cq_peek_pop_xsfdrread {
+	u8 __reserved1;
+	u16 cqid;
+	u8 dcpid;
+	u8 ct;
+	u16 xsfdr;
+	u8 __reserved2[56];
+} __packed;
+
+#define CEETM_QUERY_DEQUEUE_STATISTICS 0x00
+#define CEETM_QUERY_DEQUEUE_CLEAR_STATISTICS 0x01
+#define CEETM_WRITE_DEQUEUE_STATISTICS 0x02
+#define CEETM_QUERY_REJECT_STATISTICS 0x03
+#define CEETM_QUERY_REJECT_CLEAR_STATISTICS 0x04
+#define CEETM_WRITE_REJECT_STATISTICS 0x05
+struct qm_mcc_ceetm_statistics_query_write {
+	u8 __reserved1;
+	u16 cid;
+	u8 dcpid;
+	u8 ct;
+	u8 __reserved2[13];
+	u64 frm_cnt:40;
+	u8 __reserved3[2];
+	u64 byte_cnt:48;
+	u8 __reserved[32];
+} __packed;
+
+struct qm_mc_command {
+	u8 __dont_write_directly__verb;
+	union {
+		struct qm_mcc_initfq initfq;
+		struct qm_mcc_queryfq queryfq;
+		struct qm_mcc_queryfq_np queryfq_np;
+		struct qm_mcc_alterfq alterfq;
+		struct qm_mcc_initcgr initcgr;
+		struct qm_mcc_cgrtestwrite cgrtestwrite;
+		struct qm_mcc_querycgr querycgr;
+		struct qm_mcc_querycongestion querycongestion;
+		struct qm_mcc_querywq querywq;
+		struct qm_mcc_ceetm_lfqmt_config lfqmt_config;
+		struct qm_mcc_ceetm_lfqmt_query lfqmt_query;
+		struct qm_mcc_ceetm_cq_config cq_config;
+		struct qm_mcc_ceetm_cq_query cq_query;
+		struct qm_mcc_ceetm_dct_config dct_config;
+		struct qm_mcc_ceetm_dct_query dct_query;
+		struct qm_mcc_ceetm_class_scheduler_config csch_config;
+		struct qm_mcc_ceetm_class_scheduler_query csch_query;
+		struct qm_mcc_ceetm_mapping_shaper_tcfc_config mst_config;
+		struct qm_mcc_ceetm_mapping_shaper_tcfc_query mst_query;
+		struct qm_mcc_ceetm_ccgr_config ccgr_config;
+		struct qm_mcc_ceetm_ccgr_query ccgr_query;
+		struct qm_mcc_ceetm_cq_peek_pop_xsfdrread cq_ppxr;
+		struct qm_mcc_ceetm_statistics_query_write stats_query_write;
+	};
+} __packed;
+#define QM_MCC_VERB_VBIT		0x80
+#define QM_MCC_VERB_MASK		0x7f	/* where the verb contains; */
+#define QM_MCC_VERB_INITFQ_PARKED	0x40
+#define QM_MCC_VERB_INITFQ_SCHED	0x41
+#define QM_MCC_VERB_QUERYFQ		0x44
+#define QM_MCC_VERB_QUERYFQ_NP		0x45	/* "non-programmable" fields */
+#define QM_MCC_VERB_QUERYWQ		0x46
+#define QM_MCC_VERB_QUERYWQ_DEDICATED	0x47
+#define QM_MCC_VERB_ALTER_SCHED		0x48	/* Schedule FQ */
+#define QM_MCC_VERB_ALTER_FE		0x49	/* Force Eligible FQ */
+#define QM_MCC_VERB_ALTER_RETIRE	0x4a	/* Retire FQ */
+#define QM_MCC_VERB_ALTER_OOS		0x4b	/* Take FQ out of service */
+#define QM_MCC_VERB_ALTER_FQXON		0x4d	/* FQ XON */
+#define QM_MCC_VERB_ALTER_FQXOFF	0x4e	/* FQ XOFF */
+#define QM_MCC_VERB_INITCGR		0x50
+#define QM_MCC_VERB_MODIFYCGR		0x51
+#define QM_MCC_VERB_CGRTESTWRITE	0x52
+#define QM_MCC_VERB_QUERYCGR		0x58
+#define QM_MCC_VERB_QUERYCONGESTION	0x59
+/* INITFQ-specific flags */
+#define QM_INITFQ_WE_MASK		0x01ff	/* 'Write Enable' flags; */
+#define QM_INITFQ_WE_OAC		0x0100
+#define QM_INITFQ_WE_ORPC		0x0080
+#define QM_INITFQ_WE_CGID		0x0040
+#define QM_INITFQ_WE_FQCTRL		0x0020
+#define QM_INITFQ_WE_DESTWQ		0x0010
+#define QM_INITFQ_WE_ICSCRED		0x0008
+#define QM_INITFQ_WE_TDTHRESH		0x0004
+#define QM_INITFQ_WE_CONTEXTB		0x0002
+#define QM_INITFQ_WE_CONTEXTA		0x0001
+/* INITCGR/MODIFYCGR-specific flags */
+#define QM_CGR_WE_MASK			0x07ff	/* 'Write Enable Mask'; */
+#define QM_CGR_WE_WR_PARM_G		0x0400
+#define QM_CGR_WE_WR_PARM_Y		0x0200
+#define QM_CGR_WE_WR_PARM_R		0x0100
+#define QM_CGR_WE_WR_EN_G		0x0080
+#define QM_CGR_WE_WR_EN_Y		0x0040
+#define QM_CGR_WE_WR_EN_R		0x0020
+#define QM_CGR_WE_CSCN_EN		0x0010
+#define QM_CGR_WE_CSCN_TARG		0x0008
+#define QM_CGR_WE_CSTD_EN		0x0004
+#define QM_CGR_WE_CS_THRES		0x0002
+#define QM_CGR_WE_MODE			0x0001
+
+/* See 1.5.9.7 CEETM Management Commands */
+#define QM_CEETM_VERB_LFQMT_CONFIG	0x70
+#define QM_CEETM_VERB_LFQMT_QUERY	0x71
+#define QM_CEETM_VERB_CQ_CONFIG		0x72
+#define QM_CEETM_VERB_CQ_QUERY		0x73
+#define QM_CEETM_VERB_DCT_CONFIG	0x74
+#define QM_CEETM_VERB_DCT_QUERY		0x75
+#define QM_CEETM_VERB_CLASS_SCHEDULER_CONFIG		0x76
+#define QM_CEETM_VERB_CLASS_SCHEDULER_QUERY		0x77
+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_CONFIG	0x78
+#define QM_CEETM_VERB_MAPPING_SHAPER_TCFC_QUERY		0x79
+#define QM_CEETM_VERB_CCGR_CONFIG			0x7A
+#define QM_CEETM_VERB_CCGR_QUERY			0x7B
+#define QM_CEETM_VERB_CQ_PEEK_POP_XFDRREAD		0x7C
+#define QM_CEETM_VERB_STATISTICS_QUERY_WRITE		0x7D
+
+/* See 1.5.8.5.1: "Initialize FQ" */
+/* See 1.5.8.5.2: "Query FQ" */
+/* See 1.5.8.5.3: "Query FQ Non-Programmable Fields" */
+/* See 1.5.8.5.4: "Alter FQ State Commands " */
+/* See 1.5.8.6.1: "Initialize/Modify CGR" */
+/* See 1.5.8.6.2: "CGR Test Write" */
+/* See 1.5.8.6.3: "Query CGR" */
+/* See 1.5.8.6.4: "Query Congestion Group State" */
+struct qm_mcr_initfq {
+	u8 __reserved1[62];
+} __packed;
+struct qm_mcr_queryfq {
+	u8 __reserved1[8];
+	struct qm_fqd fqd;	/* the FQD fields are here */
+	u8 __reserved2[30];
+} __packed;
+struct qm_mcr_queryfq_np {
+	u8 __reserved1;
+	u8 state;	/* QM_MCR_NP_STATE_*** */
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u8 __reserved2;
+	u32 fqd_link:24;
+	u16 __reserved3:2;
+	u16 odp_seq:14;
+	u16 __reserved4:2;
+	u16 orp_nesn:14;
+	u16 __reserved5:1;
+	u16 orp_ea_hseq:15;
+	u16 __reserved6:1;
+	u16 orp_ea_tseq:15;
+	u8 __reserved7;
+	u32 orp_ea_hptr:24;
+	u8 __reserved8;
+	u32 orp_ea_tptr:24;
+	u8 __reserved9;
+	u32 pfdr_hptr:24;
+	u8 __reserved10;
+	u32 pfdr_tptr:24;
+	u8 __reserved11[5];
+	u8 __reserved12:7;
+	u8 is:1;
+	u16 ics_surp;
+	u32 byte_cnt;
+	u8 __reserved13;
+	u32 frm_cnt:24;
+	u32 __reserved14;
+	u16 ra1_sfdr;	/* QM_MCR_NP_RA1_*** */
+	u16 ra2_sfdr;	/* QM_MCR_NP_RA2_*** */
+	u16 __reserved15;
+	u16 od1_sfdr;	/* QM_MCR_NP_OD1_*** */
+	u16 od2_sfdr;	/* QM_MCR_NP_OD2_*** */
+	u16 od3_sfdr;	/* QM_MCR_NP_OD3_*** */
+#else
+	u8 __reserved2;
+	u32 fqd_link:24;
+
+	u16 odp_seq:14;
+	u16 __reserved3:2;
+
+	u16 orp_nesn:14;
+	u16 __reserved4:2;
+
+	u16 orp_ea_hseq:15;
+	u16 __reserved5:1;
+
+	u16 orp_ea_tseq:15;
+	u16 __reserved6:1;
+
+	u8 __reserved7;
+	u32 orp_ea_hptr:24;
+
+	u8 __reserved8;
+	u32 orp_ea_tptr:24;
+
+	u8 __reserved9;
+	u32 pfdr_hptr:24;
+
+	u8 __reserved10;
+	u32 pfdr_tptr:24;
+
+	u8 __reserved11[5];
+	u8 is:1;
+	u8 __reserved12:7;
+	u16 ics_surp;
+	u32 byte_cnt;
+	u8 __reserved13;
+	u32 frm_cnt:24;
+	u32 __reserved14;
+	u16 ra1_sfdr;	/* QM_MCR_NP_RA1_*** */
+	u16 ra2_sfdr;	/* QM_MCR_NP_RA2_*** */
+	u16 __reserved15;
+	u16 od1_sfdr;	/* QM_MCR_NP_OD1_*** */
+	u16 od2_sfdr;	/* QM_MCR_NP_OD2_*** */
+	u16 od3_sfdr;	/* QM_MCR_NP_OD3_*** */
+#endif
+} __packed;
+
+
+struct qm_mcr_alterfq {
+	u8 fqs;		/* Frame Queue Status */
+	u8 __reserved1[61];
+} __packed;
+struct qm_mcr_initcgr {
+	u8 __reserved1[62];
+} __packed;
+struct qm_mcr_cgrtestwrite {
+	u16 __reserved1;
+	struct __qm_mc_cgr cgr; /* CGR fields */
+	u8 __reserved2[3];
+	u32 __reserved3:24;
+	u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
+	u32 i_bcnt_lo;	/* low 32-bits of 40-bit */
+	u32 __reserved4:24;
+	u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
+	u32 a_bcnt_lo;	/* low 32-bits of 40-bit */
+	u16 lgt;	/* Last Group Tick */
+	u16 wr_prob_g;
+	u16 wr_prob_y;
+	u16 wr_prob_r;
+	u8 __reserved5[8];
+} __packed;
+struct qm_mcr_querycgr {
+	u16 __reserved1;
+	struct __qm_mc_cgr cgr; /* CGR fields */
+	u8 __reserved2[3];
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u32 __reserved3:24;
+			u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
+			u32 i_bcnt_lo;	/* low 32-bits of 40-bit */
+#else
+			u32 i_bcnt_lo;	/* low 32-bits of 40-bit */
+			u32 i_bcnt_hi:8;/* high 8-bits of 40-bit "Instant" */
+			u32 __reserved3:24;
+#endif
+		};
+		u64 i_bcnt;
+	};
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u32 __reserved4:24;
+			u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
+			u32 a_bcnt_lo;	/* low 32-bits of 40-bit */
+#else
+			u32 a_bcnt_lo;	/* low 32-bits of 40-bit */
+			u32 a_bcnt_hi:8;/* high 8-bits of 40-bit "Average" */
+			u32 __reserved4:24;
+#endif
+		};
+		u64 a_bcnt;
+	};
+	union {
+		u32 cscn_targ_swp[4];
+		u8 __reserved5[16];
+	};
+} __packed;
+static inline u64 qm_mcr_querycgr_i_get64(const struct qm_mcr_querycgr *q)
+{
+	return be64_to_cpu(q->i_bcnt);
+}
+static inline u64 qm_mcr_querycgr_a_get64(const struct qm_mcr_querycgr *q)
+{
+	return be64_to_cpu(q->a_bcnt);
+}
+static inline u64 qm_mcr_cgrtestwrite_i_get64(
+					const struct qm_mcr_cgrtestwrite *q)
+{
+	return be64_to_cpu(((u64)q->i_bcnt_hi << 32) | (u64)q->i_bcnt_lo);
+}
+static inline u64 qm_mcr_cgrtestwrite_a_get64(
+					const struct qm_mcr_cgrtestwrite *q)
+{
+	return be64_to_cpu(((u64)q->a_bcnt_hi << 32) | (u64)q->a_bcnt_lo);
+}
+/* Macro, so we compile better if 'v' isn't always 64-bit */
+#define qm_mcr_querycgr_i_set64(q, v) \
+	do { \
+		struct qm_mcr_querycgr *__q931 = (fd); \
+		__q931->i_bcnt_hi = upper_32_bits(v); \
+		__q931->i_bcnt_lo = lower_32_bits(v); \
+	} while (0)
+#define qm_mcr_querycgr_a_set64(q, v) \
+	do { \
+		struct qm_mcr_querycgr *__q931 = (fd); \
+		__q931->a_bcnt_hi = upper_32_bits(v); \
+		__q931->a_bcnt_lo = lower_32_bits(v); \
+	} while (0)
+struct __qm_mcr_querycongestion {
+	u32 __state[8];
+};
+struct qm_mcr_querycongestion {
+	u8 __reserved[30];
+	/* Access this struct using QM_MCR_QUERYCONGESTION() */
+	struct __qm_mcr_querycongestion state;
+} __packed;
+struct qm_mcr_querywq {
+	union {
+		u16 channel_wq; /* ignores wq (3 lsbits) */
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u16 id:13; /* qm_channel */
+			u16 __reserved:3;
+#else
+			u16 __reserved:3;
+			u16 id:13; /* qm_channel */
+#endif
+		} __packed channel;
+	};
+	u8 __reserved[28];
+	u32 wq_len[8];
+} __packed;
+
+/* QMAN CEETM Management Command Response */
+struct qm_mcr_ceetm_lfqmt_config {
+	u8 __reserved1[62];
+} __packed;
+struct qm_mcr_ceetm_lfqmt_query {
+	u8 __reserved1[8];
+	u16 cqid;
+	u8 __reserved2[2];
+	u16 dctidx;
+	u8 __reserved3[2];
+	u16 ccgid;
+	u8 __reserved4[44];
+} __packed;
+
+struct qm_mcr_ceetm_cq_config {
+	u8 __reserved1[62];
+} __packed;
+
+struct qm_mcr_ceetm_cq_query {
+	u8 __reserved1[4];
+	u16 ccgid;
+	u16 state;
+	u32 pfdr_hptr:24;
+	u32 pfdr_tptr:24;
+	u16 od1_xsfdr;
+	u16 od2_xsfdr;
+	u16 od3_xsfdr;
+	u16 od4_xsfdr;
+	u16 od5_xsfdr;
+	u16 od6_xsfdr;
+	u16 ra1_xsfdr;
+	u16 ra2_xsfdr;
+	u8 __reserved2;
+	u32 frm_cnt:24;
+	u8 __reserved333[28];
+} __packed;
+
+struct qm_mcr_ceetm_dct_config {
+	u8 __reserved1[62];
+} __packed;
+
+struct qm_mcr_ceetm_dct_query {
+	u8 __reserved1[18];
+	u32 context_b;
+	u64 context_a;
+	u8 __reserved2[32];
+} __packed;
+
+struct qm_mcr_ceetm_class_scheduler_config {
+	u8 __reserved1[62];
+} __packed;
+
+struct qm_mcr_ceetm_class_scheduler_query {
+	u8 __reserved1[9];
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+	u8 gpc_reserved:1;
+	u8 gpc_combine_flag:1;
+	u8 gpc_prio_b:3;
+	u8 gpc_prio_a:3;
+#else
+	u8 gpc_prio_a:3;
+	u8 gpc_prio_b:3;
+	u8 gpc_combine_flag:1;
+	u8 gpc_reserved:1;
+#endif
+	u16 crem;
+	u16 erem;
+	u8 w[8];
+	u8 __reserved2[5];
+	u32 wbfslist:24;
+	u32 d8;
+	u32 d9;
+	u32 d10;
+	u32 d11;
+	u32 d12;
+	u32 d13;
+	u32 d14;
+	u32 d15;
+} __packed;
+
+struct qm_mcr_ceetm_mapping_shaper_tcfc_config {
+	u16 cid;
+	u8 __reserved2[60];
+} __packed;
+
+struct qm_mcr_ceetm_mapping_shaper_tcfc_query {
+	u16 cid;
+	u8 __reserved1;
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 map_shaped:1;
+			u8 map_reserved:4;
+			u8 map_lni_id:3;
+#else
+			u8 map_lni_id:3;
+			u8 map_reserved:4;
+			u8 map_shaped:1;
+#endif
+			u8 __reserved2[58];
+		} __packed channel_mapping_query;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 map_reserved:5;
+			u8 map_lni_id:3;
+#else
+			u8 map_lni_id:3;
+			u8 map_reserved:5;
+#endif
+			u8 __reserved2[58];
+		} __packed sp_mapping_query;
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 cpl:1;
+			u8 cpl_reserved:2;
+			u8 oal:5;
+#else
+			u8 oal:5;
+			u8 cpl_reserved:2;
+			u8 cpl:1;
+#endif
+			u32 crtcr:24;
+			u32 ertcr:24;
+			u16 crtbl;
+			u16 ertbl;
+			u8 mps;
+			u8 __reserved2[15];
+			u32 crat;
+			u32 erat;
+			u8 __reserved3[24];
+		} __packed shaper_query;
+		struct {
+			u8 __reserved1[11];
+			u64 lnitcfcc;
+			u8 __reserved3[40];
+		} __packed tcfc_query;
+	};
+} __packed;
+
+struct qm_mcr_ceetm_ccgr_config {
+	u8 __reserved1[46];
+	union {
+		u8 __reserved2[8];
+		struct {
+			u16 timestamp;
+			u16 wr_porb_g;
+			u16 wr_prob_y;
+			u16 wr_prob_r;
+		} __packed test_write;
+	};
+	u8 __reserved3[8];
+} __packed;
+
+struct qm_mcr_ceetm_ccgr_query {
+	u8 __reserved1[6];
+	union {
+		struct {
+#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+			u8 ctl_reserved:1;
+			u8 ctl_wr_en_g:1;
+			u8 ctl_wr_en_y:1;
+			u8 ctl_wr_en_r:1;
+			u8 ctl_td_en:1;
+			u8 ctl_td_mode:1;
+			u8 ctl_cscn_en:1;
+			u8 ctl_mode:1;
+#else
+			u8 ctl_mode:1;
+			u8 ctl_cscn_en:1;
+			u8 ctl_td_mode:1;
+			u8 ctl_td_en:1;
+			u8 ctl_wr_en_r:1;
+			u8 ctl_wr_en_y:1;
+			u8 ctl_wr_en_g:1;
+			u8 ctl_reserved:1;
+#endif
+			u8 cdv;
+			u8 __reserved2[2];
+			u8 oal;
+			u8 __reserved3;
+			struct qm_cgr_cs_thres cs_thres;
+			struct qm_cgr_cs_thres cs_thres_x;
+			struct qm_cgr_cs_thres td_thres;
+			struct qm_cgr_wr_parm wr_parm_g;
+			struct qm_cgr_wr_parm wr_parm_y;
+			struct qm_cgr_wr_parm wr_parm_r;
+			u16 cscn_targ_dcp;
+			u8 dcp_lsn;
+			u64 i_cnt:40;
+			u8 __reserved4[3];
+			u64 a_cnt:40;
+			u32 cscn_targ_swp[4];
+		} __packed cm_query;
+		struct {
+			u8 dnc;
+			u8 dn0;
+			u8 dn1;
+			u64 dnba:40;
+			u8 __reserved2[2];
+			u16 dnth_0;
+			u8 __reserved3[2];
+			u16 dnth_1;
+			u8 __reserved4[10];
+			u16 dnacc_0;
+			u8 __reserved5[2];
+			u16 dnacc_1;
+			u8 __reserved6[24];
+		} __packed dn_query;
+		struct {
+			u8 __reserved2[24];
+			struct  __qm_mcr_querycongestion state;
+		} __packed congestion_state;
+
+	};
+} __packed;
+
+struct qm_mcr_ceetm_cq_peek_pop_xsfdrread {
+	u8 stat;
+	u8 __reserved1[11];
+	u16 dctidx;
+	struct qm_fd fd;
+	u8 __reserved2[32];
+} __packed;
+
+struct qm_mcr_ceetm_statistics_query {
+	u8 __reserved1[17];
+	u64 frm_cnt:40;
+	u8 __reserved2[2];
+	u64 byte_cnt:48;
+	u8 __reserved3[32];
+} __packed;
+
+struct qm_mc_result {
+	u8 verb;
+	u8 result;
+	union {
+		struct qm_mcr_initfq initfq;
+		struct qm_mcr_queryfq queryfq;
+		struct qm_mcr_queryfq_np queryfq_np;
+		struct qm_mcr_alterfq alterfq;
+		struct qm_mcr_initcgr initcgr;
+		struct qm_mcr_cgrtestwrite cgrtestwrite;
+		struct qm_mcr_querycgr querycgr;
+		struct qm_mcr_querycongestion querycongestion;
+		struct qm_mcr_querywq querywq;
+		struct qm_mcr_ceetm_lfqmt_config lfqmt_config;
+		struct qm_mcr_ceetm_lfqmt_query lfqmt_query;
+		struct qm_mcr_ceetm_cq_config cq_config;
+		struct qm_mcr_ceetm_cq_query cq_query;
+		struct qm_mcr_ceetm_dct_config dct_config;
+		struct qm_mcr_ceetm_dct_query dct_query;
+		struct qm_mcr_ceetm_class_scheduler_config csch_config;
+		struct qm_mcr_ceetm_class_scheduler_query csch_query;
+		struct qm_mcr_ceetm_mapping_shaper_tcfc_config mst_config;
+		struct qm_mcr_ceetm_mapping_shaper_tcfc_query mst_query;
+		struct qm_mcr_ceetm_ccgr_config ccgr_config;
+		struct qm_mcr_ceetm_ccgr_query ccgr_query;
+		struct qm_mcr_ceetm_cq_peek_pop_xsfdrread cq_ppxr;
+		struct qm_mcr_ceetm_statistics_query stats_query;
+	};
+} __packed;
+
+#define QM_MCR_VERB_RRID		0x80
+#define QM_MCR_VERB_MASK		QM_MCC_VERB_MASK
+#define QM_MCR_VERB_INITFQ_PARKED	QM_MCC_VERB_INITFQ_PARKED
+#define QM_MCR_VERB_INITFQ_SCHED	QM_MCC_VERB_INITFQ_SCHED
+#define QM_MCR_VERB_QUERYFQ		QM_MCC_VERB_QUERYFQ
+#define QM_MCR_VERB_QUERYFQ_NP		QM_MCC_VERB_QUERYFQ_NP
+#define QM_MCR_VERB_QUERYWQ		QM_MCC_VERB_QUERYWQ
+#define QM_MCR_VERB_QUERYWQ_DEDICATED	QM_MCC_VERB_QUERYWQ_DEDICATED
+#define QM_MCR_VERB_ALTER_SCHED		QM_MCC_VERB_ALTER_SCHED
+#define QM_MCR_VERB_ALTER_FE		QM_MCC_VERB_ALTER_FE
+#define QM_MCR_VERB_ALTER_RETIRE	QM_MCC_VERB_ALTER_RETIRE
+#define QM_MCR_VERB_ALTER_OOS		QM_MCC_VERB_ALTER_OOS
+#define QM_MCR_RESULT_NULL		0x00
+#define QM_MCR_RESULT_OK		0xf0
+#define QM_MCR_RESULT_ERR_FQID		0xf1
+#define QM_MCR_RESULT_ERR_FQSTATE	0xf2
+#define QM_MCR_RESULT_ERR_NOTEMPTY	0xf3	/* OOS fails if FQ is !empty */
+#define QM_MCR_RESULT_ERR_BADCHANNEL	0xf4
+#define QM_MCR_RESULT_PENDING		0xf8
+#define QM_MCR_RESULT_ERR_BADCOMMAND	0xff
+#define QM_MCR_NP_STATE_FE		0x10
+#define QM_MCR_NP_STATE_R		0x08
+#define QM_MCR_NP_STATE_MASK		0x07	/* Reads FQD::STATE; */
+#define QM_MCR_NP_STATE_OOS		0x00
+#define QM_MCR_NP_STATE_RETIRED		0x01
+#define QM_MCR_NP_STATE_TEN_SCHED	0x02
+#define QM_MCR_NP_STATE_TRU_SCHED	0x03
+#define QM_MCR_NP_STATE_PARKED		0x04
+#define QM_MCR_NP_STATE_ACTIVE		0x05
+#define QM_MCR_NP_PTR_MASK		0x07ff	/* for RA[12] & OD[123] */
+#define QM_MCR_NP_RA1_NRA(v)		(((v) >> 14) & 0x3)	/* FQD::NRA */
+#define QM_MCR_NP_RA2_IT(v)		(((v) >> 14) & 0x1)	/* FQD::IT */
+#define QM_MCR_NP_OD1_NOD(v)		(((v) >> 14) & 0x3)	/* FQD::NOD */
+#define QM_MCR_NP_OD3_NPC(v)		(((v) >> 14) & 0x3)	/* FQD::NPC */
+#define QM_MCR_FQS_ORLPRESENT		0x02	/* ORL fragments to come */
+#define QM_MCR_FQS_NOTEMPTY		0x01	/* FQ has enqueued frames */
+/* This extracts the state for congestion group 'n' from a query response.
+ * Eg.
+ *   u8 cgr = [...];
+ *   struct qm_mc_result *res = [...];
+ *   printf("congestion group %d congestion state: %d\n", cgr,
+ *       QM_MCR_QUERYCONGESTION(&res->querycongestion.state, cgr));
+ */
+#define __CGR_WORD(num)		(num >> 5)
+#define __CGR_SHIFT(num)	(num & 0x1f)
+#define __CGR_NUM		(sizeof(struct __qm_mcr_querycongestion) << 3)
+static inline int QM_MCR_QUERYCONGESTION(struct __qm_mcr_querycongestion *p,
+					u8 cgr)
+{
+	return be32_to_cpu(p->__state[__CGR_WORD(cgr)]) &
+	       (0x80000000 >> __CGR_SHIFT(cgr));
+}
+
+
+/*********************/
+/* Utility interface */
+/*********************/
+
+/* Represents an allocator over a range of FQIDs. NB, accesses are not locked,
+ * spinlock them yourself if needed. */
+struct qman_fqid_pool;
+
+/* Create/destroy a FQID pool, num must be a multiple of 32. NB, _destroy()
+ * always succeeds, but returns non-zero if there were "leaked" FQID
+ * allocations. */
+struct qman_fqid_pool *qman_fqid_pool_create(u32 fqid_start, u32 num);
+int qman_fqid_pool_destroy(struct qman_fqid_pool *pool);
+/* Alloc/free a FQID from the range. _alloc() returns zero for success. */
+int qman_fqid_pool_alloc(struct qman_fqid_pool *pool, u32 *fqid);
+void qman_fqid_pool_free(struct qman_fqid_pool *pool, u32 fqid);
+u32 qman_fqid_pool_used(struct qman_fqid_pool *pool);
+
+/*******************************************************************/
+/* Managed (aka "shared" or "mux/demux") portal, high-level i/face */
+/*******************************************************************/
+
+	/* Portal and Frame Queues */
+	/* ----------------------- */
+/* Represents a managed portal */
+struct qman_portal;
+
+/* This object type represents Qman frame queue descriptors (FQD), it is
+ * cacheline-aligned, and initialised by qman_create_fq(). The structure is
+ * defined further down. */
+struct qman_fq;
+
+/* This object type represents a Qman congestion group, it is defined further
+ * down. */
+struct qman_cgr;
+
+struct qman_portal_config {
+	/* If the caller enables DQRR stashing (and thus wishes to operate the
+	 * portal from only one cpu), this is the logical CPU that the portal
+	 * will stash to. Whether stashing is enabled or not, this setting is
+	 * also used for any "core-affine" portals, ie. default portals
+	 * associated to the corresponding cpu. -1 implies that there is no core
+	 * affinity configured. */
+	int cpu;
+	/* portal interrupt line */
+	int irq;
+	/* the unique index of this portal */
+	u32 index;
+	/* Is this portal shared? (If so, it has coarser locking and demuxes
+	 * processing on behalf of other CPUs.) */
+	int is_shared;
+	/* The portal's dedicated channel id, use this value for initialising
+	 * frame queues to target this portal when scheduled. */
+	u16 channel;
+	/* A mask of which pool channels this portal has dequeue access to
+	 * (using QM_SDQCR_CHANNELS_POOL(n) for the bitmask) */
+	u32 pools;
+};
+
+/* This enum, and the callback type that returns it, are used when handling
+ * dequeued frames via DQRR. Note that for "null" callbacks registered with the
+ * portal object (for handling dequeues that do not demux because contextB is
+ * NULL), the return value *MUST* be qman_cb_dqrr_consume. */
+enum qman_cb_dqrr_result {
+	/* DQRR entry can be consumed */
+	qman_cb_dqrr_consume,
+	/* Like _consume, but requests parking - FQ must be held-active */
+	qman_cb_dqrr_park,
+	/* Does not consume, for DCA mode only. This allows out-of-order
+	 * consumes by explicit calls to qman_dca() and/or the use of implicit
+	 * DCA via EQCR entries. */
+	qman_cb_dqrr_defer,
+	/* Stop processing without consuming this ring entry. Exits the current
+	 * qman_poll_dqrr() or interrupt-handling, as appropriate. If within an
+	 * interrupt handler, the callback would typically call
+	 * qman_irqsource_remove(QM_PIRQ_DQRI) before returning this value,
+	 * otherwise the interrupt will reassert immediately. */
+	qman_cb_dqrr_stop,
+	/* Like qman_cb_dqrr_stop, but consumes the current entry. */
+	qman_cb_dqrr_consume_stop
+};
+typedef enum qman_cb_dqrr_result (*qman_cb_dqrr)(struct qman_portal *qm,
+					struct qman_fq *fq,
+					const struct qm_dqrr_entry *dqrr);
+
+/* This callback type is used when handling ERNs, FQRNs and FQRLs via MR. They
+ * are always consumed after the callback returns. */
+typedef void (*qman_cb_mr)(struct qman_portal *qm, struct qman_fq *fq,
+				const struct qm_mr_entry *msg);
+
+/* This callback type is used when handling DCP ERNs */
+typedef void (*qman_cb_dc_ern)(struct qman_portal *qm,
+				const struct qm_mr_entry *msg);
+
+/* s/w-visible states. Ie. tentatively scheduled + truly scheduled + active +
+ * held-active + held-suspended are just "sched". Things like "retired" will not
+ * be assumed until it is complete (ie. QMAN_FQ_STATE_CHANGING is set until
+ * then, to indicate it's completing and to gate attempts to retry the retire
+ * command). Note, park commands do not set QMAN_FQ_STATE_CHANGING because it's
+ * technically impossible in the case of enqueue DCAs (which refer to DQRR ring
+ * index rather than the FQ that ring entry corresponds to), so repeated park
+ * commands are allowed (if you're silly enough to try) but won't change FQ
+ * state, and the resulting park notifications move FQs from "sched" to
+ * "parked". */
+enum qman_fq_state {
+	qman_fq_state_oos,
+	qman_fq_state_parked,
+	qman_fq_state_sched,
+	qman_fq_state_retired
+};
+
+/* Frame queue objects (struct qman_fq) are stored within memory passed to
+ * qman_create_fq(), as this allows stashing of caller-provided demux callback
+ * pointers at no extra cost to stashing of (driver-internal) FQ state. If the
+ * caller wishes to add per-FQ state and have it benefit from dequeue-stashing,
+ * they should;
+ *
+ * (a) extend the qman_fq structure with their state; eg.
+ *
+ *     // myfq is allocated and driver_fq callbacks filled in;
+ *     struct my_fq {
+ *         struct qman_fq base;
+ *         int an_extra_field;
+ *         [ ... add other fields to be associated with each FQ ...]
+ *     } *myfq = some_my_fq_allocator();
+ *     struct qman_fq *fq = qman_create_fq(fqid, flags, &myfq->base);
+ *
+ *     // in a dequeue callback, access extra fields from 'fq' via a cast;
+ *     struct my_fq *myfq = (struct my_fq *)fq;
+ *     do_something_with(myfq->an_extra_field);
+ *     [...]
+ *
+ * (b) when and if configuring the FQ for context stashing, specify how ever
+ *     many cachelines are required to stash 'struct my_fq', to accelerate not
+ *     only the Qman driver but the callback as well.
+ */
+
+struct qman_fq_cb {
+	qman_cb_dqrr dqrr;      /* for dequeued frames */
+	qman_cb_mr ern;         /* for s/w ERNs */
+	qman_cb_mr fqs;         /* frame-queue state changes*/
+};
+
+struct qman_fq {
+	/* Caller of qman_create_fq() provides these demux callbacks */
+	struct qman_fq_cb cb;
+	/* These are internal to the driver, don't touch. In particular, they
+	 * may change, be removed, or extended (so you shouldn't rely on
+	 * sizeof(qman_fq) being a constant). */
+	spinlock_t fqlock;
+	u32 fqid;
+	volatile unsigned long flags;
+	enum qman_fq_state state;
+	int cgr_groupid;
+	struct rb_node node;
+#ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
+	u32 key;
+#endif
+};
+
+/* This callback type is used when handling congestion group entry/exit.
+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit. */
+typedef void (*qman_cb_cgr)(struct qman_portal *qm,
+			struct qman_cgr *cgr, int congested);
+
+struct qman_cgr {
+	/* Set these prior to qman_create_cgr() */
+	u32 cgrid; /* 0..255, but u32 to allow specials like -1, 256, etc.*/
+	qman_cb_cgr cb;
+	/* These are private to the driver */
+	u16 chan; /* portal channel this object is created on */
+	struct list_head node;
+};
+
+/* Flags to qman_create_fq() */
+#define QMAN_FQ_FLAG_NO_ENQUEUE      0x00000001 /* can't enqueue */
+#define QMAN_FQ_FLAG_NO_MODIFY       0x00000002 /* can only enqueue */
+#define QMAN_FQ_FLAG_TO_DCPORTAL     0x00000004 /* consumed by CAAM/PME/Fman */
+#define QMAN_FQ_FLAG_LOCKED          0x00000008 /* multi-core locking */
+#define QMAN_FQ_FLAG_AS_IS           0x00000010 /* query h/w state */
+#define QMAN_FQ_FLAG_DYNAMIC_FQID    0x00000020 /* (de)allocate fqid */
+
+/* Flags to qman_destroy_fq() */
+#define QMAN_FQ_DESTROY_PARKED       0x00000001 /* FQ can be parked or OOS */
+
+/* Flags from qman_fq_state() */
+#define QMAN_FQ_STATE_CHANGING       0x80000000 /* 'state' is changing */
+#define QMAN_FQ_STATE_NE             0x40000000 /* retired FQ isn't empty */
+#define QMAN_FQ_STATE_ORL            0x20000000 /* retired FQ has ORL */
+#define QMAN_FQ_STATE_BLOCKOOS       0xe0000000 /* if any are set, no OOS */
+#define QMAN_FQ_STATE_CGR_EN         0x10000000 /* CGR enabled */
+#define QMAN_FQ_STATE_VDQCR          0x08000000 /* being volatile dequeued */
+
+/* Flags to qman_init_fq() */
+#define QMAN_INITFQ_FLAG_SCHED       0x00000001 /* schedule rather than park */
+#define QMAN_INITFQ_FLAG_LOCAL       0x00000004 /* set dest portal */
+
+/* Flags to qman_volatile_dequeue() */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+#define QMAN_VOLATILE_FLAG_WAIT      0x00000001 /* wait if VDQCR is in use */
+#define QMAN_VOLATILE_FLAG_WAIT_INT  0x00000002 /* if wait, interruptible? */
+#define QMAN_VOLATILE_FLAG_FINISH    0x00000004 /* wait till VDQCR completes */
+#endif
+
+/* Flags to qman_enqueue(). NB, the strange numbering is to align with hardware,
+ * bit-wise. (NB: the PME API is sensitive to these precise numberings too, so
+ * any change here should be audited in PME.) */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT
+#define QMAN_ENQUEUE_FLAG_WAIT       0x00010000 /* wait if EQCR is full */
+#define QMAN_ENQUEUE_FLAG_WAIT_INT   0x00020000 /* if wait, interruptible? */
+#ifdef CONFIG_FSL_DPA_CAN_WAIT_SYNC
+#define QMAN_ENQUEUE_FLAG_WAIT_SYNC  0x00000004 /* if wait, until consumed? */
+#endif
+#endif
+#define QMAN_ENQUEUE_FLAG_WATCH_CGR  0x00080000 /* watch congestion state */
+#define QMAN_ENQUEUE_FLAG_DCA        0x00008000 /* perform enqueue-DCA */
+#define QMAN_ENQUEUE_FLAG_DCA_PARK   0x00004000 /* If DCA, requests park */
+#define QMAN_ENQUEUE_FLAG_DCA_PTR(p)		/* If DCA, p is DQRR entry */ \
+		(((u32)(p) << 2) & 0x00000f00)
+#define QMAN_ENQUEUE_FLAG_C_GREEN    0x00000000 /* choose one C_*** flag */
+#define QMAN_ENQUEUE_FLAG_C_YELLOW   0x00000008
+#define QMAN_ENQUEUE_FLAG_C_RED      0x00000010
+#define QMAN_ENQUEUE_FLAG_C_OVERRIDE 0x00000018
+/* For the ORP-specific qman_enqueue_orp() variant;
+ * - this flag indicates "Not Last In Sequence", ie. all but the final fragment
+ *   of a frame. */
+#define QMAN_ENQUEUE_FLAG_NLIS       0x01000000
+/* - this flag performs no enqueue but fills in an ORP sequence number that
+ *   would otherwise block it (eg. if a frame has been dropped). */
+#define QMAN_ENQUEUE_FLAG_HOLE       0x02000000
+/* - this flag performs no enqueue but advances NESN to the given sequence
+ *   number. */
+#define QMAN_ENQUEUE_FLAG_NESN       0x04000000
+
+/* Flags to qman_modify_cgr() */
+#define QMAN_CGR_FLAG_USE_INIT       0x00000001
+#define QMAN_CGR_MODE_FRAME          0x00000001
+
+	/* Portal Management */
+	/* ----------------- */
+/**
+ * qman_get_portal_config - get portal configuration settings
+ *
+ * This returns a read-only view of the current cpu's affine portal settings.
+ */
+const struct qman_portal_config *qman_get_portal_config(void);
+
+/**
+ * qman_irqsource_get - return the portal work that is interrupt-driven
+ *
+ * Returns a bitmask of QM_PIRQ_**I processing sources that are currently
+ * enabled for interrupt handling on the current cpu's affine portal. These
+ * sources will trigger the portal interrupt and the interrupt handler (or a
+ * tasklet/bottom-half it defers to) will perform the corresponding processing
+ * work. The qman_poll_***() functions will only process sources that are not in
+ * this bitmask. If the current CPU is sharing a portal hosted on another CPU,
+ * this always returns zero.
+ */
+u32 qman_irqsource_get(void);
+
+/**
+ * qman_irqsource_add - add processing sources to be interrupt-driven
+ * @bits: bitmask of QM_PIRQ_**I processing sources
+ *
+ * Adds processing sources that should be interrupt-driven (rather than
+ * processed via qman_poll_***() functions). Returns zero for success, or
+ * -EINVAL if the current CPU is sharing a portal hosted on another CPU.
+ */
+int qman_irqsource_add(u32 bits);
+
+/**
+ * qman_irqsource_remove - remove processing sources from being interrupt-driven
+ * @bits: bitmask of QM_PIRQ_**I processing sources
+ *
+ * Removes processing sources from being interrupt-driven, so that they will
+ * instead be processed via qman_poll_***() functions. Returns zero for success,
+ * or -EINVAL if the current CPU is sharing a portal hosted on another CPU.
+ */
+int qman_irqsource_remove(u32 bits);
+
+/**
+ * qman_affine_cpus - return a mask of cpus that have affine portals
+ */
+const cpumask_t *qman_affine_cpus(void);
+
+/**
+ * qman_affine_channel - return the channel ID of an portal
+ * @cpu: the cpu whose affine portal is the subject of the query
+ *
+ * If @cpu is -1, the affine portal for the current CPU will be used. It is a
+ * bug to call this function for any value of @cpu (other than -1) that is not a
+ * member of the mask returned from qman_affine_cpus().
+ */
+u16 qman_affine_channel(int cpu);
+
+/**
+ * qman_get_affine_portal - return the portal pointer affine to cpu
+ * @cpu: the cpu whose affine portal is the subject of the query
+ *
+ */
+void *qman_get_affine_portal(int cpu);
+
+/**
+ * qman_poll_dqrr - process DQRR (fast-path) entries
+ * @limit: the maximum number of DQRR entries to process
+ *
+ * Use of this function requires that DQRR processing not be interrupt-driven.
+ * Ie. the value returned by qman_irqsource_get() should not include
+ * QM_PIRQ_DQRI. If the current CPU is sharing a portal hosted on another CPU,
+ * this function will return -EINVAL, otherwise the return value is >=0 and
+ * represents the number of DQRR entries processed.
+ */
+int qman_poll_dqrr(unsigned int limit);
+
+/**
+ * qman_poll_slow - process anything (except DQRR) that isn't interrupt-driven.
+ *
+ * This function does any portal processing that isn't interrupt-driven. If the
+ * current CPU is sharing a portal hosted on another CPU, this function will
+ * return (u32)-1, otherwise the return value is a bitmask of QM_PIRQ_* sources
+ * indicating what interrupt sources were actually processed by the call.
+ */
+u32 qman_poll_slow(void);
+
+/**
+ * qman_poll - legacy wrapper for qman_poll_dqrr() and qman_poll_slow()
+ *
+ * Dispatcher logic on a cpu can use this to trigger any maintenance of the
+ * affine portal. There are two classes of portal processing in question;
+ * fast-path (which involves demuxing dequeue ring (DQRR) entries and tracking
+ * enqueue ring (EQCR) consumption), and slow-path (which involves EQCR
+ * thresholds, congestion state changes, etc). This function does whatever
+ * processing is not triggered by interrupts.
+ *
+ * Note, if DQRR and some slow-path processing are poll-driven (rather than
+ * interrupt-driven) then this function uses a heuristic to determine how often
+ * to run slow-path processing - as slow-path processing introduces at least a
+ * minimum latency each time it is run, whereas fast-path (DQRR) processing is
+ * close to zero-cost if there is no work to be done. Applications can tune this
+ * behaviour themselves by using qman_poll_dqrr() and qman_poll_slow() directly
+ * rather than going via this wrapper.
+ */
+void qman_poll(void);
+
+/**
+ * qman_stop_dequeues - Stop h/w dequeuing to the s/w portal
+ *
+ * Disables DQRR processing of the portal. This is reference-counted, so
+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to
+ * truly re-enable dequeuing.
+ */
+void qman_stop_dequeues(void);
+
+/**
+ * qman_start_dequeues - (Re)start h/w dequeuing to the s/w portal
+ *
+ * Enables DQRR processing of the portal. This is reference-counted, so
+ * qman_start_dequeues() must be called as many times as qman_stop_dequeues() to
+ * truly re-enable dequeuing.
+ */
+void qman_start_dequeues(void);
+
+/**
+ * qman_static_dequeue_add - Add pool channels to the portal SDQCR
+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n)
+ *
+ * Adds a set of pool channels to the portal's static dequeue command register
+ * (SDQCR). The requested pools are limited to those the portal has dequeue
+ * access to.
+ */
+void qman_static_dequeue_add(u32 pools);
+
+/**
+ * qman_static_dequeue_del - Remove pool channels from the portal SDQCR
+ * @pools: bit-mask of pool channels, using QM_SDQCR_CHANNELS_POOL(n)
+ *
+ * Removes a set of pool channels from the portal's static dequeue command
+ * register (SDQCR). The requested pools are limited to those the portal has
+ * dequeue access to.
+ */
+void qman_static_dequeue_del(u32 pools);
+
+/**
+ * qman_static_dequeue_get - return the portal's current SDQCR
+ *
+ * Returns the portal's current static dequeue command register (SDQCR). The
+ * entire register is returned, so if only the currently-enabled pool channels
+ * are desired, mask the return value with QM_SDQCR_CHANNELS_POOL_MASK.
+ */
+u32 qman_static_dequeue_get(void);
+
+/**
+ * qman_dca - Perform a Discrete Consumption Acknowledgement
+ * @dq: the DQRR entry to be consumed
+ * @park_request: indicates whether the held-active @fq should be parked
+ *
+ * Only allowed in DCA-mode portals, for DQRR entries whose handler callback had
+ * previously returned 'qman_cb_dqrr_defer'. NB, as with the other APIs, this
+ * does not take a 'portal' argument but implies the core affine portal from the
+ * cpu that is currently executing the function. For reasons of locking, this
+ * function must be called from the same CPU as that which processed the DQRR
+ * entry in the first place.
+ */
+void qman_dca(struct qm_dqrr_entry *dq, int park_request);
+
+/**
+ * qman_eqcr_is_empty - Determine if portal's EQCR is empty
+ *
+ * For use in situations where a cpu-affine caller needs to determine when all
+ * enqueues for the local portal have been processed by Qman but can't use the
+ * QMAN_ENQUEUE_FLAG_WAIT_SYNC flag to do this from the final qman_enqueue().
+ * The function forces tracking of EQCR consumption (which normally doesn't
+ * happen until enqueue processing needs to find space to put new enqueue
+ * commands), and returns zero if the ring still has unprocessed entries,
+ * non-zero if it is empty.
+ */
+int qman_eqcr_is_empty(void);
+
+/**
+ * qman_set_dc_ern - Set the handler for DCP enqueue rejection notifications
+ * @handler: callback for processing DCP ERNs
+ * @affine: whether this handler is specific to the locally affine portal
+ *
+ * If a hardware block's interface to Qman (ie. its direct-connect portal, or
+ * DCP) is configured not to receive enqueue rejections, then any enqueues
+ * through that DCP that are rejected will be sent to a given software portal.
+ * If @affine is non-zero, then this handler will only be used for DCP ERNs
+ * received on the portal affine to the current CPU. If multiple CPUs share a
+ * portal and they all call this function, they will be setting the handler for
+ * the same portal! If @affine is zero, then this handler will be global to all
+ * portals handled by this instance of the driver. Only those portals that do
+ * not have their own affine handler will use the global handler.
+ */
+void qman_set_dc_ern(qman_cb_dc_ern handler, int affine);
+
+	/* FQ management */
+	/* ------------- */
+/**
+ * qman_create_fq - Allocates a FQ
+ * @fqid: the index of the FQD to encapsulate, must be "Out of Service"
+ * @flags: bit-mask of QMAN_FQ_FLAG_*** options
+ * @fq: memory for storing the 'fq', with callbacks filled in
+ *
+ * Creates a frame queue object for the given @fqid, unless the
+ * QMAN_FQ_FLAG_DYNAMIC_FQID flag is set in @flags, in which case a FQID is
+ * dynamically allocated (or the function fails if none are available). Once
+ * created, the caller should not touch the memory at 'fq' except as extended to
+ * adjacent memory for user-defined fields (see the definition of "struct
+ * qman_fq" for more info). NO_MODIFY is only intended for enqueuing to
+ * pre-existing frame-queues that aren't to be otherwise interfered with, it
+ * prevents all other modifications to the frame queue. The TO_DCPORTAL flag
+ * causes the driver to honour any contextB modifications requested in the
+ * qm_init_fq() API, as this indicates the frame queue will be consumed by a
+ * direct-connect portal (PME, CAAM, or Fman). When frame queues are consumed by
+ * software portals, the contextB field is controlled by the driver and can't be
+ * modified by the caller. If the AS_IS flag is specified, management commands
+ * will be used on portal @p to query state for frame queue @fqid and construct
+ * a frame queue object based on that, rather than assuming/requiring that it be
+ * Out of Service.
+ */
+int qman_create_fq(u32 fqid, u32 flags, struct qman_fq *fq);
+
+/**
+ * qman_destroy_fq - Deallocates a FQ
+ * @fq: the frame queue object to release
+ * @flags: bit-mask of QMAN_FQ_FREE_*** options
+ *
+ * The memory for this frame queue object ('fq' provided in qman_create_fq()) is
+ * not deallocated but the caller regains ownership, to do with as desired. The
+ * FQ must be in the 'out-of-service' state unless the QMAN_FQ_FREE_PARKED flag
+ * is specified, in which case it may also be in the 'parked' state.
+ */
+void qman_destroy_fq(struct qman_fq *fq, u32 flags);
+
+/**
+ * qman_fq_fqid - Queries the frame queue ID of a FQ object
+ * @fq: the frame queue object to query
+ */
+u32 qman_fq_fqid(struct qman_fq *fq);
+
+/**
+ * qman_fq_state - Queries the state of a FQ object
+ * @fq: the frame queue object to query
+ * @state: pointer to state enum to return the FQ scheduling state
+ * @flags: pointer to state flags to receive QMAN_FQ_STATE_*** bitmask
+ *
+ * Queries the state of the FQ object, without performing any h/w commands.
+ * This captures the state, as seen by the driver, at the time the function
+ * executes.
+ */
+void qman_fq_state(struct qman_fq *fq, enum qman_fq_state *state, u32 *flags);
+
+/**
+ * qman_init_fq - Initialises FQ fields, leaves the FQ "parked" or "scheduled"
+ * @fq: the frame queue object to modify, must be 'parked' or new.
+ * @flags: bit-mask of QMAN_INITFQ_FLAG_*** options
+ * @opts: the FQ-modification settings, as defined in the low-level API
+ *
+ * The @opts parameter comes from the low-level portal API. Select
+ * QMAN_INITFQ_FLAG_SCHED in @flags to cause the frame queue to be scheduled
+ * rather than parked. NB, @opts can be NULL.
+ *
+ * Note that some fields and options within @opts may be ignored or overwritten
+ * by the driver;
+ * 1. the 'count' and 'fqid' fields are always ignored (this operation only
+ * affects one frame queue: @fq).
+ * 2. the QM_INITFQ_WE_CONTEXTB option of the 'we_mask' field and the associated
+ * 'fqd' structure's 'context_b' field are sometimes overwritten;
+ *   - if @fq was not created with QMAN_FQ_FLAG_TO_DCPORTAL, then context_b is
+ *     initialised to a value used by the driver for demux.
+ *   - if context_b is initialised for demux, so is context_a in case stashing
+ *     is requested (see item 4).
+ * (So caller control of context_b is only possible for TO_DCPORTAL frame queue
+ * objects.)
+ * 3. if @flags contains QMAN_INITFQ_FLAG_LOCAL, the 'fqd' structure's
+ * 'dest::channel' field will be overwritten to match the portal used to issue
+ * the command. If the WE_DESTWQ write-enable bit had already been set by the
+ * caller, the channel workqueue will be left as-is, otherwise the write-enable
+ * bit is set and the workqueue is set to a default of 4. If the "LOCAL" flag
+ * isn't set, the destination channel/workqueue fields and the write-enable bit
+ * are left as-is.
+ * 4. if the driver overwrites context_a/b for demux, then if
+ * QM_INITFQ_WE_CONTEXTA is set, the driver will only overwrite
+ * context_a.address fields and will leave the stashing fields provided by the
+ * user alone, otherwise it will zero out the context_a.stashing fields.
+ */
+int qman_init_fq(struct qman_fq *fq, u32 flags, struct qm_mcc_initfq *opts);
+
+/**
+ * qman_schedule_fq - Schedules a FQ
+ * @fq: the frame queue object to schedule, must be 'parked'
+ *
+ * Schedules the frame queue, which must be Parked, which takes it to
+ * Tentatively-Scheduled or Truly-Scheduled depending on its fill-level.
+ */
+int qman_schedule_fq(struct qman_fq *fq);
+
+/**
+ * qman_retire_fq - Retires a FQ
+ * @fq: the frame queue object to retire
+ * @flags: FQ flags (as per qman_fq_state) if retirement completes immediately
+ *
+ * Retires the frame queue. This returns zero if it succeeds immediately, +1 if
+ * the retirement was started asynchronously, otherwise it returns negative for
+ * failure. When this function returns zero, @flags is set to indicate whether
+ * the retired FQ is empty and/or whether it has any ORL fragments (to show up
+ * as ERNs). Otherwise the corresponding flags will be known when a subsequent
+ * FQRN message shows up on the portal's message ring.
+ *
+ * NB, if the retirement is asynchronous (the FQ was in the Truly Scheduled or
+ * Active state), the completion will be via the message ring as a FQRN - but
+ * the corresponding callback may occur before this function returns!! Ie. the
+ * caller should be prepared to accept the callback as the function is called,
+ * not only once it has returned.
+ */
+int qman_retire_fq(struct qman_fq *fq, u32 *flags);
+
+/**
+ * qman_oos_fq - Puts a FQ "out of service"
+ * @fq: the frame queue object to be put out-of-service, must be 'retired'
+ *
+ * The frame queue must be retired and empty, and if any order restoration list
+ * was released as ERNs at the time of retirement, they must all be consumed.
+ */
+int qman_oos_fq(struct qman_fq *fq);
+
+/**
+ * qman_fq_flow_control - Set the XON/XOFF state of a FQ
+ * @fq: the frame queue object to be set to XON/XOFF state, must not be 'oos',
+ * or 'retired' or 'parked' state
+ * @xon: boolean to set fq in XON or XOFF state
+ *
+ * The frame should be in Tentatively Scheduled state or Truly Schedule sate,
+ * otherwise the IFSI interrupt will be asserted.
+ */
+int qman_fq_flow_control(struct qman_fq *fq, int xon);
+
+/**
+ * qman_query_fq - Queries FQD fields (via h/w query command)
+ * @fq: the frame queue object to be queried
+ * @fqd: storage for the queried FQD fields
+ */
+int qman_query_fq(struct qman_fq *fq, struct qm_fqd *fqd);
+
+/**
+ * qman_query_fq_np - Queries non-programmable FQD fields
+ * @fq: the frame queue object to be queried
+ * @np: storage for the queried FQD fields
+ */
+int qman_query_fq_np(struct qman_fq *fq, struct qm_mcr_queryfq_np *np);
+
+/**
+ * qman_query_wq - Queries work queue lengths
+ * @query_dedicated: If non-zero, query length of WQs in the channel dedicated
+ *		to this software portal. Otherwise, query length of WQs in a
+ *		channel  specified in wq.
+ * @wq: storage for the queried WQs lengths. Also specified the channel to
+ *	to query if query_dedicated is zero.
+ */
+int qman_query_wq(u8 query_dedicated, struct qm_mcr_querywq *wq);
+
+/**
+ * qman_volatile_dequeue - Issue a volatile dequeue command
+ * @fq: the frame queue object to dequeue from
+ * @flags: a bit-mask of QMAN_VOLATILE_FLAG_*** options
+ * @vdqcr: bit mask of QM_VDQCR_*** options, as per qm_dqrr_vdqcr_set()
+ *
+ * Attempts to lock access to the portal's VDQCR volatile dequeue functionality.
+ * The function will block and sleep if QMAN_VOLATILE_FLAG_WAIT is specified and
+ * the VDQCR is already in use, otherwise returns non-zero for failure. If
+ * QMAN_VOLATILE_FLAG_FINISH is specified, the function will only return once
+ * the VDQCR command has finished executing (ie. once the callback for the last
+ * DQRR entry resulting from the VDQCR command has been called). If not using
+ * the FINISH flag, completion can be determined either by detecting the
+ * presence of the QM_DQRR_STAT_UNSCHEDULED and QM_DQRR_STAT_DQCR_EXPIRED bits
+ * in the "stat" field of the "struct qm_dqrr_entry" passed to the FQ's dequeue
+ * callback, or by waiting for the QMAN_FQ_STATE_VDQCR bit to disappear from the
+ * "flags" retrieved from qman_fq_state().
+ */
+int qman_volatile_dequeue(struct qman_fq *fq, u32 flags, u32 vdqcr);
+
+/**
+ * qman_enqueue - Enqueue a frame to a frame queue
+ * @fq: the frame queue object to enqueue to
+ * @fd: a descriptor of the frame to be enqueued
+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
+ *
+ * Fills an entry in the EQCR of portal @qm to enqueue the frame described by
+ * @fd. The descriptor details are copied from @fd to the EQCR entry, the 'pid'
+ * field is ignored. The return value is non-zero on error, such as ring full
+ * (and FLAG_WAIT not specified), congestion avoidance (FLAG_WATCH_CGR
+ * specified), etc. If the ring is full and FLAG_WAIT is specified, this
+ * function will block. If FLAG_INTERRUPT is set, the EQCI bit of the portal
+ * interrupt will assert when Qman consumes the EQCR entry (subject to "status
+ * disable", "enable", and "inhibit" registers). If FLAG_DCA is set, Qman will
+ * perform an implied "discrete consumption acknowledgement" on the dequeue
+ * ring's (DQRR) entry, at the ring index specified by the FLAG_DCA_IDX(x)
+ * macro. (As an alternative to issuing explicit DCA actions on DQRR entries,
+ * this implicit DCA can delay the release of a "held active" frame queue
+ * corresponding to a DQRR entry until Qman consumes the EQCR entry - providing
+ * order-preservation semantics in packet-forwarding scenarios.) If FLAG_DCA is
+ * set, then FLAG_DCA_PARK can also be set to imply that the DQRR consumption
+ * acknowledgement should "park request" the "held active" frame queue. Ie.
+ * when the portal eventually releases that frame queue, it will be left in the
+ * Parked state rather than Tentatively Scheduled or Truly Scheduled. If the
+ * portal is watching congestion groups, the QMAN_ENQUEUE_FLAG_WATCH_CGR flag
+ * is requested, and the FQ is a member of a congestion group, then this
+ * function returns -EAGAIN if the congestion group is currently congested.
+ * Note, this does not eliminate ERNs, as the async interface means we can be
+ * sending enqueue commands to an un-congested FQ that becomes congested before
+ * the enqueue commands are processed, but it does minimise needless thrashing
+ * of an already busy hardware resource by throttling many of the to-be-dropped
+ * enqueues "at the source".
+ */
+int qman_enqueue(struct qman_fq *fq, const struct qm_fd *fd, u32 flags);
+
+typedef int (*qman_cb_precommit) (void *arg);
+/**
+ * qman_enqueue_precommit - Enqueue a frame to a frame queue and call cb
+ * @fq: the frame queue object to enqueue to
+ * @fd: a descriptor of the frame to be enqueued
+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
+ * @cb: user supplied callback function to invoke before writing commit verb.
+ * @cb_arg: callback function argument
+ *
+ * This is similar to qman_enqueue except that it will invoke a user supplied
+ * callback function just before writng the commit verb. This is useful
+ * when the user want to do something *just before* enqueuing the request and
+ * the enqueue can't fail.
+ */
+int qman_enqueue_precommit(struct qman_fq *fq, const struct qm_fd *fd,
+		u32 flags, qman_cb_precommit cb, void *cb_arg);
+
+/**
+ * qman_enqueue_orp - Enqueue a frame to a frame queue using an ORP
+ * @fq: the frame queue object to enqueue to
+ * @fd: a descriptor of the frame to be enqueued
+ * @flags: bit-mask of QMAN_ENQUEUE_FLAG_*** options
+ * @orp: the frame queue object used as an order restoration point.
+ * @orp_seqnum: the sequence number of this frame in the order restoration path
+ *
+ * Similar to qman_enqueue(), but with the addition of an Order Restoration
+ * Point (@orp) and corresponding sequence number (@orp_seqnum) for this
+ * enqueue operation to employ order restoration. Each frame queue object acts
+ * as an Order Definition Point (ODP) by providing each frame dequeued from it
+ * with an incrementing sequence number, this value is generally ignored unless
+ * that sequence of dequeued frames will need order restoration later. Each
+ * frame queue object also encapsulates an Order Restoration Point (ORP), which
+ * is a re-assembly context for re-ordering frames relative to their sequence
+ * numbers as they are enqueued. The ORP does not have to be within the frame
+ * queue that receives the enqueued frame, in fact it is usually the frame
+ * queue from which the frames were originally dequeued. For the purposes of
+ * order restoration, multiple frames (or "fragments") can be enqueued for a
+ * single sequence number by setting the QMAN_ENQUEUE_FLAG_NLIS flag for all
+ * enqueues except the final fragment of a given sequence number. Ordering
+ * between sequence numbers is guaranteed, even if fragments of different
+ * sequence numbers are interlaced with one another. Fragments of the same
+ * sequence number will retain the order in which they are enqueued. If no
+ * enqueue is to performed, QMAN_ENQUEUE_FLAG_HOLE indicates that the given
+ * sequence number is to be "skipped" by the ORP logic (eg. if a frame has been
+ * dropped from a sequence), or QMAN_ENQUEUE_FLAG_NESN indicates that the given
+ * sequence number should become the ORP's "Next Expected Sequence Number".
+ *
+ * Side note: a frame queue object can be used purely as an ORP, without
+ * carrying any frames at all. Care should be taken not to deallocate a frame
+ * queue object that is being actively used as an ORP, as a future allocation
+ * of the frame queue object may start using the internal ORP before the
+ * previous use has finished.
+ */
+int qman_enqueue_orp(struct qman_fq *fq, const struct qm_fd *fd, u32 flags,
+			struct qman_fq *orp, u16 orp_seqnum);
+
+/**
+ * qman_alloc_fqid_range - Allocate a contiguous range of FQIDs
+ * @result: is set by the API to the base FQID of the allocated range
+ * @count: the number of FQIDs required
+ * @align: required alignment of the allocated range
+ * @partial: non-zero if the API can return fewer than @count FQIDs
+ *
+ * Returns the number of frame queues allocated, or a negative error code. If
+ * @partial is non zero, the allocation request may return a smaller range of
+ * FQs than requested (though alignment will be as requested). If @partial is
+ * zero, the return value will either be 'count' or negative.
+ */
+int qman_alloc_fqid_range(u32 *result, u32 count, u32 align, int partial);
+static inline int qman_alloc_fqid(u32 *result)
+{
+	int ret = qman_alloc_fqid_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+
+/**
+ * qman_release_fqid_range - Release the specified range of frame queue IDs
+ * @fqid: the base FQID of the range to deallocate
+ * @count: the number of FQIDs in the range
+ *
+ * This function can also be used to seed the allocator with ranges of FQIDs
+ * that it can subsequently allocate from.
+ */
+void qman_release_fqid_range(u32 fqid, unsigned int count);
+static inline void qman_release_fqid(u32 fqid)
+{
+	qman_release_fqid_range(fqid, 1);
+}
+
+void qman_seed_fqid_range(u32 fqid, unsigned int count);
+
+
+int qman_shutdown_fq(u32 fqid);
+
+/**
+ * qman_reserve_fqid_range - Reserve the specified range of frame queue IDs
+ * @fqid: the base FQID of the range to deallocate
+ * @count: the number of FQIDs in the range
+ */
+int qman_reserve_fqid_range(u32 fqid, unsigned int count);
+static inline int qman_reserve_fqid(u32 fqid)
+{
+	return qman_reserve_fqid_range(fqid, 1);
+}
+
+	/* Pool-channel management */
+	/* ----------------------- */
+/**
+ * qman_alloc_pool_range - Allocate a contiguous range of pool-channel IDs
+ * @result: is set by the API to the base pool-channel ID of the allocated range
+ * @count: the number of pool-channel IDs required
+ * @align: required alignment of the allocated range
+ * @partial: non-zero if the API can return fewer than @count
+ *
+ * Returns the number of pool-channel IDs allocated, or a negative error code.
+ * If @partial is non zero, the allocation request may return a smaller range of
+ * than requested (though alignment will be as requested). If @partial is zero,
+ * the return value will either be 'count' or negative.
+ */
+int qman_alloc_pool_range(u32 *result, u32 count, u32 align, int partial);
+static inline int qman_alloc_pool(u32 *result)
+{
+	int ret = qman_alloc_pool_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+
+/**
+ * qman_release_pool_range - Release the specified range of pool-channel IDs
+ * @id: the base pool-channel ID of the range to deallocate
+ * @count: the number of pool-channel IDs in the range
+ */
+void qman_release_pool_range(u32 id, unsigned int count);
+static inline void qman_release_pool(u32 id)
+{
+	qman_release_pool_range(id, 1);
+}
+
+/**
+ * qman_reserve_pool_range - Reserve the specified range of pool-channel IDs
+ * @id: the base pool-channel ID of the range to reserve
+ * @count: the number of pool-channel IDs in the range
+ */
+int qman_reserve_pool_range(u32 id, unsigned int count);
+static inline int qman_reserve_pool(u32 id)
+{
+	return qman_reserve_pool_range(id, 1);
+}
+
+void qman_seed_pool_range(u32 id, unsigned int count);
+
+	/* CGR management */
+	/* -------------- */
+/**
+ * qman_create_cgr - Register a congestion group object
+ * @cgr: the 'cgr' object, with fields filled in
+ * @flags: QMAN_CGR_FLAG_* values
+ * @opts: optional state of CGR settings
+ *
+ * Registers this object to receiving congestion entry/exit callbacks on the
+ * portal affine to the cpu portal on which this API is executed. If opts is
+ * NULL then only the callback (cgr->cb) function is registered. If @flags
+ * contains QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset
+ * any unspecified parameters) will be used rather than a modify hw hardware
+ * (which only modifies the specified parameters).
+ */
+int qman_create_cgr(struct qman_cgr *cgr, u32 flags,
+			struct qm_mcc_initcgr *opts);
+
+/**
+ * qman_create_cgr_to_dcp - Register a congestion group object to DCP portal
+ * @cgr: the 'cgr' object, with fields filled in
+ * @flags: QMAN_CGR_FLAG_* values
+ * @dcp_portal: the DCP portal to which the cgr object is registered.
+ * @opts: optional state of CGR settings
+ *
+ */
+int qman_create_cgr_to_dcp(struct qman_cgr *cgr, u32 flags, u16 dcp_portal,
+				struct qm_mcc_initcgr *opts);
+
+/**
+ * qman_delete_cgr - Deregisters a congestion group object
+ * @cgr: the 'cgr' object to deregister
+ *
+ * "Unplugs" this CGR object from the portal affine to the cpu on which this API
+ * is executed. This must be excuted on the same affine portal on which it was
+ * created.
+ */
+int qman_delete_cgr(struct qman_cgr *cgr);
+
+/**
+ * qman_delete_cgr_safe - Deregisters a congestion group object from any CPU
+ * @cgr: the 'cgr' object to deregister
+ *
+ * This will select the proper CPU and run there qman_delete_cgr().
+ */
+void qman_delete_cgr_safe(struct qman_cgr *cgr);
+
+/**
+ * qman_modify_cgr - Modify CGR fields
+ * @cgr: the 'cgr' object to modify
+ * @flags: QMAN_CGR_FLAG_* values
+ * @opts: the CGR-modification settings
+ *
+ * The @opts parameter comes from the low-level portal API, and can be NULL.
+ * Note that some fields and options within @opts may be ignored or overwritten
+ * by the driver, in particular the 'cgrid' field is ignored (this operation
+ * only affects the given CGR object). If @flags contains
+ * QMAN_CGR_FLAG_USE_INIT, then an init hw command (which will reset any
+ * unspecified parameters) will be used rather than a modify hw hardware (which
+ * only modifies the specified parameters).
+ */
+int qman_modify_cgr(struct qman_cgr *cgr, u32 flags,
+			struct qm_mcc_initcgr *opts);
+
+/**
+* qman_query_cgr - Queries CGR fields
+* @cgr: the 'cgr' object to query
+* @result: storage for the queried congestion group record
+*/
+int qman_query_cgr(struct qman_cgr *cgr, struct qm_mcr_querycgr *result);
+
+/**
+ * qman_query_congestion - Queries the state of all congestion groups
+ * @congestion: storage for the queried state of all congestion groups
+ */
+int qman_query_congestion(struct qm_mcr_querycongestion *congestion);
+
+/**
+ * qman_alloc_cgrid_range - Allocate a contiguous range of CGR IDs
+ * @result: is set by the API to the base CGR ID of the allocated range
+ * @count: the number of CGR IDs required
+ * @align: required alignment of the allocated range
+ * @partial: non-zero if the API can return fewer than @count
+ *
+ * Returns the number of CGR IDs allocated, or a negative error code.
+ * If @partial is non zero, the allocation request may return a smaller range of
+ * than requested (though alignment will be as requested). If @partial is zero,
+ * the return value will either be 'count' or negative.
+ */
+int qman_alloc_cgrid_range(u32 *result, u32 count, u32 align, int partial);
+static inline int qman_alloc_cgrid(u32 *result)
+{
+	int ret = qman_alloc_cgrid_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+
+/**
+ * qman_release_cgrid_range - Release the specified range of CGR IDs
+ * @id: the base CGR ID of the range to deallocate
+ * @count: the number of CGR IDs in the range
+ */
+void qman_release_cgrid_range(u32 id, unsigned int count);
+static inline void qman_release_cgrid(u32 id)
+{
+	qman_release_cgrid_range(id, 1);
+}
+
+/**
+ * qman_reserve_cgrid_range - Reserve the specified range of CGR ID
+ * @id: the base CGR ID of the range to reserve
+ * @count: the number of CGR IDs in the range
+ */
+int qman_reserve_cgrid_range(u32 id, unsigned int count);
+static inline int qman_reserve_cgrid(u32 id)
+{
+	return qman_reserve_cgrid_range(id, 1);
+}
+
+void qman_seed_cgrid_range(u32 id, unsigned int count);
+
+
+	/* Helpers */
+	/* ------- */
+/**
+ * qman_poll_fq_for_init - Check if an FQ has been initialised from OOS
+ * @fqid: the FQID that will be initialised by other s/w
+ *
+ * In many situations, a FQID is provided for communication between s/w
+ * entities, and whilst the consumer is responsible for initialising and
+ * scheduling the FQ, the producer(s) generally create a wrapper FQ object using
+ * and only call qman_enqueue() (no FQ initialisation, scheduling, etc). Ie;
+ *     qman_create_fq(..., QMAN_FQ_FLAG_NO_MODIFY, ...);
+ * However, data can not be enqueued to the FQ until it is initialised out of
+ * the OOS state - this function polls for that condition. It is particularly
+ * useful for users of IPC functions - each endpoint's Rx FQ is the other
+ * endpoint's Tx FQ, so each side can initialise and schedule their Rx FQ object
+ * and then use this API on the (NO_MODIFY) Tx FQ object in order to
+ * synchronise. The function returns zero for success, +1 if the FQ is still in
+ * the OOS state, or negative if there was an error.
+ */
+static inline int qman_poll_fq_for_init(struct qman_fq *fq)
+{
+	struct qm_mcr_queryfq_np np;
+	int err;
+	err = qman_query_fq_np(fq, &np);
+	if (err)
+		return err;
+	if ((np.state & QM_MCR_NP_STATE_MASK) == QM_MCR_NP_STATE_OOS)
+		return 1;
+	return 0;
+}
+
+	/* -------------- */
+	/* CEETM :: types */
+	/* -------------- */
+/**
+ * Token Rate Structure
+ * Shaping rates are based on a "credit" system and a pre-configured h/w
+ * internal timer. The following type represents a shaper "rate" parameter as a
+ * fractional number of "tokens". Here's how it works. This (fractional) number
+ * of tokens is added to the shaper's "credit" every time the h/w timer elapses
+ * (up to a limit which is set by another shaper parameter). Every time a frame
+ * is enqueued through a shaper, the shaper deducts as many tokens as there are
+ * bytes of data in the enqueued frame. A shaper will not allow itself to
+ * enqueue any frames if its token count is negative. As such;
+ *
+ *         The rate at which data is enqueued is limited by the
+ *         rate at which tokens are added.
+ *
+ * Therefore if the user knows the period between these h/w timer updates in
+ * seconds, they can calculate the maximum traffic rate of the shaper (in
+ * bytes-per-second) from the token rate. And vice versa, they can calculate
+ * the token rate to use in order to achieve a given traffic rate.
+ */
+struct qm_ceetm_rate {
+	/* The token rate is; whole + (fraction/8192) */
+	u32 whole:11; /* 0..2047 */
+	u32 fraction:13; /* 0..8191 */
+};
+
+struct qm_ceetm_weight_code {
+	/* The weight code is; 5 msbits + 3 lsbits */
+	u8 y:5;
+	u8 x:3;
+};
+
+struct qm_ceetm {
+	unsigned int idx;
+	struct list_head sub_portals;
+	struct list_head lnis;
+	unsigned int sp_range[2];
+	unsigned int lni_range[2];
+};
+
+struct qm_ceetm_sp {
+	struct list_head node;
+	unsigned int idx;
+	unsigned int dcp_idx;
+	int is_claimed;
+	struct qm_ceetm_lni *lni;
+};
+
+/* Logical Network Interface */
+struct qm_ceetm_lni {
+	struct list_head node;
+	unsigned int idx;
+	unsigned int dcp_idx;
+	int is_claimed;
+	struct qm_ceetm_sp *sp;
+	struct list_head channels;
+	int shaper_enable;
+	int shaper_couple;
+	int oal;
+	struct qm_ceetm_rate cr_token_rate;
+	struct qm_ceetm_rate er_token_rate;
+	u16 cr_token_bucket_limit;
+	u16 er_token_bucket_limit;
+};
+
+/* Class Queue Channel */
+struct qm_ceetm_channel {
+	struct list_head node;
+	unsigned int idx;
+	unsigned int lni_idx;
+	unsigned int dcp_idx;
+	struct list_head class_queues;
+	struct list_head ccgs;
+	u8 shaper_enable;
+	u8 shaper_couple;
+	struct qm_ceetm_rate cr_token_rate;
+	struct qm_ceetm_rate er_token_rate;
+	u16 cr_token_bucket_limit;
+	u16 er_token_bucket_limit;
+};
+
+struct qm_ceetm_ccg;
+
+/* This callback type is used when handling congestion entry/exit. The
+ * 'cb_ctx' value is the opaque value associated with ccg object.
+ * 'congested' is non-zero on congestion-entry, and zero on congestion-exit.
+ */
+typedef void (*qman_cb_ccgr)(struct qm_ceetm_ccg *ccg, void *cb_ctx,
+							int congested);
+
+/* Class Congestion Group */
+struct qm_ceetm_ccg {
+	struct qm_ceetm_channel *parent;
+	struct list_head node;
+	struct list_head cb_node;
+	qman_cb_ccgr cb;
+	void *cb_ctx;
+	unsigned int idx;
+};
+
+/* Class Queue */
+struct qm_ceetm_cq {
+	struct qm_ceetm_channel *parent;
+	struct qm_ceetm_ccg *ccg;
+	struct list_head node;
+	unsigned int idx;
+	int is_claimed;
+	struct list_head bound_lfqids;
+	struct list_head binding_node;
+};
+
+/* Logical Frame Queue */
+struct qm_ceetm_lfq {
+	struct qm_ceetm_channel *parent;
+	struct list_head node;
+	unsigned int idx;
+	unsigned int dctidx;
+	u64 context_a;
+	u32 context_b;
+	qman_cb_mr ern;
+};
+
+/**
+ * qman_ceetm_bps2tokenrate - Given a desired rate 'bps' measured in bps
+ * (ie. bits-per-second), compute the 'token_rate' fraction that best
+ * approximates that rate.
+ * @bps: the desired shaper rate in bps.
+ * @token_rate: the output token rate computed with the given kbps.
+ * @rounding: dictates how to round if an exact conversion is not possible; if
+ * it is negative then 'token_rate' will round down to the highest value that
+ * does not exceed the desired rate, if it is positive then 'token_rate' will
+ * round up to the lowest value that is greater than or equal to the desired
+ * rate, and if it is zero then it will round to the nearest approximation,
+ * whether that be up or down.
+ *
+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available.
+  */
+int qman_ceetm_bps2tokenrate(u64 bps,
+				struct qm_ceetm_rate *token_rate,
+				int rounding);
+
+/**
+ * qman_ceetm_tokenrate2bps - Given a 'token_rate', compute the
+ * corresponding number of 'bps'.
+ * @token_rate: the input desired token_rate fraction.
+ * @bps: the output shaper rate in bps computed with the give token rate.
+ * @rounding: has the same semantics as the previous function.
+ *
+ * Return 0 for success, or -EINVAL if prescaler or qman clock is not available.
+ */
+int qman_ceetm_tokenrate2bps(const struct qm_ceetm_rate *token_rate,
+			      u64 *bps,
+			      int rounding);
+
+int qman_alloc_ceetm0_channel_range(u32 *result, u32 count, u32 align,
+								int partial);
+static inline int qman_alloc_ceetm0_channel(u32 *result)
+{
+	int ret = qman_alloc_ceetm0_channel_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+void qman_release_ceetm0_channel_range(u32 channelid, u32 count);
+static inline void qman_release_ceetm0_channelid(u32 channelid)
+{
+	qman_release_ceetm0_channel_range(channelid, 1);
+}
+
+int qman_reserve_ceetm0_channel_range(u32 channelid, u32 count);
+static inline int qman_reserve_ceetm0_channelid(u32 channelid)
+{
+	return qman_reserve_ceetm0_channel_range(channelid, 1);
+}
+
+void qman_seed_ceetm0_channel_range(u32 channelid, u32 count);
+
+
+int qman_alloc_ceetm1_channel_range(u32 *result, u32 count, u32 align,
+								int partial);
+static inline int qman_alloc_ceetm1_channel(u32 *result)
+{
+	int ret = qman_alloc_ceetm1_channel_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+void qman_release_ceetm1_channel_range(u32 channelid, u32 count);
+static inline void qman_release_ceetm1_channelid(u32 channelid)
+{
+	qman_release_ceetm1_channel_range(channelid, 1);
+}
+int qman_reserve_ceetm1_channel_range(u32 channelid, u32 count);
+static inline int qman_reserve_ceetm1_channelid(u32 channelid)
+{
+	return qman_reserve_ceetm1_channel_range(channelid, 1);
+}
+
+void qman_seed_ceetm1_channel_range(u32 channelid, u32 count);
+
+
+int qman_alloc_ceetm0_lfqid_range(u32 *result, u32 count, u32 align,
+								int partial);
+static inline int qman_alloc_ceetm0_lfqid(u32 *result)
+{
+	int ret = qman_alloc_ceetm0_lfqid_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+void qman_release_ceetm0_lfqid_range(u32 lfqid, u32 count);
+static inline void qman_release_ceetm0_lfqid(u32 lfqid)
+{
+	qman_release_ceetm0_lfqid_range(lfqid, 1);
+}
+int qman_reserve_ceetm0_lfqid_range(u32 lfqid, u32 count);
+static inline int qman_reserve_ceetm0_lfqid(u32 lfqid)
+{
+	return qman_reserve_ceetm0_lfqid_range(lfqid, 1);
+}
+
+void qman_seed_ceetm0_lfqid_range(u32 lfqid, u32 count);
+
+
+int qman_alloc_ceetm1_lfqid_range(u32 *result, u32 count, u32 align,
+								int partial);
+static inline int qman_alloc_ceetm1_lfqid(u32 *result)
+{
+	int ret = qman_alloc_ceetm1_lfqid_range(result, 1, 0, 0);
+	return (ret > 0) ? 0 : ret;
+}
+void qman_release_ceetm1_lfqid_range(u32 lfqid, u32 count);
+static inline void qman_release_ceetm1_lfqid(u32 lfqid)
+{
+	qman_release_ceetm1_lfqid_range(lfqid, 1);
+}
+int qman_reserve_ceetm1_lfqid_range(u32 lfqid, u32 count);
+static inline int qman_reserve_ceetm1_lfqid(u32 lfqid)
+{
+	return qman_reserve_ceetm1_lfqid_range(lfqid, 1);
+}
+
+void qman_seed_ceetm1_lfqid_range(u32 lfqid, u32 count);
+
+
+	/* ----------------------------- */
+	/* CEETM :: sub-portals          */
+	/* ----------------------------- */
+
+/**
+ * qman_ceetm_claim_sp - Claims the given sub-portal, provided it is available
+ * to us and configured for traffic-management.
+ * @sp: the returned sub-portal object, if successful.
+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM
+ * instance),
+ * @sp_idx" is the desired sub-portal index from 0 to 15.
+ *
+ * Returns zero for success, or -ENODEV if the sub-portal is in use,  or -EINVAL
+ * if the sp_idx is out of range.
+ *
+ * Note that if there are multiple driver domains (eg. a linux kernel versus
+ * user-space drivers in USDPAA, or multiple guests running under a hypervisor)
+ * then a sub-portal may be accessible by more than one instance of a qman
+ * driver and so it may be claimed multiple times. If this is the case, it is
+ * up to the system architect to prevent conflicting configuration actions
+ * coming from the different driver domains. The qman drivers do not have any
+ * behind-the-scenes coordination to prevent this from happening.
+ */
+int qman_ceetm_sp_claim(struct qm_ceetm_sp **sp,
+			enum qm_dc_portal dcp_idx,
+			unsigned int sp_idx);
+
+/**
+ * qman_ceetm_sp_release - Releases a previously claimed sub-portal.
+ * @sp: the sub-portal to be released.
+ *
+ * Returns 0 for success, or -EBUSY for failure if the dependencies are not
+ * released.
+ */
+int qman_ceetm_sp_release(struct qm_ceetm_sp *sp);
+
+	/* ----------------------------------- */
+	/* CEETM :: logical network interfaces */
+	/* ----------------------------------- */
+
+/**
+ * qman_ceetm_lni_claim - Claims an unclaimed LNI.
+ * @lni: the returned LNI object, if successful.
+ * @dcp_id: specifies the desired Fman block (and thus the relevant CEETM
+ * instance)
+ * @lni_idx: is the desired LNI index.
+ *
+ * Returns zero for success, or -EINVAL on failure, which will happen if the LNI
+ * is not available or has already been claimed (and not yet successfully
+ * released), or lni_dix is out of range.
+ *
+ * Note that there may be multiple driver domains (or instances) that need to
+ * transmit out the same LNI, so this claim is only guaranteeing exclusivity
+ * within the domain of the driver being called. See qman_ceetm_sp_claim() and
+ * qman_ceetm_sp_get_lni() for more information.
+ */
+int qman_ceetm_lni_claim(struct qm_ceetm_lni **lni,
+			 enum qm_dc_portal dcp_id,
+			 unsigned int lni_idx);
+
+/**
+ * qman_ceetm_lni_releaes - Releases a previously claimed LNI.
+ * @lni: the lni needs to be released.
+ *
+ * This will only succeed if all dependent objects have been released.
+ * Returns zero for success, or -EBUSY if the dependencies are not released.
+ */
+int qman_ceetm_lni_release(struct qm_ceetm_lni *lni);
+
+/**
+ * qman_ceetm_sp_set_lni
+ * qman_ceetm_sp_get_lni - Set/get the LNI that the sub-portal is currently
+ * mapped to.
+ * @sp: the given sub-portal.
+ * @lni(in "set"function): the LNI object which the sp will be mappaed to.
+ * @lni_idx(in "get" function): the LNI index which the sp is mapped to.
+ *
+ * Returns zero for success, or -EINVAL for the "set" function when this sp-lni
+ * mapping has been set, or configure mapping command returns error, and
+ * -EINVAL for "get" function when this sp-lni mapping is not set or the query
+ * mapping command returns error.
+ *
+ * This may be useful in situations where multiple driver domains have access
+ * to the same sub-portals in order to all be able to transmit out the same
+ * physical interface (perhaps they're on different IP addresses or VPNs, so
+ * Fman is splitting Rx traffic and here we need to converge Tx traffic). In
+ * that case, a control-plane is likely to use qman_ceetm_lni_claim() followed
+ * by qman_ceetm_sp_set_lni() to configure the sub-portal, and other domains
+ * are likely to use qman_ceetm_sp_get_lni() followed by qman_ceetm_lni_claim()
+ * in order to determine the LNI that the control-plane had assigned. This is
+ * why the "get" returns an index, whereas the "set" takes an (already claimed)
+ * LNI object.
+ */
+int qman_ceetm_sp_set_lni(struct qm_ceetm_sp *sp,
+			  struct qm_ceetm_lni *lni);
+int qman_ceetm_sp_get_lni(struct qm_ceetm_sp *sp,
+			  unsigned int *lni_idx);
+
+/**
+ * qman_ceetm_lni_enable_shaper
+ * qman_ceetm_lni_disable_shaper - Enables/disables shaping on the LNI.
+ * @lni: the given LNI.
+ * @coupled: indicates whether CR and ER are coupled.
+ * @oal: the overhead accounting length which is added to the actual length of
+ * each frame when performing shaper calculations.
+ *
+ * When the number of (unused) committed-rate tokens reach the committed-rate
+ * token limit, 'coupled' indicates whether surplus tokens should be added to
+ * the excess-rate token count (up to the excess-rate token limit).
+ * When LNI is claimed, the shaper is disabled by default. The enable function
+ * will turn on this shaper for this lni.
+ * Whenever a claimed LNI is first enabled for shaping, its committed and
+ * excess token rates and limits are zero, so will need to be changed to do
+ * anything useful. The shaper can subsequently be enabled/disabled without
+ * resetting the shaping parameters, but the shaping parameters will be reset
+ * when the LNI is released.
+ *
+ * Returns zero for success, or  errno for "enable" function in the cases as:
+ * a) -EINVAL if the shaper is already enabled,
+ * b) -EIO if the configure shaper command returns error.
+ * For "disable" function, returns:
+ * a) -EINVAL if the shaper is has already disabled.
+ * b) -EIO if calling configure shaper command returns error.
+ */
+int qman_ceetm_lni_enable_shaper(struct qm_ceetm_lni *lni, int coupled,
+								int oal);
+int qman_ceetm_lni_disable_shaper(struct qm_ceetm_lni *lni);
+
+/**
+ * qman_ceetm_lni_is_shaper_enabled - Check LNI shaper status
+ * @lni: the give LNI
+ */
+int qman_ceetm_lni_is_shaper_enabled(struct qm_ceetm_lni *lni);
+
+/**
+ * qman_ceetm_lni_set_commit_rate
+ * qman_ceetm_lni_get_commit_rate
+ * qman_ceetm_lni_set_excess_rate
+ * qman_ceetm_lni_get_excess_rate - Set/get the shaper CR/ER token rate and
+ * token limit for the given LNI.
+ * @lni: the given LNI.
+ * @token_rate: the desired token rate for "set" fuction, or the token rate of
+ * the LNI queried by "get" function.
+ * @token_limit: the desired token bucket limit for "set" function, or the token
+ * limit of the given LNI queried by "get" function.
+ *
+ * Returns zero for success. The "set" function returns -EINVAL if the given
+ * LNI is unshapped or -EIO if the configure shaper command returns error.
+ * The "get" function returns -EINVAL if the token rate or the token limit is
+ * not set or the query command returns error.
+ */
+int qman_ceetm_lni_set_commit_rate(struct qm_ceetm_lni *lni,
+				   const struct qm_ceetm_rate *token_rate,
+				   u16 token_limit);
+int qman_ceetm_lni_get_commit_rate(struct qm_ceetm_lni *lni,
+				   struct qm_ceetm_rate *token_rate,
+				   u16 *token_limit);
+int qman_ceetm_lni_set_excess_rate(struct qm_ceetm_lni *lni,
+				   const struct qm_ceetm_rate *token_rate,
+				   u16 token_limit);
+int qman_ceetm_lni_get_excess_rate(struct qm_ceetm_lni *lni,
+				   struct qm_ceetm_rate *token_rate,
+				   u16 *token_limit);
+/**
+ * qman_ceetm_lni_set_commit_rate_bps
+ * qman_ceetm_lni_get_commit_rate_bps
+ * qman_ceetm_lni_set_excess_rate_bps
+ * qman_ceetm_lni_get_excess_rate_bps - Set/get the shaper CR/ER rate
+ * and token limit for the given LNI.
+ * @lni: the given LNI.
+ * @bps: the desired shaping rate in bps for "set" fuction, or the shaping rate
+ * of the LNI queried by "get" function.
+ * @token_limit: the desired token bucket limit for "set" function, or the token
+ * limit of the given LNI queried by "get" function.
+ *
+ * Returns zero for success. The "set" function returns -EINVAL if the given
+ * LNI is unshapped or -EIO if the configure shaper command returns error.
+ * The "get" function returns -EINVAL if the token rate or the token limit is
+ * not set or the query command returns error.
+ */
+int qman_ceetm_lni_set_commit_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 bps,
+				       u16 token_limit);
+int qman_ceetm_lni_get_commit_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 *bps, u16 *token_limit);
+int qman_ceetm_lni_set_excess_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 bps,
+				       u16 token_limit);
+int qman_ceetm_lni_get_excess_rate_bps(struct qm_ceetm_lni *lni,
+				       u64 *bps, u16 *token_limit);
+
+/**
+ * qman_ceetm_lni_set_tcfcc
+ * qman_ceetm_lni_get_tcfcc - Configure/query "Traffic Class Flow Control".
+ * @lni: the given LNI.
+ * @cq_level: is between 0 and 15, representing individual class queue levels
+ * (CQ0 to CQ7 for every channel) and grouped class queue levels (CQ8 to CQ15
+ * for every channel).
+ * @traffic_class: is between 0 and 7 when associating a given class queue level
+ * to a traffic class, or -1 when disabling traffic class flow control for this
+ * class queue level.
+ *
+ * Return zero for success, or -EINVAL if the cq_level or traffic_class is out
+ * of range as indicated above, or -EIO if the configure/query tcfcc command
+ * returns error.
+ *
+ * Refer to the section of QMan CEETM traffic class flow control in the
+ * Reference Manual.
+ */
+int qman_ceetm_lni_set_tcfcc(struct qm_ceetm_lni *lni,
+			     unsigned int cq_level,
+			     int traffic_class);
+int qman_ceetm_lni_get_tcfcc(struct qm_ceetm_lni *lni,
+			     unsigned int cq_level,
+			     int *traffic_class);
+
+	/* ----------------------------- */
+	/* CEETM :: class queue channels */
+	/* ----------------------------- */
+
+/**
+ * qman_ceetm_channel_claim - Claims an unclaimed CQ channel that is mapped to
+ * the given LNI.
+ * @channel: the returned class queue channel object, if successful.
+ * @lni: the LNI that the channel belongs to.
+ *
+ * Channels are always initially "unshaped".
+ *
+ * Return zero for success, or -ENODEV if there is no channel available(all 32
+ * channels are claimed) or -EINVAL if the channel mapping command returns
+ * error.
+ */
+int qman_ceetm_channel_claim(struct qm_ceetm_channel **channel,
+			     struct qm_ceetm_lni *lni);
+
+/**
+ * qman_ceetm_channel_release - Releases a previously claimed CQ channel.
+ * @channel: the channel needs to be released.
+ *
+ * Returns zero for success, or -EBUSY if the dependencies are still in use.
+ *
+ * Note any shaping of the channel will be cleared to leave it in an unshaped
+ * state.
+ */
+int qman_ceetm_channel_release(struct qm_ceetm_channel *channel);
+
+/**
+ * qman_ceetm_channel_enable_shaper
+ * qman_ceetm_channel_disable_shaper - Enables/disables shaping on the channel.
+ * @channel: the given channel.
+ * @coupled: indicates whether surplus CR tokens should be added to the
+ * excess-rate token count (up to the excess-rate token limit) when the number
+ * of (unused) committed-rate tokens reach the committed_rate token limit.
+ *
+ * Whenever a claimed channel is first enabled for shaping, its committed and
+ * excess token rates and limits are zero, so will need to be changed to do
+ * anything useful. The shaper can subsequently be enabled/disabled without
+ * resetting the shaping parameters, but the shaping parameters will be reset
+ * when the channel is released.
+ *
+ * Return 0 for success, or -EINVAL for failure, in the case that the channel
+ * shaper has been enabled/disabled or the management command returns error.
+ */
+int qman_ceetm_channel_enable_shaper(struct qm_ceetm_channel *channel,
+							 int coupled);
+int qman_ceetm_channel_disable_shaper(struct qm_ceetm_channel *channel);
+
+/**
+ * qman_ceetm_channel_is_shaper_enabled - Check channel shaper status.
+ * @channel: the give channel.
+ */
+int qman_ceetm_channel_is_shaper_enabled(struct qm_ceetm_channel *channel);
+
+/**
+ * qman_ceetm_channel_set_commit_rate
+ * qman_ceetm_channel_get_commit_rate
+ * qman_ceetm_channel_set_excess_rate
+ * qman_ceetm_channel_get_excess_rate - Set/get channel CR/ER shaper parameters.
+ * @channel: the given channel.
+ * @token_rate: the desired token rate for "set" function, or the queried token
+ * rate for "get" function.
+ * @token_limit: the desired token limit for "set" function, or the queried
+ * token limit for "get" function.
+ *
+ * Return zero for success. The "set" function returns -EINVAL if the channel
+ * is unshaped, or -EIO if the configure shapper command returns error. The
+ * "get" function returns -EINVAL if token rate of token limit is not set, or
+ * the query shaper command returns error.
+ */
+int qman_ceetm_channel_set_commit_rate(struct qm_ceetm_channel *channel,
+				   const struct qm_ceetm_rate *token_rate,
+				   u16 token_limit);
+int qman_ceetm_channel_get_commit_rate(struct qm_ceetm_channel *channel,
+				   struct qm_ceetm_rate *token_rate,
+				   u16 *token_limit);
+int qman_ceetm_channel_set_excess_rate(struct qm_ceetm_channel *channel,
+				   const struct qm_ceetm_rate *token_rate,
+				   u16 token_limit);
+int qman_ceetm_channel_get_excess_rate(struct qm_ceetm_channel *channel,
+				   struct qm_ceetm_rate *token_rate,
+				   u16 *token_limit);
+/**
+ * qman_ceetm_channel_set_commit_rate_bps
+ * qman_ceetm_channel_get_commit_rate_bps
+ * qman_ceetm_channel_set_excess_rate_bps
+ * qman_ceetm_channel_get_excess_rate_bps - Set/get channel CR/ER shaper
+ * parameters.
+ * @channel: the given channel.
+ * @token_rate: the desired shaper rate in bps for "set" function, or the
+ * shaper rate in bps for "get" function.
+ * @token_limit: the desired token limit for "set" function, or the queried
+ * token limit for "get" function.
+ *
+ * Return zero for success. The "set" function returns -EINVAL if the channel
+ * is unshaped, or -EIO if the configure shapper command returns error. The
+ * "get" function returns -EINVAL if token rate of token limit is not set, or
+ * the query shaper command returns error.
+ */
+int qman_ceetm_channel_set_commit_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 bps, u16 token_limit);
+int qman_ceetm_channel_get_commit_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 *bps, u16 *token_limit);
+int qman_ceetm_channel_set_excess_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 bps, u16 token_limit);
+int qman_ceetm_channel_get_excess_rate_bps(struct qm_ceetm_channel *channel,
+					   u64 *bps, u16 *token_limit);
+
+/**
+ * qman_ceetm_channel_set_weight
+ * qman_ceetm_channel_get_weight - Set/get the weight for unshaped channel
+ * @channel: the given channel.
+ * @token_limit: the desired token limit as the weight of the unshaped channel
+ * for "set" function, or the queried token limit for "get" function.
+ *
+ * The algorithm of unshaped fair queuing (uFQ) is used for unshaped channel.
+ * It allows the unshaped channels to be included in the CR time eligible list,
+ * and thus use the configured CR token limit value as their fair queuing
+ * weight.
+ *
+ * Return zero for success, or -EINVAL if the channel is a shaped channel or
+ * the management command returns error.
+ */
+int qman_ceetm_channel_set_weight(struct qm_ceetm_channel *channel,
+				  u16 token_limit);
+int qman_ceetm_channel_get_weight(struct qm_ceetm_channel *channel,
+				  u16 *token_limit);
+
+/**
+ * qman_ceetm_channel_set_group
+ * qman_ceetm_channel_get_group - Set/get the grouping of the class scheduler.
+ * @channel: the given channel.
+ * @group_b: indicates whether there is group B in this channel.
+ * @prio_a: the priority of group A.
+ * @prio_b: the priority of group B.
+ *
+ * There are 8 individual class queues (CQ0-CQ7), and 8 grouped class queues
+ * (CQ8-CQ15). If 'group_b' is zero, then all the grouped class queues are in
+ * group A, otherwise they are split into group A (CQ8-11) and group B
+ * (CQ12-C15). The individual class queues and the group(s) are in strict
+ * priority order relative to each other. Within the group(s), the scheduling
+ * is not strict priority order, but the result of scheduling within a group
+ * is in strict priority order relative to the other class queues in the
+ * channel. 'prio_a' and 'prio_b' control the priority order of the groups
+ * relative to the individual class queues, and take values from 0-7. Eg. if
+ * 'group_b' is non-zero, 'prio_a' is 2 and 'prio_b' is 6, then the strict
+ * priority order would be;
+ *      CQ0, CQ1, CQ2, GROUPA, CQ3, CQ4, CQ5, CQ6, GROUPB, CQ7
+ *
+ * Return 0 for success. For "set" function, returns -EINVAL if prio_a or
+ * prio_b are out of the range 0 - 7 (priority of group A or group B can not
+ * be 0, CQ0 is always the highest class queue in this channel.), or -EIO if
+ * the configure scheduler command returns error. For "get" function, return
+ * -EINVAL if the query scheduler command returns error.
+ */
+int qman_ceetm_channel_set_group(struct qm_ceetm_channel *channel,
+			     int group_b,
+			     unsigned int prio_a,
+			     unsigned int prio_b);
+int qman_ceetm_channel_get_group(struct qm_ceetm_channel *channel,
+			     int *group_b,
+			     unsigned int *prio_a,
+			     unsigned int *prio_b);
+
+/**
+ * qman_ceetm_channel_set_group_cr_eligibility
+ * qman_ceetm_channel_set_group_er_eligibility - Set channel group eligibility
+ * @channel: the given channel object
+ * @group_b: indicates whether there is group B in this channel.
+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable.
+ *
+ * Return zero for success, or -EINVAL if eligibility setting fails.
+*/
+int qman_ceetm_channel_set_group_cr_eligibility(struct qm_ceetm_channel
+				*channel, int group_b, int cre);
+int qman_ceetm_channel_set_group_er_eligibility(struct qm_ceetm_channel
+				*channel, int group_b, int ere);
+
+/**
+ * qman_ceetm_channel_set_cq_cr_eligibility
+ * qman_ceetm_channel_set_cq_er_eligibility - Set channel cq eligibility
+ * @channel: the given channel object
+ * @idx: is from 0 to 7 (representing CQ0 to CQ7).
+ * @cre: the commit rate eligibility, 1 for enable, 0 for disable.
+ *
+ * Return zero for success, or -EINVAL if eligibility setting fails.
+*/
+int qman_ceetm_channel_set_cq_cr_eligibility(struct qm_ceetm_channel *channel,
+					unsigned int idx, int cre);
+int qman_ceetm_channel_set_cq_er_eligibility(struct qm_ceetm_channel *channel,
+					unsigned int idx, int ere);
+
+	/* --------------------- */
+	/* CEETM :: class queues */
+	/* --------------------- */
+
+/**
+ * qman_ceetm_cq_claim - Claims an individual class queue.
+ * @cq: the returned class queue object, if successful.
+ * @channel: the class queue channel.
+ * @idx: is from 0 to 7 (representing CQ0 to CQ7).
+ * @ccg: represents the class congestion group that this class queue should be
+ * subscribed to, or NULL if no congestion group membership is desired.
+ *
+ * Returns zero for success, or -EINVAL if @idx is out of range 0 - 7 or
+ * if this class queue has been claimed, or configure class queue command
+ * returns error, or returns -ENOMEM if allocating CQ memory fails.
+ */
+int qman_ceetm_cq_claim(struct qm_ceetm_cq **cq,
+			struct qm_ceetm_channel *channel,
+			unsigned int idx,
+			struct qm_ceetm_ccg *ccg);
+
+/**
+ * qman_ceetm_cq_claim_A - Claims a class queue group A.
+ * @cq: the returned class queue object, if successful.
+ * @channel: the class queue channel.
+ * @idx: is from 8 to 15 if only group A exits, otherwise, it is from 8 to 11.
+ * @ccg: represents the class congestion group that this class queue should be
+ * subscribed to, or NULL if no congestion group membership is desired.
+ *
+ * Return zero for success, or -EINVAL if @idx is out the range or if
+ * this class queue has been claimed or configure class queue command returns
+ * error, or returns -ENOMEM if allocating CQ memory fails.
+ */
+int qman_ceetm_cq_claim_A(struct qm_ceetm_cq **cq,
+				struct qm_ceetm_channel *channel,
+				unsigned int idx,
+				struct qm_ceetm_ccg *ccg);
+
+/**
+ * qman_ceetm_cq_claim_B - Claims a class queue group B.
+ * @cq: the returned class queue object, if successful.
+ * @channel: the class queue channel.
+ * @idx: is from 0 to 3 (CQ12 to CQ15).
+ * @ccg: represents the class congestion group that this class queue should be
+ * subscribed to, or NULL if no congestion group membership is desired.
+ *
+ * Return zero for success, or -EINVAL if @idx is out the range or if
+ * this class queue has been claimed or configure class queue command returns
+ * error, or returns -ENOMEM if allocating CQ memory fails.
+ */
+int qman_ceetm_cq_claim_B(struct qm_ceetm_cq **cq,
+				struct qm_ceetm_channel *channel,
+				unsigned int idx,
+				struct qm_ceetm_ccg *ccg);
+
+/**
+ * qman_ceetm_cq_release - Releases a previously claimed class queue.
+ * @cq: The class queue to be released.
+ *
+ * Return zero for success, or -EBUSY if the dependent objects (eg. logical
+ * FQIDs) have not been released.
+ */
+int qman_ceetm_cq_release(struct qm_ceetm_cq *cq);
+
+/**
+ * qman_ceetm_set_queue_weight
+ * qman_ceetm_get_queue_weight - Configure/query the weight of a grouped class
+ * queue.
+ * @cq: the given class queue.
+ * @weight_code: the desired weight code to set for the given class queue for
+ * "set" function or the queired weight code for "get" function.
+ *
+ * Grouped class queues have a default weight code of zero, which corresponds to
+ * a scheduler weighting of 1. This function can be used to modify a grouped
+ * class queue to another weight, (Use the helpers qman_ceetm_wbfs2ratio()
+ * and qman_ceetm_ratio2wbfs() to convert between these 'weight_code' values
+ * and the corresponding sharing weight.)
+ *
+ * Returns zero for success, or -EIO if the configure weight command returns
+ * error for "set" function, or -EINVAL if the query command returns
+ * error for "get" function.
+ * See section "CEETM Weighted Scheduling among Grouped Classes" in Reference
+ * Manual for weight and weight code.
+ */
+int qman_ceetm_set_queue_weight(struct qm_ceetm_cq *cq,
+				struct qm_ceetm_weight_code *weight_code);
+int qman_ceetm_get_queue_weight(struct qm_ceetm_cq *cq,
+				struct qm_ceetm_weight_code *weight_code);
+
+/**
+ * qman_ceetm_set_queue_weight_in_ratio
+ * qman_ceetm_get_queue_weight_in_ratio - Configure/query the weight of a
+ * grouped class queue.
+ * @cq: the given class queue.
+ * @ratio: the weight in ratio. It should be the real ratio number multiplied
+ * by 100 to get rid of fraction.
+ *
+ * Returns zero for success, or -EIO if the configure weight command returns
+ * error for "set" function, or -EINVAL if the query command returns
+ * error for "get" function.
+ */
+int qman_ceetm_set_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 ratio);
+int qman_ceetm_get_queue_weight_in_ratio(struct qm_ceetm_cq *cq, u32 *ratio);
+
+/* Weights are encoded using a pseudo-exponential scheme. The weight codes 0,
+ * 32, 64, [...] correspond to weights of 1, 2, 4, [...]. The weights
+ * corresponding to intermediate weight codes are calculated using linear
+ * interpolation on the inverted values. Or put another way, the inverse weights
+ * for each 32nd weight code are 1, 1/2, 1/4, [...], and so the intervals
+ * between these are divided linearly into 32 intermediate values, the inverses
+ * of which form the remaining weight codes.
+ *
+ * The Weighted Bandwidth Fair Scheduling (WBFS) algorithm provides a form of
+ * scheduling within a group of class queues (group A or B). Weights are used to
+ * normalise the class queues to an underlying BFS algorithm where all class
+ * queues are assumed to require "equal bandwidth". So the weights referred to
+ * by the weight codes act as divisors on the size of frames being enqueued. Ie.
+ * one class queue in a group is assigned a weight of 2 whilst the other class
+ * queues in the group keep the default weight of 1, then the WBFS scheduler
+ * will effectively treat all frames enqueued on the weight-2 class queue as
+ * having half the number of bytes they really have. Ie. if all other things are
+ * equal, that class queue would get twice as much bytes-per-second bandwidth as
+ * the others. So weights should be chosen to provide bandwidth ratios between
+ * members of the same class queue group. These weights have no bearing on
+ * behaviour outside that group's WBFS mechanism though.
+ */
+
+/**
+ * qman_ceetm_wbfs2ratio - Given a weight code ('wbfs'), an accurate fractional
+ * representation of the corresponding weight is given (in order to not lose
+ * any precision).
+ * @weight_code: The given weight code in WBFS.
+ * @numerator: the numerator part of the weight computed by the weight code.
+ * @denominator: the denominator part of the weight computed by the weight code
+ *
+ * Returns zero for success or -EINVAL if the given weight code is illegal.
+ */
+int qman_ceetm_wbfs2ratio(struct qm_ceetm_weight_code *weight_code,
+			   u32 *numerator,
+			   u32 *denominator);
+/**
+ * qman_ceetm_ratio2wbfs - Given a weight, find the nearest possible weight code
+ * If the user needs to know how close this is, convert the resulting weight
+ * code back to a weight and compare.
+ * @numerator: numerator part of the given weight.
+ * @denominator: denominator part of the given weight.
+ * @weight_code: the weight code computed from the given weight.
+ *
+ * Returns zero for success, or -ERANGE if "numerator/denominator" is outside
+ * the range of weights.
+ */
+int qman_ceetm_ratio2wbfs(u32 numerator,
+			   u32 denominator,
+			   struct qm_ceetm_weight_code *weight_code,
+			   int rounding);
+
+#define QMAN_CEETM_FLAG_CLEAR_STATISTICS_COUNTER	0x1
+/**
+ * qman_ceetm_cq_get_dequeue_statistics - Get the statistics provided by CEETM
+ * CQ counters.
+ * @cq: the given CQ object.
+ * @flags: indicates whether the statistics counter will be cleared after query.
+ * @frame_count: The number of the frames that have been counted since the
+ * counter was cleared last time.
+ * @byte_count: the number of bytes in all frames that have been counted.
+ *
+ * Return zero for success or -EINVAL if query statistics command returns error.
+ *
+ */
+int qman_ceetm_cq_get_dequeue_statistics(struct qm_ceetm_cq *cq, u32 flags,
+					u64 *frame_count, u64 *byte_count);
+
+/**
+ * qman_ceetm_drain_cq - drain the CQ till it is empty.
+ * @cq: the give CQ object.
+ * Return 0 for success or -EINVAL for unsuccessful command to empty CQ.
+ */
+int qman_ceetm_drain_cq(struct qm_ceetm_cq *cq);
+
+	/* ---------------------- */
+	/* CEETM :: logical FQIDs */
+	/* ---------------------- */
+/**
+ * qman_ceetm_lfq_claim - Claims an unused logical FQID, associates it with
+ * the given class queue.
+ * @lfq: the returned lfq object, if successful.
+ * @cq: the class queue which needs to claim a LFQID.
+ *
+ * Return zero for success, or -ENODEV if no LFQID is available or -ENOMEM if
+ * allocating memory for lfq fails, or -EINVAL if configuring LFQMT fails.
+ */
+int qman_ceetm_lfq_claim(struct qm_ceetm_lfq **lfq,
+				struct qm_ceetm_cq *cq);
+
+/**
+ * qman_ceetm_lfq_release - Releases a previously claimed logical FQID.
+ * @lfq: the lfq to be released.
+ *
+ * Return zero for success.
+ */
+int qman_ceetm_lfq_release(struct qm_ceetm_lfq *lfq);
+
+/**
+ * qman_ceetm_lfq_set_context
+ * qman_ceetm_lfq_get_context - Set/get the context_a/context_b pair to the
+ * "dequeue context table" associated with the logical FQID.
+ * @lfq: the given logical FQ object.
+ * @context_a: contextA of the dequeue context.
+ * @context_b: contextB of the dequeue context.
+ *
+ * Returns zero for success, or -EINVAL if there is error to set/get the
+ * context pair.
+ */
+int qman_ceetm_lfq_set_context(struct qm_ceetm_lfq *lfq,
+				u64 context_a,
+				u32 context_b);
+int qman_ceetm_lfq_get_context(struct qm_ceetm_lfq *lfq,
+				u64 *context_a,
+				u32 *context_b);
+
+/**
+ * qman_ceetm_create_fq - Initialise a FQ object for the LFQ.
+ * @lfq: the given logic fq.
+ * @fq: the fq object created for the given logic fq.
+ *
+ * The FQ object can be used in qman_enqueue() and qman_enqueue_orp() APIs to
+ * target a logical FQID (and the class queue it is associated with).
+ * Note that this FQ object can only be used for enqueues, and
+ * in the case of qman_enqueue_orp() it can not be used as the 'orp' parameter,
+ * only as 'fq'. This FQ object can not (and shouldn't) be destroyed, it is only
+ * valid as long as the underlying 'lfq' remains claimed. It is the user's
+ * responsibility to ensure that the underlying 'lfq' is not released until any
+ * enqueues to this FQ object have completed. The only field the user needs to
+ * fill in is fq->cb.ern, as that enqueue rejection handler is the callback that
+ * could conceivably be called on this FQ object. This API can be called
+ * multiple times to create multiple FQ objects referring to the same logical
+ * FQID, and any enqueue rejections will respect the callback of the object that
+ * issued the enqueue (and will identify the object via the parameter passed to
+ * the callback too). There is no 'flags' parameter to this API as there is for
+ * qman_create_fq() - the created FQ object behaves as though qman_create_fq()
+ * had been called with the single flag QMAN_FQ_FLAG_NO_MODIFY.
+ *
+ * Returns 0 for success.
+ */
+int qman_ceetm_create_fq(struct qm_ceetm_lfq *lfq, struct qman_fq *fq);
+
+	/* -------------------------------- */
+	/* CEETM :: class congestion groups */
+	/* -------------------------------- */
+
+/**
+ * qman_ceetm_ccg_claim - Claims an unused CCG.
+ * @ccg: the returned CCG object, if successful.
+ * @channel: the given class queue channel
+ * @cscn: the callback function of this CCG.
+ * @cb_ctx: the corresponding context to be used used if state change
+ * notifications are later enabled for this CCG.
+ *
+ * The congestion group is local to the given class queue channel, so only
+ * class queues within the channel can be associated with that congestion group.
+ * The association of class queues to congestion groups occurs  when the class
+ * queues are claimed, see qman_ceetm_cq_claim() and related functions.
+ * Congestion groups are in a "zero" state when initially claimed, and they are
+ * returned to that state when released.
+ *
+ * Return zero for success, or -EINVAL if no CCG in the channel is available.
+ */
+int qman_ceetm_ccg_claim(struct qm_ceetm_ccg **ccg,
+			 struct qm_ceetm_channel *channel,
+			 unsigned int idx,
+			 void (*cscn)(struct qm_ceetm_ccg *,
+				       void *cb_ctx,
+				       int congested),
+			 void *cb_ctx);
+
+/**
+ * qman_ceetm_ccg_release - Releases a previously claimed CCG.
+ * @ccg: the given ccg.
+ *
+ * Returns zero for success, or -EBUSY if the given ccg's dependent objects
+ * (class queues that are associated with the CCG) have not been released.
+ */
+int qman_ceetm_ccg_release(struct qm_ceetm_ccg *ccg);
+
+/* This struct is used to specify attributes for a CCG. The 'we_mask' field
+ * controls which CCG attributes are to be updated, and the remainder specify
+ * the values for those attributes. A CCG counts either frames or the bytes
+ * within those frames, but not both ('mode'). A CCG can optionally cause
+ * enqueues to be rejected, due to tail-drop or WRED, or both (they are
+ * independent options, 'td_en' and 'wr_en_g,wr_en_y,wr_en_r'). Tail-drop can be
+ * level-triggered due to a single threshold ('td_thres') or edge-triggered due
+ * to a "congestion state", but not both ('td_mode'). Congestion state has
+ * distinct entry and exit thresholds ('cs_thres_in' and 'cs_thres_out'), and
+ * notifications can be sent to software the CCG goes in to and out of this
+ * congested state ('cscn_en'). */
+struct qm_ceetm_ccg_params {
+	/* Boolean fields together in a single bitfield struct */
+	struct {
+		/* Whether to count bytes or frames. 1==frames */
+		u8 mode:1;
+		/* En/disable tail-drop. 1==enable */
+		u8 td_en:1;
+		/* Tail-drop on congestion-state or threshold. 1=threshold */
+		u8 td_mode:1;
+		/* Generate congestion state change notifications. 1==enable */
+		u8 cscn_en:1;
+		/* Enable WRED rejections (per colour). 1==enable */
+		u8 wr_en_g:1;
+		u8 wr_en_y:1;
+		u8 wr_en_r:1;
+	} __packed;
+	/* Tail-drop threshold. See qm_cgr_thres_[gs]et64(). */
+	struct qm_cgr_cs_thres td_thres;
+	/* Congestion state thresholds, for entry and exit. */
+	struct qm_cgr_cs_thres cs_thres_in;
+	struct qm_cgr_cs_thres cs_thres_out;
+	/* Overhead accounting length. Per-packet "tax", from -128 to +127 */
+	signed char oal;
+	/* Congestion state change notification for DCP portal, virtual CCGID*/
+	/* WRED parameters. */
+	struct qm_cgr_wr_parm wr_parm_g;
+	struct qm_cgr_wr_parm wr_parm_y;
+	struct qm_cgr_wr_parm wr_parm_r;
+};
+/* Bits used in 'we_mask' to qman_ceetm_ccg_set(), controls which attributes of
+ * the CCGR are to be updated. */
+#define QM_CCGR_WE_MODE         0x0001 /* mode (bytes/frames) */
+#define QM_CCGR_WE_CS_THRES_IN  0x0002 /* congestion state entry threshold */
+#define QM_CCGR_WE_TD_EN        0x0004 /* congestion state tail-drop enable */
+#define QM_CCGR_WE_CSCN_TUPD	0x0008 /* CSCN target update */
+#define QM_CCGR_WE_CSCN_EN      0x0010 /* congestion notification enable */
+#define QM_CCGR_WE_WR_EN_R      0x0020 /* WRED enable - red */
+#define QM_CCGR_WE_WR_EN_Y      0x0040 /* WRED enable - yellow */
+#define QM_CCGR_WE_WR_EN_G      0x0080 /* WRED enable - green */
+#define QM_CCGR_WE_WR_PARM_R    0x0100 /* WRED parameters - red */
+#define QM_CCGR_WE_WR_PARM_Y    0x0200 /* WRED parameters - yellow */
+#define QM_CCGR_WE_WR_PARM_G    0x0400 /* WRED parameters - green */
+#define QM_CCGR_WE_OAL          0x0800 /* overhead accounting length */
+#define QM_CCGR_WE_CS_THRES_OUT 0x1000 /* congestion state exit threshold */
+#define QM_CCGR_WE_TD_THRES     0x2000 /* tail-drop threshold */
+#define QM_CCGR_WE_TD_MODE      0x4000 /* tail-drop mode (state/threshold) */
+#define QM_CCGR_WE_CDV		0x8000 /* cdv */
+
+/**
+ * qman_ceetm_ccg_set
+ * qman_ceetm_ccg_get - Configure/query a subset of CCG attributes.
+ * @ccg: the given CCG object.
+ * @we_mask: the write enable mask.
+ * @params: the parameters setting for this ccg
+ *
+ * Return 0 for success, or -EIO if configure ccg command returns error for
+ * "set" function, or -EINVAL if query ccg command returns error for "get"
+ * function.
+ */
+int qman_ceetm_ccg_set(struct qm_ceetm_ccg *ccg,
+			u16 we_mask,
+			const struct qm_ceetm_ccg_params *params);
+int qman_ceetm_ccg_get(struct qm_ceetm_ccg *ccg,
+			struct qm_ceetm_ccg_params *params);
+
+/** qman_ceetm_cscn_swp_set - Add or remove a software portal from the target
+ * mask.
+ * qman_ceetm_cscn_swp_get - Query whether a given software portal index is
+ * in the cscn target mask.
+ * @ccg: the give CCG object.
+ * @swp_idx: the index of the software portal.
+ * @cscn_enabled: 1: Set the swp to be cscn target. 0: remove the swp from
+ * the target mask.
+ * @we_mask: the write enable mask.
+ * @params: the parameters setting for this ccg
+ *
+ * Return 0 for success, or -EINVAL if command in set/get function fails.
+ */
+int qman_ceetm_cscn_swp_set(struct qm_ceetm_ccg *ccg,
+				u16 swp_idx,
+				unsigned int cscn_enabled,
+				u16 we_mask,
+				const struct qm_ceetm_ccg_params *params);
+int qman_ceetm_cscn_swp_get(struct qm_ceetm_ccg *ccg,
+				u16 swp_idx,
+				unsigned int *cscn_enabled);
+
+/** qman_ceetm_cscn_dcp_set - Add or remove a direct connect portal from the\
+ * target mask.
+ * qman_ceetm_cscn_swp_get - Query whether a given direct connect portal index
+ * is in the cscn target mask.
+ * @ccg: the give CCG object.
+ * @dcp_idx: the index of the direct connect portal.
+ * @vcgid: congestion state change notification for dcp portal, virtual CGID.
+ * @cscn_enabled: 1: Set the dcp to be cscn target. 0: remove the dcp from
+ * the target mask.
+ * @we_mask: the write enable mask.
+ * @params: the parameters setting for this ccg
+ *
+ * Return 0 for success, or -EINVAL if command in set/get function fails.
+  */
+int qman_ceetm_cscn_dcp_set(struct qm_ceetm_ccg *ccg,
+				u16 dcp_idx,
+				u8 vcgid,
+				unsigned int cscn_enabled,
+				u16 we_mask,
+				const struct qm_ceetm_ccg_params *params);
+int qman_ceetm_cscn_dcp_get(struct qm_ceetm_ccg *ccg,
+				u16 dcp_idx,
+				u8 *vcgid,
+				unsigned int *cscn_enabled);
+
+/**
+ * qman_ceetm_ccg_get_reject_statistics - Get the statistics provided by
+ * CEETM CCG counters.
+ * @ccg: the given CCG object.
+ * @flags: indicates whether the statistics counter will be cleared after query.
+ * @frame_count: The number of the frames that have been counted since the
+ * counter was cleared last time.
+ * @byte_count: the number of bytes in all frames that have been counted.
+ *
+ * Return zero for success or -EINVAL if query statistics command returns error.
+ *
+ */
+int qman_ceetm_ccg_get_reject_statistics(struct qm_ceetm_ccg *ccg, u32 flags,
+					u64 *frame_count, u64 *byte_count);
+
+/**
+ * qman_ceetm_query_lfqmt - Query the logical frame queue mapping table
+ * @lfqid: Logical Frame Queue ID
+ * @lfqmt_query: Results of the query command
+ *
+ * Returns zero for success or -EIO if the query command returns error.
+ *
+ */
+int qman_ceetm_query_lfqmt(int lfqid,
+			   struct qm_mcr_ceetm_lfqmt_query *lfqmt_query);
+
+/**
+ * qman_ceetm_query_write_statistics - Query (and optionally write) statistics
+ * @cid: Target ID (CQID or CCGRID)
+ * @dcp_idx: CEETM portal ID
+ * @command_type: One of the following:
+ *   0 = Query dequeue statistics. CID carries the CQID to be queried.
+ *   1 = Query and clear dequeue statistics. CID carries the CQID to be queried
+ *   2 = Write dequeue statistics. CID carries the CQID to be written.
+ *   3 = Query reject statistics. CID carries the CCGRID to be queried.
+ *   4 = Query and clear reject statistics. CID carries the CCGRID to be queried
+ *   5 = Write reject statistics. CID carries the CCGRID to be written
+ * @frame_count: Frame count value to be written if this is a write command
+ * @byte_count: Bytes count value to be written if this is a write command
+ *
+ * Returns zero for success or -EIO if the query command returns error.
+ */
+int qman_ceetm_query_write_statistics(u16 cid, enum qm_dc_portal dcp_idx,
+				      u16 command_type, u64 frame_count,
+				      u64 byte_count);
+
+/**
+ * qman_set_wpm - Set waterfall power management
+ *
+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm.
+ *
+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not
+ * accessible.
+ */
+int qman_set_wpm(int wpm_enable);
+
+/**
+ * qman_get_swp - Query the waterfall power management setting
+ *
+ * @wpm_enable: boolean, 1 = enable wpm, 0 = disable wpm.
+ *
+ * Return 0 for success, return -ENODEV if QMan misc_cfg register is not
+ * accessible.
+ */
+int qman_get_wpm(int *wpm_enable);
+
+/* The below qman_p_***() variants might be called in a migration situation
+ * (e.g. cpu hotplug). They are used to continue accessing the portal that
+ * execution was affine to prior to migration.
+ * @qman_portal specifies which portal the APIs will use.
+*/
+const struct qman_portal_config *qman_p_get_portal_config(struct qman_portal
+									 *p);
+int qman_p_irqsource_add(struct qman_portal *p, u32 bits);
+int qman_p_irqsource_remove(struct qman_portal *p, u32 bits);
+int qman_p_poll_dqrr(struct qman_portal *p, unsigned int limit);
+u32 qman_p_poll_slow(struct qman_portal *p);
+void qman_p_poll(struct qman_portal *p);
+void qman_p_stop_dequeues(struct qman_portal *p);
+void qman_p_start_dequeues(struct qman_portal *p);
+void qman_p_static_dequeue_add(struct qman_portal *p, u32 pools);
+void qman_p_static_dequeue_del(struct qman_portal *p, u32 pools);
+u32 qman_p_static_dequeue_get(struct qman_portal *p);
+void qman_p_dca(struct qman_portal *p, struct qm_dqrr_entry *dq,
+						int park_request);
+int qman_p_volatile_dequeue(struct qman_portal *p, struct qman_fq *fq,
+				u32 flags __maybe_unused, u32 vdqcr);
+int qman_p_enqueue(struct qman_portal *p, struct qman_fq *fq,
+					const struct qm_fd *fd, u32 flags);
+int qman_p_enqueue_orp(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_fd *fd, u32 flags,
+				struct qman_fq *orp, u16 orp_seqnum);
+int qman_p_enqueue_precommit(struct qman_portal *p, struct qman_fq *fq,
+				const struct qm_fd *fd, u32 flags,
+				qman_cb_precommit cb, void *cb_arg);
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FSL_QMAN_H */
--- /dev/null
+++ b/include/linux/fsl_usdpaa.h
@@ -0,0 +1,372 @@
+/* Copyright 2011-2012 Freescale Semiconductor, Inc.
+ *
+ * This file is licensed under the terms of the GNU General Public License
+ * version 2.  This program is licensed "as is" without any warranty of any
+ * kind, whether express or implied.
+ */
+
+#ifndef FSL_USDPAA_H
+#define FSL_USDPAA_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <linux/uaccess.h>
+#include <linux/ioctl.h>
+#include <linux/fsl_qman.h> /* For "enum qm_channel" */
+#include <linux/compat.h>
+
+#ifdef CONFIG_FSL_USDPAA
+
+/******************************/
+/* Allocation of resource IDs */
+/******************************/
+
+/* This enum is used to distinguish between the type of underlying object being
+ * manipulated. */
+enum usdpaa_id_type {
+	usdpaa_id_fqid,
+	usdpaa_id_bpid,
+	usdpaa_id_qpool,
+	usdpaa_id_cgrid,
+	usdpaa_id_ceetm0_lfqid,
+	usdpaa_id_ceetm0_channelid,
+	usdpaa_id_ceetm1_lfqid,
+	usdpaa_id_ceetm1_channelid,
+	usdpaa_id_max /* <-- not a valid type, represents the number of types */
+};
+#define USDPAA_IOCTL_MAGIC 'u'
+struct usdpaa_ioctl_id_alloc {
+	uint32_t base; /* Return value, the start of the allocated range */
+	enum usdpaa_id_type id_type; /* what kind of resource(s) to allocate */
+	uint32_t num; /* how many IDs to allocate (and return value) */
+	uint32_t align; /* must be a power of 2, 0 is treated like 1 */
+	int partial; /* whether to allow less than 'num' */
+};
+struct usdpaa_ioctl_id_release {
+	/* Input; */
+	enum usdpaa_id_type id_type;
+	uint32_t base;
+	uint32_t num;
+};
+struct usdpaa_ioctl_id_reserve {
+	enum usdpaa_id_type id_type;
+	uint32_t base;
+	uint32_t num;
+};
+
+
+/* ioctl() commands */
+#define USDPAA_IOCTL_ID_ALLOC \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x01, struct usdpaa_ioctl_id_alloc)
+#define USDPAA_IOCTL_ID_RELEASE \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x02, struct usdpaa_ioctl_id_release)
+#define USDPAA_IOCTL_ID_RESERVE \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x0A, struct usdpaa_ioctl_id_reserve)
+
+/**********************/
+/* Mapping DMA memory */
+/**********************/
+
+/* Maximum length for a map name, including NULL-terminator */
+#define USDPAA_DMA_NAME_MAX 16
+/* Flags for requesting DMA maps. Maps are private+unnamed or sharable+named.
+ * For a sharable and named map, specify _SHARED (whether creating one or
+ * binding to an existing one). If _SHARED is specified and _CREATE is not, then
+ * the mapping must already exist. If _SHARED and _CREATE are specified and the
+ * mapping doesn't already exist, it will be created. If _SHARED and _CREATE are
+ * specified and the mapping already exists, the mapping will fail unless _LAZY
+ * is specified. When mapping to a pre-existing sharable map, the length must be
+ * an exact match. Lengths must be a power-of-4 multiple of page size.
+ *
+ * Note that this does not actually map the memory to user-space, that is done
+ * by a subsequent mmap() using the page offset returned from this ioctl(). The
+ * ioctl() is what gives the process permission to do this, and a page-offset
+ * with which to do so.
+ */
+#define USDPAA_DMA_FLAG_SHARE    0x01
+#define USDPAA_DMA_FLAG_CREATE   0x02
+#define USDPAA_DMA_FLAG_LAZY     0x04
+#define USDPAA_DMA_FLAG_RDONLY   0x08
+struct usdpaa_ioctl_dma_map {
+	/* Output parameters - virtual and physical addresses */
+	void *ptr;
+	uint64_t phys_addr;
+	/* Input parameter, the length of the region to be created (or if
+	 * mapping an existing region, this must match it). Must be a power-of-4
+	 * multiple of page size. */
+	uint64_t len;
+	/* Input parameter, the USDPAA_DMA_FLAG_* settings. */
+	uint32_t flags;
+	/* If _FLAG_SHARE is specified, the name of the region to be created (or
+	 * of the existing mapping to use). */
+	char name[USDPAA_DMA_NAME_MAX];
+	/* If this ioctl() creates the mapping, this is an input parameter
+	 * stating whether the region supports locking. If mapping an existing
+	 * region, this is a return value indicating the same thing. */
+	int has_locking;
+	/* In the case of a successful map with _CREATE and _LAZY, this return
+	 * value indicates whether we created the mapped region or whether it
+	 * already existed. */
+	int did_create;
+};
+
+#ifdef CONFIG_COMPAT
+struct usdpaa_ioctl_dma_map_compat {
+	/* Output parameters - virtual and physical addresses */
+	compat_uptr_t ptr;
+	uint64_t phys_addr;
+	/* Input parameter, the length of the region to be created (or if
+	 * mapping an existing region, this must match it). Must be a power-of-4
+	 * multiple of page size. */
+	uint64_t len;
+	/* Input parameter, the USDPAA_DMA_FLAG_* settings. */
+	uint32_t flags;
+	/* If _FLAG_SHARE is specified, the name of the region to be created (or
+	 * of the existing mapping to use). */
+	char name[USDPAA_DMA_NAME_MAX];
+	/* If this ioctl() creates the mapping, this is an input parameter
+	 * stating whether the region supports locking. If mapping an existing
+	 * region, this is a return value indicating the same thing. */
+	int has_locking;
+	/* In the case of a successful map with _CREATE and _LAZY, this return
+	 * value indicates whether we created the mapped region or whether it
+	 * already existed. */
+	int did_create;
+};
+
+#define USDPAA_IOCTL_DMA_MAP_COMPAT \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map_compat)
+#endif
+
+
+#define USDPAA_IOCTL_DMA_MAP \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x03, struct usdpaa_ioctl_dma_map)
+/* munmap() does not remove the DMA map, just the user-space mapping to it.
+ * This ioctl will do both (though you can munmap() before calling the ioctl
+ * too). */
+#define USDPAA_IOCTL_DMA_UNMAP \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x04, unsigned char)
+/* We implement a cross-process locking scheme per DMA map. Call this ioctl()
+ * with a mmap()'d address, and the process will (interruptible) sleep if the
+ * lock is already held by another process. Process destruction will
+ * automatically clean up any held locks. */
+#define USDPAA_IOCTL_DMA_LOCK \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x05, unsigned char)
+#define USDPAA_IOCTL_DMA_UNLOCK \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x06, unsigned char)
+
+/***************************************/
+/* Mapping and using QMan/BMan portals */
+/***************************************/
+enum usdpaa_portal_type {
+	 usdpaa_portal_qman,
+	 usdpaa_portal_bman,
+};
+
+#define QBMAN_ANY_PORTAL_IDX 0xffffffff
+
+struct usdpaa_ioctl_portal_map {
+	/* Input parameter, is a qman or bman portal required. */
+
+	enum usdpaa_portal_type type;
+	/* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
+	   for don't care.  The portal index will be populated by the
+	   driver when the ioctl() successfully completes */
+	uint32_t index;
+
+	/* Return value if the map succeeds, this gives the mapped
+	 * cache-inhibited (cinh) and cache-enabled (cena) addresses. */
+	struct usdpaa_portal_map {
+		void *cinh;
+		void *cena;
+	} addr;
+	/* Qman-specific return values */
+	uint16_t channel;
+	uint32_t pools;
+};
+
+#ifdef CONFIG_COMPAT
+struct compat_usdpaa_ioctl_portal_map {
+	/* Input parameter, is a qman or bman portal required. */
+	enum usdpaa_portal_type type;
+	/* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
+	   for don't care.  The portal index will be populated by the
+	   driver when the ioctl() successfully completes */
+	uint32_t index;
+	/* Return value if the map succeeds, this gives the mapped
+	 * cache-inhibited (cinh) and cache-enabled (cena) addresses. */
+	struct usdpaa_portal_map_compat {
+		compat_uptr_t cinh;
+		compat_uptr_t cena;
+	} addr;
+	/* Qman-specific return values */
+	uint16_t channel;
+	uint32_t pools;
+};
+#define USDPAA_IOCTL_PORTAL_MAP_COMPAT \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct compat_usdpaa_ioctl_portal_map)
+#define USDPAA_IOCTL_PORTAL_UNMAP_COMPAT \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map_compat)
+#endif
+
+#define USDPAA_IOCTL_PORTAL_MAP \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x07, struct usdpaa_ioctl_portal_map)
+#define USDPAA_IOCTL_PORTAL_UNMAP \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x08, struct usdpaa_portal_map)
+
+struct usdpaa_ioctl_irq_map {
+	enum usdpaa_portal_type type; /* Type of portal to map */
+	int fd; /* File descriptor that contains the portal */
+	void *portal_cinh; /* Cache inhibited area to identify the portal */
+};
+
+#define USDPAA_IOCTL_PORTAL_IRQ_MAP \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x09, struct usdpaa_ioctl_irq_map)
+
+#ifdef CONFIG_COMPAT
+
+struct compat_ioctl_irq_map {
+	enum usdpaa_portal_type type; /* Type of portal to map */
+	compat_int_t fd; /* File descriptor that contains the portal */
+	compat_uptr_t portal_cinh; /* Used identify the portal */};
+
+#define USDPAA_IOCTL_PORTAL_IRQ_MAP_COMPAT \
+	_IOW(USDPAA_IOCTL_MAGIC, 0x09, struct compat_ioctl_irq_map)
+#endif
+
+/* ioctl to query the amount of DMA memory used in the system */
+struct usdpaa_ioctl_dma_used {
+	uint64_t free_bytes;
+	uint64_t total_bytes;
+};
+#define USDPAA_IOCTL_DMA_USED \
+	_IOR(USDPAA_IOCTL_MAGIC, 0x0B, struct usdpaa_ioctl_dma_used)
+
+/* ioctl to allocate a raw portal */
+struct usdpaa_ioctl_raw_portal {
+	/* inputs */
+	enum usdpaa_portal_type type; /* Type of portal to allocate */
+
+	 /* set to non zero to turn on stashing */
+	uint8_t enable_stash;
+	/* Stashing attributes for the portal */
+	uint32_t cpu;
+	uint32_t cache;
+	uint32_t window;
+
+	/* Specifies the stash request queue this portal should use */
+	uint8_t sdest;
+
+	/* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
+	 * for don't care.  The portal index will be populated by the
+	 * driver when the ioctl() successfully completes */
+	uint32_t index;
+
+	/* outputs */
+	uint64_t cinh;
+	uint64_t cena;
+};
+
+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct usdpaa_ioctl_raw_portal)
+
+#define USDPAA_IOCTL_FREE_RAW_PORTAL \
+	_IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct usdpaa_ioctl_raw_portal)
+
+#ifdef CONFIG_COMPAT
+
+struct compat_ioctl_raw_portal {
+	/* inputs */
+	enum usdpaa_portal_type type; /* Type of portal to allocate */
+
+	 /* set to non zero to turn on stashing */
+	uint8_t enable_stash;
+	/* Stashing attributes for the portal */
+	uint32_t cpu;
+	uint32_t cache;
+	uint32_t window;
+	/* Specifies the stash request queue this portal should use */
+	uint8_t sdest;
+
+	/* Specifes a specific portal index to map or QBMAN_ANY_PORTAL_IDX
+	 * for don't care.  The portal index will be populated by the
+	 * driver when the ioctl() successfully completes */
+	uint32_t index;
+
+	/* outputs */
+	uint64_t cinh;
+	uint64_t cena;
+};
+
+#define USDPAA_IOCTL_ALLOC_RAW_PORTAL_COMPAT \
+	_IOWR(USDPAA_IOCTL_MAGIC, 0x0C, struct compat_ioctl_raw_portal)
+
+#define USDPAA_IOCTL_FREE_RAW_PORTAL_COMPAT \
+	_IOR(USDPAA_IOCTL_MAGIC, 0x0D, struct compat_ioctl_raw_portal)
+
+#endif
+
+#ifdef __KERNEL__
+
+/* Early-boot hook */
+int __init fsl_usdpaa_init_early(void);
+
+/* Fault-handling in arch/powerpc/mm/mem.c gives USDPAA an opportunity to detect
+ * faults within its ranges via this hook. */
+int usdpaa_test_fault(unsigned long pfn, u64 *phys_addr, u64 *size);
+
+#endif /* __KERNEL__ */
+
+#endif /* CONFIG_FSL_USDPAA */
+
+#ifdef __KERNEL__
+/* This interface is needed in a few places and though it's not specific to
+ * USDPAA as such, creating a new header for it doesn't make any sense. The
+ * qbman kernel driver implements this interface and uses it as the backend for
+ * both the FQID and BPID allocators. The fsl_usdpaa driver also uses this
+ * interface for tracking per-process allocations handed out to user-space. */
+struct dpa_alloc {
+	struct list_head free;
+	spinlock_t lock;
+	struct list_head used;
+};
+#define DECLARE_DPA_ALLOC(name) \
+	struct dpa_alloc name = { \
+		.free = { \
+			.prev = &name.free, \
+			.next = &name.free \
+		}, \
+		.lock = __SPIN_LOCK_UNLOCKED(name.lock), \
+		.used = { \
+			 .prev = &name.used, \
+			 .next = &name.used \
+		 } \
+	}
+static inline void dpa_alloc_init(struct dpa_alloc *alloc)
+{
+	INIT_LIST_HEAD(&alloc->free);
+	INIT_LIST_HEAD(&alloc->used);
+	spin_lock_init(&alloc->lock);
+}
+int dpa_alloc_new(struct dpa_alloc *alloc, u32 *result, u32 count, u32 align,
+		  int partial);
+void dpa_alloc_free(struct dpa_alloc *alloc, u32 base_id, u32 count);
+void dpa_alloc_seed(struct dpa_alloc *alloc, u32 fqid, u32 count);
+
+/* Like 'new' but specifies the desired range, returns -ENOMEM if the entire
+ * desired range is not available, or 0 for success. */
+int dpa_alloc_reserve(struct dpa_alloc *alloc, u32 base_id, u32 count);
+/* Pops and returns contiguous ranges from the allocator. Returns -ENOMEM when
+ * 'alloc' is empty. */
+int dpa_alloc_pop(struct dpa_alloc *alloc, u32 *result, u32 *count);
+/* Returns 1 if the specified id is alloced, 0 otherwise */
+int dpa_alloc_check(struct dpa_alloc *list, u32 id);
+#endif /* __KERNEL__ */
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* FSL_USDPAA_H */