[PATCH 15/19] dmaengine: raid5 dma client
From: Dan Williams
Date: Mon Sep 11 2006 - 19:17:37 EST
From: Dan Williams <dan.j.williams@xxxxxxxxx>
Adds a dmaengine client that is the hardware accelerated version of
raid5_do_soft_block_ops. It utilizes the raid5 workqueue implementation to
operate on multiple stripes simultaneously. See the iop-adma.c driver for
an example of a driver that enables hardware accelerated raid5.
Changelog:
* mark operations as _Dma rather than _Done until all outstanding
operations have completed. Once all operations have completed update the
state and return it to the handle list
* add a helper routine to retrieve the last used cookie
* use dma_async_zero_sum_dma_list for checking parity which optionally
allows parity check operations to not dirty the parity block in the cache
(if 'disks' is less than 'MAX_ADMA_XOR_SOURCES')
* remove dependencies on iop13xx
* take into account the fact that dma engines have a staging buffer so we
can perform 1 less block operation compared to software xor
* added __arch_raid5_dma_chan_request __arch_raid5_dma_next_channel and
__arch_raid5_dma_check_channel to make the driver architecture independent
* added channel switching capability for architectures that implement
different operations (i.e. copy & xor) on individual channels
* added initial support for "non-blocking" channel switching
Signed-off-by: Dan Williams <dan.j.williams@xxxxxxxxx>
---
drivers/dma/Kconfig | 9 +
drivers/dma/Makefile | 1
drivers/dma/raid5-dma.c | 730 ++++++++++++++++++++++++++++++++++++++++++++
drivers/md/Kconfig | 11 +
drivers/md/raid5.c | 66 ++++
include/linux/dmaengine.h | 5
include/linux/raid/raid5.h | 24 +
7 files changed, 839 insertions(+), 7 deletions(-)
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 30d021d..fced8c3 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -22,6 +22,15 @@ config NET_DMA
Since this is the main user of the DMA engine, it should be enabled;
say Y here.
+config RAID5_DMA
+ tristate "MD raid5: block operations offload"
+ depends on INTEL_IOP_ADMA && MD_RAID456
+ default y
+ ---help---
+ This enables the use of DMA engines in the MD-RAID5 driver to
+ offload stripe cache operations, freeing CPU cycles.
+ say Y here
+
comment "DMA Devices"
config INTEL_IOATDMA
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index bdcfdbd..4e36d6e 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,3 +1,4 @@
obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
obj-$(CONFIG_NET_DMA) += iovlock.o
+obj-$(CONFIG_RAID5_DMA) += raid5-dma.o
obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
diff --git a/drivers/dma/raid5-dma.c b/drivers/dma/raid5-dma.c
new file mode 100644
index 0000000..04a1790
--- /dev/null
+++ b/drivers/dma/raid5-dma.c
@@ -0,0 +1,730 @@
+/*
+ * Offload raid5 operations to hardware RAID engines
+ * Copyright(c) 2006 Intel Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; either version 2 of the License, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 59
+ * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * The full GNU General Public License is included in this distribution in the
+ * file called COPYING.
+ */
+
+#include <linux/raid/raid5.h>
+#include <linux/dmaengine.h>
+
+static struct dma_client *raid5_dma_client;
+static atomic_t raid5_count;
+extern void release_stripe(struct stripe_head *sh);
+extern void __arch_raid5_dma_chan_request(struct dma_client *client);
+extern struct dma_chan *__arch_raid5_dma_next_channel(struct dma_client *client);
+
+#define MAX_HW_XOR_SRCS 16
+
+#ifndef STRIPE_SIZE
+#define STRIPE_SIZE PAGE_SIZE
+#endif
+
+#ifndef STRIPE_SECTORS
+#define STRIPE_SECTORS (STRIPE_SIZE>>9)
+#endif
+
+#ifndef r5_next_bio
+#define r5_next_bio(bio, sect) ( ( (bio)->bi_sector + ((bio)->bi_size>>9) < sect + STRIPE_SECTORS) ? (bio)->bi_next : NULL)
+#endif
+
+#define DMA_RAID5_DEBUG 0
+#define PRINTK(x...) ((void)(DMA_RAID5_DEBUG && printk(x)))
+
+/*
+ * Copy data between a page in the stripe cache, and one or more bion
+ * The page could align with the middle of the bio, or there could be
+ * several bion, each with several bio_vecs, which cover part of the page
+ * Multiple bion are linked together on bi_next. There may be extras
+ * at the end of this list. We ignore them.
+ */
+static dma_cookie_t dma_raid_copy_data(int frombio, struct bio *bio,
+ dma_addr_t dma, sector_t sector, struct dma_chan *chan,
+ dma_cookie_t cookie)
+{
+ struct bio_vec *bvl;
+ struct page *bio_page;
+ int i;
+ int dma_offset;
+ dma_cookie_t last_cookie = cookie;
+
+ if (bio->bi_sector >= sector)
+ dma_offset = (signed)(bio->bi_sector - sector) * 512;
+ else
+ dma_offset = (signed)(sector - bio->bi_sector) * -512;
+ bio_for_each_segment(bvl, bio, i) {
+ int len = bio_iovec_idx(bio,i)->bv_len;
+ int clen;
+ int b_offset = 0;
+
+ if (dma_offset < 0) {
+ b_offset = -dma_offset;
+ dma_offset += b_offset;
+ len -= b_offset;
+ }
+
+ if (len > 0 && dma_offset + len > STRIPE_SIZE)
+ clen = STRIPE_SIZE - dma_offset;
+ else clen = len;
+
+ if (clen > 0) {
+ b_offset += bio_iovec_idx(bio,i)->bv_offset;
+ bio_page = bio_iovec_idx(bio,i)->bv_page;
+ if (frombio)
+ do {
+ cookie = dma_async_memcpy_pg_to_dma(chan,
+ dma + dma_offset,
+ bio_page,
+ b_offset,
+ clen);
+ if (cookie == -ENOMEM)
+ dma_sync_wait(chan, last_cookie);
+ else
+ WARN_ON(cookie <= 0);
+ } while (cookie == -ENOMEM);
+ else
+ do {
+ cookie = dma_async_memcpy_dma_to_pg(chan,
+ bio_page,
+ b_offset,
+ dma + dma_offset,
+ clen);
+ if (cookie == -ENOMEM)
+ dma_sync_wait(chan, last_cookie);
+ else
+ WARN_ON(cookie <= 0);
+ } while (cookie == -ENOMEM);
+ }
+ last_cookie = cookie;
+ if (clen < len) /* hit end of page */
+ break;
+ dma_offset += len;
+ }
+
+ return last_cookie;
+}
+
+#define issue_xor() do { \
+ do { \
+ cookie = dma_async_xor_dma_list_to_dma( \
+ sh->ops.dma_chan, \
+ xor_destination_addr, \
+ dma, \
+ count, \
+ STRIPE_SIZE); \
+ if (cookie == -ENOMEM) \
+ dma_sync_wait(sh->ops.dma_chan, \
+ sh->ops.dma_cookie); \
+ else \
+ WARN_ON(cookie <= 0); \
+ } while (cookie == -ENOMEM); \
+ sh->ops.dma_cookie = cookie; \
+ dma[0] = xor_destination_addr; \
+ count = 1; \
+ } while(0)
+#define check_xor() do { \
+ if (count == MAX_HW_XOR_SRCS) \
+ issue_xor(); \
+ } while (0)
+
+#ifdef CONFIG_RAID5_DMA_ARCH_NEEDS_CHAN_SWITCH
+extern struct dma_chan *__arch_raid5_dma_check_channel(struct dma_chan *chan,
+ dma_cookie_t cookie,
+ struct dma_client *client,
+ unsigned long capabilities);
+
+#ifdef CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE
+#define check_channel(cap, bookmark) do { \
+bookmark: \
+ next_chan = __arch_raid5_dma_check_channel(sh->ops.dma_chan, \
+ sh->ops.dma_cookie, \
+ raid5_dma_client, \
+ (cap)); \
+ if (!next_chan) { \
+ BUG_ON(sh->ops.ops_bookmark); \
+ sh->ops.ops_bookmark = &&bookmark; \
+ goto raid5_dma_retry; \
+ } else { \
+ sh->ops.dma_chan = next_chan; \
+ sh->ops.dma_cookie = dma_async_get_last_cookie( \
+ next_chan); \
+ sh->ops.ops_bookmark = NULL; \
+ } \
+} while (0)
+#else
+#define check_channel(cap, bookmark) do { \
+bookmark: \
+ next_chan = __arch_raid5_dma_check_channel(sh->ops.dma_chan, \
+ sh->ops.dma_cookie, \
+ raid5_dma_client, \
+ (cap)); \
+ if (!next_chan) { \
+ dma_sync_wait(sh->ops.dma_chan, sh->ops.dma_cookie); \
+ goto bookmark; \
+ } else { \
+ sh->ops.dma_chan = next_chan; \
+ sh->ops.dma_cookie = dma_async_get_last_cookie( \
+ next_chan); \
+ } \
+} while (0)
+#endif /* CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE */
+#else
+#define check_channel(cap, bookmark) do { } while (0)
+#endif /* CONFIG_RAID5_DMA_ARCH_NEEDS_CHAN_SWITCH */
+
+/*
+ * dma_do_raid5_block_ops - perform block memory operations on stripe data
+ * outside the spin lock with dma engines
+ *
+ * A note about the need for __arch_raid5_dma_check_channel:
+ * This function is only needed to support architectures where a single raid
+ * operation spans multiple hardware channels. For example on a reconstruct
+ * write, memory copy operations are submitted to a memcpy channel and then
+ * the routine must switch to the xor channel to complete the raid operation.
+ * __arch_raid5_dma_check_channel makes sure the previous operation has
+ * completed before returning the new channel.
+ * Some efficiency can be gained by putting the stripe back on the work
+ * queue rather than spin waiting. This code is a work in progress and is
+ * available via the 'broken' option CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE.
+ * If 'wait via requeue' is not defined the check_channel macro live waits
+ * for the next channel.
+ */
+static void dma_do_raid5_block_ops(void *stripe_head_ref)
+{
+ struct stripe_head *sh = stripe_head_ref;
+ int i, pd_idx = sh->pd_idx, disks = sh->disks;
+ dma_addr_t dma[MAX_HW_XOR_SRCS];
+ int overlap=0;
+ unsigned long state, ops_state, ops_state_orig;
+ raid5_conf_t *conf = sh->raid_conf;
+ dma_cookie_t cookie;
+ #ifdef CONFIG_RAID5_DMA_ARCH_NEEDS_CHAN_SWITCH
+ struct dma_chan *next_chan;
+ #endif
+
+ if (!sh->ops.dma_chan) {
+ sh->ops.dma_chan = __arch_raid5_dma_next_channel(raid5_dma_client);
+ dma_chan_get(sh->ops.dma_chan);
+ /* retrieve the last used cookie on this channel */
+ sh->ops.dma_cookie = dma_async_get_last_cookie(sh->ops.dma_chan);
+ }
+
+ /* take a snapshot of what needs to be done at this point in time */
+ spin_lock(&sh->lock);
+ state = sh->state;
+ ops_state_orig = ops_state = sh->ops.state;
+ spin_unlock(&sh->lock);
+
+ #ifdef CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE
+ /* pick up where we left off */
+ if (sh->ops.ops_bookmark)
+ goto *sh->ops.ops_bookmark;
+ #endif
+
+ if (test_bit(STRIPE_OP_BIOFILL, &state) &&
+ !test_bit(STRIPE_OP_BIOFILL_Dma, &ops_state)) {
+ struct bio *return_bi;
+ PRINTK("%s: stripe %llu STRIPE_OP_BIOFILL op_state: %lx\n",
+ __FUNCTION__, (unsigned long long)sh->sector,
+ ops_state);
+
+ check_channel(DMA_MEMCPY, stripe_op_biofill);
+ return_bi = NULL;
+
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ if (test_bit(R5_ReadReq, &dev->flags)) {
+ struct bio *rbi, *rbi2;
+ spin_lock_irq(&conf->device_lock);
+ rbi = dev->toread;
+ dev->toread = NULL;
+ spin_unlock_irq(&conf->device_lock);
+ overlap++;
+ while (rbi && rbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+ sh->ops.dma_cookie = dma_raid_copy_data(0,
+ rbi, dev->dma, dev->sector,
+ sh->ops.dma_chan,
+ sh->ops.dma_cookie);
+ rbi2 = r5_next_bio(rbi, dev->sector);
+ spin_lock_irq(&conf->device_lock);
+ if (--rbi->bi_phys_segments == 0) {
+ rbi->bi_next = return_bi;
+ return_bi = rbi;
+ }
+ spin_unlock_irq(&conf->device_lock);
+ rbi = rbi2;
+ }
+ dev->read = return_bi;
+ }
+ }
+ if (overlap)
+ set_bit(STRIPE_OP_BIOFILL_Dma, &ops_state);
+ }
+
+ if (test_bit(STRIPE_OP_COMPUTE, &state) &&
+ !test_bit(STRIPE_OP_COMPUTE_Dma, &ops_state)) {
+
+ /* dma engines do not need to pre-zero the destination */
+ if (test_and_clear_bit(STRIPE_OP_COMPUTE_Prep, &ops_state))
+ set_bit(STRIPE_OP_COMPUTE_Parity, &ops_state);
+
+ if (test_and_clear_bit(STRIPE_OP_COMPUTE_Parity, &ops_state)) {
+ dma_addr_t xor_destination_addr;
+ int dd_idx;
+ int count;
+
+ check_channel(DMA_XOR, stripe_op_compute_parity);
+ dd_idx = -1;
+ count = 0;
+
+ for (i=disks ; i-- ; )
+ if (test_bit(R5_ComputeReq, &sh->dev[i].flags)) {
+ dd_idx = i;
+ PRINTK("%s: stripe %llu STRIPE_OP_COMPUTE "
+ "op_state: %lx block: %d\n",
+ __FUNCTION__,
+ (unsigned long long)sh->sector,
+ ops_state, dd_idx);
+ break;
+ }
+
+ BUG_ON(dd_idx < 0);
+
+ xor_destination_addr = sh->dev[dd_idx].dma;
+
+ for (i=disks ; i-- ; )
+ if (i != dd_idx) {
+ dma[count++] = sh->dev[i].dma;
+ check_xor();
+ }
+
+ if (count > 1)
+ issue_xor();
+
+ set_bit(STRIPE_OP_COMPUTE_Dma, &ops_state);
+ }
+ }
+
+ if (test_bit(STRIPE_OP_RMW, &state) &&
+ !test_bit(STRIPE_OP_RMW_Dma, &ops_state)) {
+ BUG_ON(test_bit(STRIPE_OP_RCW, &state));
+
+ PRINTK("%s: stripe %llu STRIPE_OP_RMW op_state: %lx\n",
+ __FUNCTION__, (unsigned long long)sh->sector,
+ ops_state);
+
+ if (test_and_clear_bit(STRIPE_OP_RMW_ParityPre, &ops_state)) {
+ dma_addr_t xor_destination_addr;
+ int count;
+
+ check_channel(DMA_XOR, stripe_op_rmw_paritypre);
+ count = 0;
+
+ /* existing parity data is used in the xor subtraction */
+ xor_destination_addr = dma[count++] = sh->dev[pd_idx].dma;
+
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ struct bio *chosen;
+
+ /* Only process blocks that are known to be uptodate */
+ if (dev->towrite && test_bit(R5_RMWReq, &dev->flags)) {
+ dma[count++] = dev->dma;
+
+ spin_lock(&sh->lock);
+ chosen = dev->towrite;
+ dev->towrite = NULL;
+ BUG_ON(dev->written);
+ dev->written = chosen;
+ spin_unlock(&sh->lock);
+
+ overlap++;
+
+ check_xor();
+ }
+ }
+ if (count > 1)
+ issue_xor();
+
+ set_bit(STRIPE_OP_RMW_Drain, &ops_state);
+ }
+
+ if (test_and_clear_bit(STRIPE_OP_RMW_Drain, &ops_state)) {
+
+ check_channel(DMA_MEMCPY, stripe_op_rmw_drain);
+
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ struct bio *wbi = dev->written;
+
+ while (wbi && wbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+ sh->ops.dma_cookie = dma_raid_copy_data(1,
+ wbi, dev->dma, dev->sector,
+ sh->ops.dma_chan,
+ sh->ops.dma_cookie);
+ wbi = r5_next_bio(wbi, dev->sector);
+ }
+ }
+ set_bit(STRIPE_OP_RMW_ParityPost, &ops_state);
+ }
+
+ if (test_and_clear_bit(STRIPE_OP_RMW_ParityPost, &ops_state)) {
+ dma_addr_t xor_destination_addr;
+ int count;
+
+ check_channel(DMA_XOR, stripe_op_rmw_paritypost);
+ count = 0;
+
+ xor_destination_addr = dma[count++] = sh->dev[pd_idx].dma;
+
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ if (dev->written) {
+ dma[count++] = dev->dma;
+ check_xor();
+ }
+ }
+ if (count > 1)
+ issue_xor();
+
+ set_bit(STRIPE_OP_RMW_Dma, &ops_state);
+ }
+ }
+
+ if (test_bit(STRIPE_OP_RCW, &state) &&
+ !test_bit(STRIPE_OP_RCW_Dma, &ops_state)) {
+ BUG_ON(test_bit(STRIPE_OP_RMW, &state));
+
+ PRINTK("%s: stripe %llu STRIPE_OP_RCW op_state: %lx\n",
+ __FUNCTION__, (unsigned long long)sh->sector,
+ ops_state);
+
+
+ if (test_and_clear_bit(STRIPE_OP_RCW_Drain, &ops_state)) {
+
+ check_channel(DMA_MEMCPY, stripe_op_rcw_drain);
+
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ struct bio *chosen;
+ struct bio *wbi;
+
+ if (i!=pd_idx && dev->towrite &&
+ test_bit(R5_LOCKED, &dev->flags)) {
+
+ spin_lock(&sh->lock);
+ chosen = dev->towrite;
+ dev->towrite = NULL;
+ BUG_ON(dev->written);
+ wbi = dev->written = chosen;
+ spin_unlock(&sh->lock);
+
+ overlap++;
+
+ while (wbi && wbi->bi_sector < dev->sector + STRIPE_SECTORS) {
+ sh->ops.dma_cookie = dma_raid_copy_data(1,
+ wbi, dev->dma, dev->sector,
+ sh->ops.dma_chan,
+ sh->ops.dma_cookie);
+ wbi = r5_next_bio(wbi, dev->sector);
+ }
+ }
+ }
+ set_bit(STRIPE_OP_RCW_Parity, &ops_state);
+ }
+
+ if (test_and_clear_bit(STRIPE_OP_RCW_Parity, &ops_state)) {
+ dma_addr_t xor_destination_addr;
+ int count;
+
+ check_channel(DMA_XOR, stripe_op_rcw_parity);
+ count = 0;
+
+ xor_destination_addr = sh->dev[pd_idx].dma;
+
+ for (i=disks; i--;)
+ if (i != pd_idx) {
+ dma[count++] = sh->dev[i].dma;
+ check_xor();
+ }
+ if (count > 1)
+ issue_xor();
+
+ set_bit(STRIPE_OP_RCW_Dma, &ops_state);
+ }
+ }
+
+ if (test_bit(STRIPE_OP_CHECK, &state) &&
+ !test_bit(STRIPE_OP_CHECK_Dma, &ops_state)) {
+ PRINTK("%s: stripe %llu STRIPE_OP_CHECK op_state: %lx\n",
+ __FUNCTION__, (unsigned long long)sh->sector,
+ ops_state);
+
+ if (test_and_clear_bit(STRIPE_OP_CHECK_Gen, &ops_state)) {
+
+ check_channel(DMA_XOR | DMA_ZERO_SUM, stripe_op_check_gen);
+
+ if (disks > MAX_HW_XOR_SRCS) {
+ /* we need to do a destructive xor
+ * i.e. the result needs to be temporarily stored in memory
+ */
+ dma_addr_t xor_destination_addr;
+ int count = 0;
+ int skip = -1;
+
+ xor_destination_addr = dma[count++] = sh->dev[pd_idx].dma;
+
+ /* xor all but one block */
+ for (i=disks; i--;)
+ if (i != pd_idx) {
+ if (skip < 0) {
+ skip = i;
+ continue;
+ }
+ dma[count++] = sh->dev[i].dma;
+ check_xor();
+ }
+ if (count > 1)
+ issue_xor();
+
+ /* zero result check the skipped block with
+ * the new parity
+ */
+ count = 2;
+ dma[1] = sh->dev[skip].dma;
+ do {
+ cookie = dma_async_zero_sum_dma_list(
+ sh->ops.dma_chan,
+ dma,
+ count,
+ STRIPE_SIZE,
+ &sh->ops.dma_result);
+ if (cookie == -ENOMEM)
+ dma_sync_wait(sh->ops.dma_chan,
+ sh->ops.dma_cookie);
+ else
+ WARN_ON(cookie <= 0);
+ } while (cookie == -ENOMEM);
+ sh->ops.dma_cookie = cookie;
+ } else {
+ int count = 0;
+ for (i=disks; i--;)
+ dma[count++] = sh->dev[i].dma;
+ do {
+ cookie = dma_async_zero_sum_dma_list(
+ sh->ops.dma_chan,
+ dma,
+ count,
+ STRIPE_SIZE,
+ &sh->ops.dma_result);
+ if (cookie == -ENOMEM)
+ dma_sync_wait(sh->ops.dma_chan,
+ sh->ops.dma_cookie);
+ else
+ WARN_ON(cookie <= 0);
+ } while (cookie == -ENOMEM);
+ sh->ops.dma_cookie = cookie;
+ }
+ set_bit(STRIPE_OP_CHECK_Verify, &ops_state);
+ set_bit(STRIPE_OP_CHECK_Dma, &ops_state);
+ }
+ }
+
+#ifdef CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE
+raid5_dma_retry:
+#endif
+ spin_lock(&sh->lock);
+ /* Update the state of operations:
+ * -clear incoming requests
+ * -preserve output status (i.e. done status / check result / dma)
+ * -preserve requests added since 'ops_state_orig' was set
+ */
+ sh->ops.state ^= (ops_state_orig & ~STRIPE_OP_COMPLETION_MASK);
+ sh->ops.state |= ops_state;
+
+ /* if we cleared an overlap condition wake up threads in make_request */
+ if (overlap)
+ for (i= disks; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ if (test_and_clear_bit(R5_Overlap, &dev->flags))
+ wake_up(&sh->raid_conf->wait_for_overlap);
+ }
+
+ if (dma_async_operation_complete(sh->ops.dma_chan, sh->ops.dma_cookie,
+ NULL, NULL) == DMA_IN_PROGRESS)
+ dma_async_issue_pending(sh->ops.dma_chan);
+ else { /* now that dma operations have quiesced update the stripe state */
+ int written, work;
+ written = 0;
+ work = 0;
+
+ if (test_and_clear_bit(STRIPE_OP_BIOFILL_Dma, &sh->ops.state)) {
+ work++;
+ set_bit(STRIPE_OP_BIOFILL_Done, &sh->ops.state);
+ }
+ if (test_and_clear_bit(STRIPE_OP_COMPUTE_Dma, &sh->ops.state)) {
+ for (i=disks ; i-- ;)
+ if (test_and_clear_bit(R5_ComputeReq,
+ &sh->dev[i].flags)) {
+ set_bit(R5_UPTODATE,
+ &sh->dev[i].flags);
+ break;
+ }
+ work++;
+ set_bit(STRIPE_OP_COMPUTE_Done, &sh->ops.state);
+ }
+ if (test_and_clear_bit(STRIPE_OP_RCW_Dma, &sh->ops.state)) {
+ work++;
+ written++;
+ set_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
+ set_bit(STRIPE_OP_RCW_Done, &sh->ops.state);
+ }
+ if (test_and_clear_bit(STRIPE_OP_RMW_Dma, &sh->ops.state)) {
+ work++;
+ written++;
+ set_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
+ set_bit(STRIPE_OP_RMW_Done, &sh->ops.state);
+ }
+ if (written)
+ for (i=disks ; i-- ;) {
+ struct r5dev *dev = &sh->dev[i];
+ if (dev->written)
+ set_bit(R5_UPTODATE, &dev->flags);
+ }
+ if (test_and_clear_bit(STRIPE_OP_CHECK_Dma, &sh->ops.state)) {
+ if (test_and_clear_bit(STRIPE_OP_CHECK_Verify,
+ &sh->ops.state)) {
+ work++;
+ if (sh->ops.dma_result == 0) {
+ set_bit(STRIPE_OP_CHECK_IsZero,
+ &sh->ops.state);
+
+ /* if the parity is correct and we
+ * performed the check withtout dirtying
+ * the parity block, mark it up to date.
+ */
+ if (disks <= MAX_HW_XOR_SRCS)
+ set_bit(R5_UPTODATE,
+ &sh->dev[sh->pd_idx].flags);
+
+ } else
+ clear_bit(STRIPE_OP_CHECK_IsZero,
+ &sh->ops.state);
+
+ set_bit(STRIPE_OP_CHECK_Done, &sh->ops.state);
+
+ } else
+ BUG();
+ }
+
+ sh->ops.pending -= work;
+ BUG_ON(sh->ops.pending < 0);
+
+ #ifdef CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE
+ /* return to the bookmark to continue the operation */
+ if (sh->ops.ops_bookmark) {
+ overlap = 0;
+ state = sh->state;
+ ops_state_orig = ops_state = sh->ops.state;
+ spin_unlock(&sh->lock);
+ goto *sh->ops.ops_bookmark;
+ }
+ #endif
+
+ /* the stripe is done with the channel */
+ dma_chan_put(sh->ops.dma_chan);
+ sh->ops.dma_chan = NULL;
+ sh->ops.dma_cookie = 0;
+ }
+
+ BUG_ON(sh->ops.pending == 0 && sh->ops.dma_chan);
+ clear_bit(STRIPE_OP_QUEUED, &sh->state);
+ set_bit(STRIPE_HANDLE, &sh->state);
+ queue_raid_work(sh);
+ spin_unlock(&sh->lock);
+
+ release_stripe(sh);
+}
+
+static void raid5_dma_event_callback(struct dma_client *client,
+ struct dma_chan *chan, enum dma_event event)
+{
+ switch (event) {
+ case DMA_RESOURCE_SUSPEND:
+ PRINTK("%s: DMA_RESOURCE_SUSPEND\n", __FUNCTION__);
+ break;
+ case DMA_RESOURCE_RESUME:
+ PRINTK("%s: DMA_RESOURCE_RESUME\n", __FUNCTION__);
+ break;
+ case DMA_RESOURCE_ADDED:
+ PRINTK("%s: DMA_RESOURCE_ADDED\n", __FUNCTION__);
+ break;
+ case DMA_RESOURCE_REMOVED:
+ PRINTK("%s: DMA_RESOURCE_REMOVED\n", __FUNCTION__);
+ break;
+ default:
+ PRINTK("%s: unknown\n", __FUNCTION__);
+ break;
+ }
+
+}
+
+static int __init raid5_dma_init (void)
+{
+ raid5_dma_client = dma_async_client_register(
+ &raid5_dma_event_callback);
+
+ if (raid5_dma_client == NULL)
+ return -ENOMEM;
+
+ __arch_raid5_dma_chan_request(raid5_dma_client);
+
+ printk("raid5-dma: driver initialized\n");
+ return 0;
+
+}
+
+static void __init raid5_dma_exit (void)
+{
+ if (raid5_dma_client)
+ dma_async_client_unregister(raid5_dma_client);
+
+ raid5_dma_client = NULL;
+}
+
+static struct dma_chan *raid5_dma_next_channel(void)
+{
+ return __arch_raid5_dma_next_channel(raid5_dma_client);
+}
+
+void raid5_dma_get_dma(struct raid5_dma *dma)
+{
+ dma->owner = THIS_MODULE;
+ dma->channel_iterate = raid5_dma_next_channel;
+ dma->do_block_ops = dma_do_raid5_block_ops;
+ atomic_inc(&raid5_count);
+}
+
+EXPORT_SYMBOL_GPL(raid5_dma_get_dma);
+
+module_init(raid5_dma_init);
+module_exit(raid5_dma_exit);
+
+MODULE_AUTHOR("Intel Corporation");
+MODULE_DESCRIPTION("RAID5-DMA Offload Driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/md/Kconfig b/drivers/md/Kconfig
index 2a16b3b..dbd3ddc 100644
--- a/drivers/md/Kconfig
+++ b/drivers/md/Kconfig
@@ -183,6 +183,17 @@ config MD_RAID456_WORKQUEUE_MULTITHREAD
If unsure say, Y.
+config MD_RAID5_HW_OFFLOAD
+ depends on MD_RAID456 && RAID5_DMA
+ bool "Execute raid5 xor/copy operations with hardware engines"
+ default y
+ ---help---
+ On platforms with the requisite hardware capabilities MD
+ can offload RAID5 stripe cache operations (i.e. parity
+ maintenance and bio buffer copies)
+
+ If unsure say, Y.
+
config MD_MULTIPATH
tristate "Multipath I/O support"
depends on BLK_DEV_MD
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index ad6883b..4daa335 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -53,6 +53,16 @@ #include "raid6.h"
#include <linux/raid/bitmap.h>
+#ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+#include <linux/dma-mapping.h>
+extern void raid5_dma_get_dma(struct raid5_dma *dma);
+#endif /* CONFIG_MD_RAID5_HW_OFFLOAD */
+
+#ifdef CONFIG_MD_RAID6_HW_OFFLOAD
+#include <linux/dma-mapping.h>
+extern void raid6_dma_get_dma(struct raid6_dma *dma);
+#endif /* CONFIG_MD_RAID6_HW_OFFLOAD */
+
/*
* Stripe cache
*/
@@ -138,7 +148,7 @@ static void __release_stripe(raid5_conf_
}
}
}
-static void release_stripe(struct stripe_head *sh)
+void release_stripe(struct stripe_head *sh)
{
raid5_conf_t *conf = sh->raid_conf;
unsigned long flags;
@@ -193,6 +203,17 @@ static void shrink_buffers(struct stripe
p = sh->dev[i].page;
if (!p)
continue;
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ do {
+ raid5_conf_t *conf = sh->raid_conf;
+ struct dma_chan *chan = conf->dma.channel_iterate();
+ /* assumes that all channels share the same mapping
+ * characteristics
+ */
+ dma_async_unmap_page(chan, sh->dev[i].dma,
+ PAGE_SIZE, DMA_FROM_DEVICE);
+ } while (0);
+ #endif
sh->dev[i].page = NULL;
put_page(p);
}
@@ -209,6 +230,20 @@ static int grow_buffers(struct stripe_he
return 1;
}
sh->dev[i].page = page;
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ do {
+ raid5_conf_t *conf = sh->raid_conf;
+ struct dma_chan *chan = conf->dma.channel_iterate();
+ /* assumes that all channels share the same mapping
+ * characteristics
+ */
+ sh->dev[i].dma = dma_async_map_page(chan,
+ sh->dev[i].page,
+ 0,
+ PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ } while (0);
+ #endif
}
return 0;
}
@@ -576,6 +611,13 @@ #if 0
#else
set_bit(R5_UPTODATE, &sh->dev[i].flags);
#endif
+#ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ /* If the backing block device driver performed a pio
+ * read then the buffer needs to be cleaned
+ */
+ consistent_sync(page_address(sh->dev[i].page), PAGE_SIZE,
+ DMA_TO_DEVICE);
+#endif
if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
rdev = conf->disks[i].rdev;
printk(KERN_INFO "raid5:%s: read error corrected (%lu sectors at %llu on %s)\n",
@@ -666,6 +708,15 @@ static int raid5_end_write_request (stru
rdev_dec_pending(conf->disks[i].rdev, conf->mddev);
clear_bit(R5_LOCKED, &sh->dev[i].flags);
+
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ /* If the backing block device driver performed a pio
+ * write then the buffer needs to be invalidated
+ */
+ consistent_sync(page_address(sh->dev[i].page), PAGE_SIZE,
+ DMA_FROM_DEVICE);
+ #endif
+
set_bit(STRIPE_HANDLE, &sh->state);
__release_stripe(conf, sh);
spin_unlock_irqrestore(&conf->device_lock, flags);
@@ -1311,6 +1362,7 @@ static int stripe_to_pdidx(sector_t stri
return pd_idx;
}
+#ifndef CONFIG_MD_RAID5_HW_OFFLOAD
/*
* raid5_do_soft_block_ops - perform block memory operations on stripe data
* outside the spin lock.
@@ -1600,6 +1652,7 @@ static void raid5_do_soft_block_ops(void
release_stripe(sh);
}
+#endif /* #ifndef CONFIG_MD_RAID5_HW_OFFLOAD*/
/*
* handle_stripe - do things to a stripe.
@@ -3553,12 +3606,12 @@ static int run(mddev_t *mddev)
#endif
#endif
- /* To Do:
- * 1/ Offload to asynchronous copy / xor engines
- * 2/ Automated selection of optimal do_block_ops
- * routine similar to the xor template selection
- */
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ raid5_dma_get_dma(&conf->dma);
+ conf->do_block_ops = conf->dma.do_block_ops;
+ #else
conf->do_block_ops = raid5_do_soft_block_ops;
+ #endif
spin_lock_init(&conf->device_lock);
@@ -4184,6 +4237,7 @@ static void raid5_exit(void)
unregister_md_personality(&raid4_personality);
}
+EXPORT_SYMBOL(release_stripe);
module_init(raid5_init);
module_exit(raid5_exit);
MODULE_LICENSE("GPL");
diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h
index 0a70c9e..7fd5aaf 100644
--- a/include/linux/dmaengine.h
+++ b/include/linux/dmaengine.h
@@ -762,6 +762,11 @@ static inline enum dma_status dma_sync_w
return status;
}
+static inline dma_cookie_t dma_async_get_last_cookie(struct dma_chan *chan)
+{
+ return chan->cookie;
+}
+
/* --- DMA device --- */
int dma_async_device_register(struct dma_device *device);
diff --git a/include/linux/raid/raid5.h b/include/linux/raid/raid5.h
index 31ae55c..f5b021d 100644
--- a/include/linux/raid/raid5.h
+++ b/include/linux/raid/raid5.h
@@ -4,6 +4,9 @@ #define _RAID5_H
#include <linux/raid/md.h>
#include <linux/raid/xor.h>
#include <linux/workqueue.h>
+#ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+#include <linux/dmaengine.h>
+#endif
/*
*
@@ -169,16 +172,28 @@ struct stripe_head {
#ifdef CONFIG_MD_RAID456_WORKQUEUE
struct work_struct work; /* work queue descriptor */
#endif
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ u32 dma_result; /* storage for dma engine zero sum results */
+ dma_cookie_t dma_cookie; /* last issued dma operation */
+ struct dma_chan *dma_chan; /* dma channel for ops offload */
+ #ifdef CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE
+ void *ops_bookmark; /* place holder for requeued stripes */
+ #endif /* CONFIG_RAID5_DMA_WAIT_VIA_REQUEUE */
+ #endif /* CONFIG_MD_RAID5_HW_OFFLOAD */
} ops;
struct r5dev {
struct bio req;
struct bio_vec vec;
struct page *page;
+ #ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ dma_addr_t dma;
+ #endif
struct bio *toread, *read, *towrite, *written;
sector_t sector; /* sector of this page */
unsigned long flags;
} dev[1]; /* allocated with extra space depending of RAID geometry */
};
+
/* Flags */
#define R5_UPTODATE 0 /* page contains current data */
#define R5_LOCKED 1 /* IO has been submitted on "req" */
@@ -190,7 +205,6 @@ #define R5_Wantwrite 5
#define R5_Overlap 7 /* There is a pending overlapping request on this block */
#define R5_ReadError 8 /* seen a read error here recently */
#define R5_ReWrite 9 /* have tried to over-write the readerror */
-
#define R5_Expanded 10 /* This block now has post-expand data */
#define R5_Consistent 11 /* Block is HW DMA-able without a cache flush */
#define R5_ComputeReq 12 /* compute_block in progress treat as uptodate */
@@ -373,6 +387,14 @@ struct raid5_private_data {
int pool_size; /* number of disks in stripeheads in pool */
spinlock_t device_lock;
struct disk_info *disks;
+#ifdef CONFIG_MD_RAID5_HW_OFFLOAD
+ struct raid5_dma {
+ struct module *owner;
+ void (*do_block_ops)(void *stripe_ref);
+ struct dma_chan * (*channel_iterate)(void);
+ } dma;
+#endif
+
};
typedef struct raid5_private_data raid5_conf_t;
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/