[PATCH 6/9] rsxx: Restructured DMA cancel scheme.

From: Philip J. Kelleher
Date: Tue Apr 30 2013 - 15:58:22 EST

From: Philip J Kelleher <pjk1939@xxxxxxxxxxxxxxxxxx>

The previous DMA cancellation mechanism was a little buggy.
It has now been reworked with this patch.

Signed-off-by: Philip J Kelleher <pjk1939@xxxxxxxxxxxxxxxxxx>
-------------------------------------------------------------------------------


diff -uprN -X linux-block-vanilla/Documentation/dontdiff linux-block-vanilla/drivers/block/rsxx/core.c linux-block/drivers/block/rsxx/core.c
--- linux-block-vanilla/drivers/block/rsxx/core.c 2013-04-29 17:00:16.049182861 -0500
+++ linux-block/drivers/block/rsxx/core.c 2013-04-29 17:00:30.829184699 -0500
@@ -552,15 +552,26 @@ static void rsxx_eeh_failure(struct pci_
{
struct rsxx_cardinfo *card = pci_get_drvdata(dev);
int i;
+ int cnt = 0;

dev_err(&dev->dev, "IBM FlashSystem PCI: disabling failed card.\n");

card->eeh_state = 1;
+ card->halt = 1;

- for (i = 0; i < card->n_targets; i++)
- del_timer_sync(&card->ctrl[i].activity_timer);
-
- rsxx_eeh_cancel_dmas(card);
+ for (i = 0; i < card->n_targets; i++) {
+ spin_lock_bh(&card->ctrl[i].queue_lock);
+ cnt = rsxx_cleanup_dma_queue(&card->ctrl[i],
+ &card->ctrl[i].queue);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);
+
+ cnt += rsxx_dma_cancel(&card->ctrl[i]);
+
+ if (cnt)
+ dev_info(CARD_TO_DEV(card),
+ "Freed %d queued DMAs on channel %d\n",
+ cnt, card->ctrl[i].id);
+ }
}

static int rsxx_eeh_fifo_flush_poll(struct rsxx_cardinfo *card)
diff -uprN -X linux-block-vanilla/Documentation/dontdiff linux-block-vanilla/drivers/block/rsxx/dev.c linux-block/drivers/block/rsxx/dev.c
--- linux-block-vanilla/drivers/block/rsxx/dev.c 2013-04-29 16:59:56.577496349 -0500
+++ linux-block/drivers/block/rsxx/dev.c 2013-04-29 17:00:30.837187741 -0500
@@ -155,7 +155,8 @@ static void bio_dma_done_cb(struct rsxx_
atomic_set(&meta->error, 1);

if (atomic_dec_and_test(&meta->pending_dmas)) {
- disk_stats_complete(card, meta->bio, meta->start_time);
+ if (!card->eeh_state)
+ disk_stats_complete(card, meta->bio, meta->start_time);

bio_endio(meta->bio, atomic_read(&meta->error) ? -EIO : 0);
kmem_cache_free(bio_meta_pool, meta);
@@ -196,7 +197,8 @@ static void rsxx_make_request(struct req
atomic_set(&bio_meta->pending_dmas, 0);
bio_meta->start_time = jiffies;

- disk_stats_start(card, bio);
+ if (!unlikely(card->halt))
+ disk_stats_start(card, bio);

dev_dbg(CARD_TO_DEV(card), "BIO[%c]: meta: %p addr8: x%llx size: %d\n",
bio_data_dir(bio) ? 'W' : 'R', bio_meta,
diff -uprN -X linux-block-vanilla/Documentation/dontdiff linux-block-vanilla/drivers/block/rsxx/dma.c linux-block/drivers/block/rsxx/dma.c
--- linux-block-vanilla/drivers/block/rsxx/dma.c 2013-04-29 17:00:13.824252923 -0500
+++ linux-block/drivers/block/rsxx/dma.c 2013-04-29 17:02:24.278188243 -0500
@@ -97,7 +97,6 @@ struct dma_tracker_list {
struct dma_tracker list[0];
};

-
/*----------------- Misc Utility Functions -------------------*/
static unsigned int rsxx_addr8_to_laddr(u64 addr8, struct rsxx_cardinfo *card)
{
@@ -245,6 +244,23 @@ static void rsxx_complete_dma(struct rsx
kmem_cache_free(rsxx_dma_pool, dma);
}

+int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl,
+ struct list_head *q)
+{
+ struct rsxx_dma *dma;
+ struct rsxx_dma *tmp;
+ int cnt = 0;
+
+ list_for_each_entry_safe(dma, tmp, q, list) {
+ list_del(&dma->list);
+
+ rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
+ cnt++;
+ }
+
+ return cnt;
+}
+
static void rsxx_requeue_dma(struct rsxx_dma_ctrl *ctrl,
struct rsxx_dma *dma)
{
@@ -252,10 +268,10 @@ static void rsxx_requeue_dma(struct rsxx
* Requeued DMAs go to the front of the queue so they are issued
* first.
*/
- spin_lock(&ctrl->queue_lock);
+ spin_lock_bh(&ctrl->queue_lock);
ctrl->stats.sw_q_depth++;
list_add(&dma->list, &ctrl->queue);
- spin_unlock(&ctrl->queue_lock);
+ spin_unlock_bh(&ctrl->queue_lock);
}

static void rsxx_handle_dma_error(struct rsxx_dma_ctrl *ctrl,
@@ -330,6 +346,7 @@ static void rsxx_handle_dma_error(struct
static void dma_engine_stalled(unsigned long data)
{
struct rsxx_dma_ctrl *ctrl = (struct rsxx_dma_ctrl *)data;
+ int cnt;

if (atomic_read(&ctrl->stats.hw_q_depth) == 0 ||
unlikely(ctrl->card->eeh_state))
@@ -350,6 +367,18 @@ static void dma_engine_stalled(unsigned
"DMA channel %d has stalled, faulting interface.\n",
ctrl->id);
ctrl->card->dma_fault = 1;
+
+ /* Clean up the DMA queue */
+ spin_lock(&ctrl->queue_lock);
+ cnt = rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
+ spin_unlock(&ctrl->queue_lock);
+
+ cnt += rsxx_dma_cancel(ctrl);
+
+ if (cnt)
+ dev_info(CARD_TO_DEV(ctrl->card),
+ "Freed %d queued DMAs on channel %d\n",
+ cnt, ctrl->id);
}
}

@@ -369,22 +398,22 @@ static void rsxx_issue_dmas(struct work_
return;

while (1) {
- spin_lock(&ctrl->queue_lock);
+ spin_lock_bh(&ctrl->queue_lock);
if (list_empty(&ctrl->queue)) {
- spin_unlock(&ctrl->queue_lock);
+ spin_unlock_bh(&ctrl->queue_lock);
break;
}
- spin_unlock(&ctrl->queue_lock);
+ spin_unlock_bh(&ctrl->queue_lock);

tag = pop_tracker(ctrl->trackers);
if (tag == -1)
break;

- spin_lock(&ctrl->queue_lock);
+ spin_lock_bh(&ctrl->queue_lock);
dma = list_entry(ctrl->queue.next, struct rsxx_dma, list);
list_del(&dma->list);
ctrl->stats.sw_q_depth--;
- spin_unlock(&ctrl->queue_lock);
+ spin_unlock_bh(&ctrl->queue_lock);

/*
* This will catch any DMAs that slipped in right before the
@@ -521,33 +550,10 @@ static void rsxx_dma_done(struct work_st
rsxx_enable_ier(ctrl->card, CR_INTR_DMA(ctrl->id));
spin_unlock_irqrestore(&ctrl->card->irq_lock, flags);

- spin_lock(&ctrl->queue_lock);
+ spin_lock_bh(&ctrl->queue_lock);
if (ctrl->stats.sw_q_depth)
queue_work(ctrl->issue_wq, &ctrl->issue_dma_work);
- spin_unlock(&ctrl->queue_lock);
-}
-
-static int rsxx_cleanup_dma_queue(struct rsxx_cardinfo *card,
- struct list_head *q)
-{
- struct rsxx_dma *dma;
- struct rsxx_dma *tmp;
- int cnt = 0;
-
- list_for_each_entry_safe(dma, tmp, q, list) {
- list_del(&dma->list);
-
- if (dma->dma_addr)
- pci_unmap_page(card->dev, dma->dma_addr,
- get_dma_size(dma),
- (dma->cmd == HW_CMD_BLK_WRITE) ?
- PCI_DMA_TODEVICE :
- PCI_DMA_FROMDEVICE);
- kmem_cache_free(rsxx_dma_pool, dma);
- cnt++;
- }
-
- return cnt;
+ spin_unlock_bh(&ctrl->queue_lock);
}

static int rsxx_queue_discard(struct rsxx_cardinfo *card,
@@ -697,12 +703,15 @@ int rsxx_dma_queue_bio(struct rsxx_cardi
}
}

+ if (unlikely(card->halt))
+ goto bvec_err;
+
for (i = 0; i < card->n_targets; i++) {
if (!list_empty(&dma_list[i])) {
- spin_lock(&card->ctrl[i].queue_lock);
+ spin_lock_bh(&card->ctrl[i].queue_lock);
card->ctrl[i].stats.sw_q_depth += dma_cnt[i];
list_splice_tail(&dma_list[i], &card->ctrl[i].queue);
- spin_unlock(&card->ctrl[i].queue_lock);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);

queue_work(card->ctrl[i].issue_wq,
&card->ctrl[i].issue_dma_work);
@@ -712,8 +721,17 @@ int rsxx_dma_queue_bio(struct rsxx_cardi
return 0;

bvec_err:
- for (i = 0; i < card->n_targets; i++)
- rsxx_cleanup_dma_queue(card, &dma_list[i]);
+ for (i = 0; i < card->n_targets; i++) {
+ /*
+ * The sleep is used so that user land apps that continue to
+ * send DMAs even though we are halted will queue up and we
+ * can cancel them all at once.
+ */
+ msleep(100);
+ spin_lock_bh(&card->ctrl[i].queue_lock);
+ rsxx_cleanup_dma_queue(&card->ctrl[i], &dma_list[i]);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);
+ }

return st;
}
@@ -919,13 +937,30 @@ failed_dma_setup:
return st;
}

+int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl)
+{
+ struct rsxx_dma *dma;
+ int i;
+ int cnt = 0;
+
+ /* Clean up issued DMAs */
+ for (i = 0; i < RSXX_MAX_OUTSTANDING_CMDS; i++) {
+ dma = get_tracker_dma(ctrl->trackers, i);
+ if (dma) {
+ atomic_dec(&ctrl->stats.hw_q_depth);
+ rsxx_complete_dma(ctrl, dma, DMA_CANCELLED);
+ push_tracker(ctrl->trackers, i);
+ cnt++;
+ }
+ }
+
+ return cnt;
+}

void rsxx_dma_destroy(struct rsxx_cardinfo *card)
{
struct rsxx_dma_ctrl *ctrl;
- struct rsxx_dma *dma;
- int i, j;
- int cnt = 0;
+ int i;

for (i = 0; i < card->n_targets; i++) {
ctrl = &card->ctrl[i];
@@ -944,33 +979,11 @@ void rsxx_dma_destroy(struct rsxx_cardin
del_timer_sync(&ctrl->activity_timer);

/* Clean up the DMA queue */
- spin_lock(&ctrl->queue_lock);
- cnt = rsxx_cleanup_dma_queue(card, &ctrl->queue);
- spin_unlock(&ctrl->queue_lock);
+ spin_lock_bh(&ctrl->queue_lock);
+ rsxx_cleanup_dma_queue(ctrl, &ctrl->queue);
+ spin_unlock_bh(&ctrl->queue_lock);

- if (cnt)
- dev_info(CARD_TO_DEV(card),
- "Freed %d queued DMAs on channel %d\n",
- cnt, i);
-
- /* Clean up issued DMAs */
- for (j = 0; j < RSXX_MAX_OUTSTANDING_CMDS; j++) {
- dma = get_tracker_dma(ctrl->trackers, j);
- if (dma) {
- pci_unmap_page(card->dev, dma->dma_addr,
- get_dma_size(dma),
- (dma->cmd == HW_CMD_BLK_WRITE) ?
- PCI_DMA_TODEVICE :
- PCI_DMA_FROMDEVICE);
- kmem_cache_free(rsxx_dma_pool, dma);
- cnt++;
- }
- }
-
- if (cnt)
- dev_info(CARD_TO_DEV(card),
- "Freed %d pending DMAs on channel %d\n",
- cnt, i);
+ rsxx_dma_cancel(ctrl);

vfree(ctrl->trackers);

@@ -1014,7 +1027,7 @@ int rsxx_eeh_save_issued_dmas(struct rsx
cnt++;
}

- spin_lock(&card->ctrl[i].queue_lock);
+ spin_lock_bh(&card->ctrl[i].queue_lock);
list_splice(&issued_dmas[i], &card->ctrl[i].queue);

atomic_sub(cnt, &card->ctrl[i].stats.hw_q_depth);
@@ -1029,7 +1042,7 @@ int rsxx_eeh_save_issued_dmas(struct rsx
PCI_DMA_TODEVICE :
PCI_DMA_FROMDEVICE);
}
- spin_unlock(&card->ctrl[i].queue_lock);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);
}

kfree(issued_dmas);
@@ -1037,30 +1050,13 @@ int rsxx_eeh_save_issued_dmas(struct rsx
return 0;
}

-void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card)
-{
- struct rsxx_dma *dma;
- struct rsxx_dma *tmp;
- int i;
-
- for (i = 0; i < card->n_targets; i++) {
- spin_lock(&card->ctrl[i].queue_lock);
- list_for_each_entry_safe(dma, tmp, &card->ctrl[i].queue, list) {
- list_del(&dma->list);
-
- rsxx_complete_dma(&card->ctrl[i], dma, DMA_CANCELLED);
- }
- spin_unlock(&card->ctrl[i].queue_lock);
- }
-}
-
int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card)
{
struct rsxx_dma *dma;
int i;

for (i = 0; i < card->n_targets; i++) {
- spin_lock(&card->ctrl[i].queue_lock);
+ spin_lock_bh(&card->ctrl[i].queue_lock);
list_for_each_entry(dma, &card->ctrl[i].queue, list) {
dma->dma_addr = pci_map_page(card->dev, dma->page,
dma->pg_off, get_dma_size(dma),
@@ -1068,12 +1064,12 @@ int rsxx_eeh_remap_dmas(struct rsxx_card
PCI_DMA_TODEVICE :
PCI_DMA_FROMDEVICE);
if (!dma->dma_addr) {
- spin_unlock(&card->ctrl[i].queue_lock);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);
kmem_cache_free(rsxx_dma_pool, dma);
return -ENOMEM;
}
}
- spin_unlock(&card->ctrl[i].queue_lock);
+ spin_unlock_bh(&card->ctrl[i].queue_lock);
}

return 0;
diff -uprN -X linux-block-vanilla/Documentation/dontdiff linux-block-vanilla/drivers/block/rsxx/rsxx_priv.h linux-block/drivers/block/rsxx/rsxx_priv.h
--- linux-block-vanilla/drivers/block/rsxx/rsxx_priv.h 2013-04-29 17:00:14.683253231 -0500
+++ linux-block/drivers/block/rsxx/rsxx_priv.h 2013-04-29 17:00:54.347187601 -0500
@@ -39,6 +39,7 @@
#include <linux/vmalloc.h>
#include <linux/timer.h>
#include <linux/ioctl.h>
+#include <linux/delay.h>

#include "rsxx.h"
#include "rsxx_cfg.h"
@@ -375,6 +376,8 @@ typedef void (*rsxx_dma_cb)(struct rsxx_
int rsxx_dma_setup(struct rsxx_cardinfo *card);
void rsxx_dma_destroy(struct rsxx_cardinfo *card);
int rsxx_dma_init(void);
+int rsxx_cleanup_dma_queue(struct rsxx_dma_ctrl *ctrl, struct list_head *q);
+int rsxx_dma_cancel(struct rsxx_dma_ctrl *ctrl);
void rsxx_dma_cleanup(void);
void rsxx_dma_queue_reset(struct rsxx_cardinfo *card);
int rsxx_dma_configure(struct rsxx_cardinfo *card);
@@ -385,7 +388,6 @@ int rsxx_dma_queue_bio(struct rsxx_cardi
void *cb_data);
int rsxx_hw_buffers_init(struct pci_dev *dev, struct rsxx_dma_ctrl *ctrl);
int rsxx_eeh_save_issued_dmas(struct rsxx_cardinfo *card);
-void rsxx_eeh_cancel_dmas(struct rsxx_cardinfo *card);
int rsxx_eeh_remap_dmas(struct rsxx_cardinfo *card);

/***** cregs.c *****/

--
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/