[PATCH v2 02/11] md: add infra for nr_pending changes
From: tada keisuke
Date: Thu Apr 18 2024 - 03:21:58 EST
Prepare to smoothly change the type of nr_pending from atomic_t to percpu_ref.
Signed-off-by: Keisuke TADA <keisuke1.tada@xxxxxxxxxx>
Signed-off-by: Toshifumi OHTAKE <toshifumi.ootake@xxxxxxxxxx>
---
drivers/md/md-bitmap.c | 2 +-
drivers/md/md.c | 6 +++---
drivers/md/md.h | 32 +++++++++++++++++++++++++++-
drivers/md/raid1.c | 20 ++++++++---------
drivers/md/raid10.c | 46 ++++++++++++++++++++--------------------
drivers/md/raid5-cache.c | 4 ++--
drivers/md/raid5.c | 20 ++++++++---------
7 files changed, 80 insertions(+), 50 deletions(-)
diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c
index 059afc24c08b..6d49a6e4cd2f 100644
--- a/drivers/md/md-bitmap.c
+++ b/drivers/md/md-bitmap.c
@@ -190,7 +190,7 @@ static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mdde
if (rdev->raid_disk >= 0 &&
!test_bit(Faulty, &rdev->flags)) {
/* this is a usable devices */
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
rcu_read_unlock();
return rdev;
}
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 00bbafcd27bb..d53ca23bacf7 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -575,7 +575,7 @@ static void submit_flushes(struct work_struct *ws)
!test_bit(Faulty, &rdev->flags)) {
struct bio *bi;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
rcu_read_unlock();
bi = bio_alloc_bioset(rdev->bdev, 0,
REQ_OP_WRITE | REQ_PREFLUSH,
@@ -1054,7 +1054,7 @@ void md_super_write(struct mddev *mddev, struct md_rdev *rdev,
| REQ_PREFLUSH | REQ_FUA,
GFP_NOIO, &mddev->sync_set);
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
bio->bi_iter.bi_sector = sector;
__bio_add_page(bio, page, size, 0);
@@ -9269,7 +9269,7 @@ static bool rdev_removeable(struct md_rdev *rdev)
return false;
/* There are still inflight io, don't remove this rdev. */
- if (atomic_read(&rdev->nr_pending))
+ if (nr_pending_is_not_zero(rdev))
return false;
/*
diff --git a/drivers/md/md.h b/drivers/md/md.h
index 029dd0491a36..f57a894b924b 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -211,6 +211,36 @@ enum flag_bits {
Nonrot, /* non-rotational device (SSD) */
};
+static inline void nr_pending_inc(struct md_rdev *rdev)
+{
+ atomic_inc(&rdev->nr_pending);
+}
+
+static inline void nr_pending_dec(struct md_rdev *rdev)
+{
+ atomic_dec(&rdev->nr_pending);
+}
+
+static inline bool nr_pending_is_zero(struct md_rdev *rdev)
+{
+ return atomic_read(&rdev->nr_pending) == 0;
+}
+
+static inline bool nr_pending_is_not_zero(struct md_rdev *rdev)
+{
+ return atomic_read(&rdev->nr_pending) != 0;
+}
+
+static inline unsigned int nr_pending_read(struct md_rdev *rdev)
+{
+ return atomic_read(&rdev->nr_pending);
+}
+
+static inline bool nr_pending_dec_and_test(struct md_rdev *rdev)
+{
+ return atomic_dec_and_test(&rdev->nr_pending);
+}
+
static inline int is_badblock(struct md_rdev *rdev, sector_t s, int sectors,
sector_t *first_bad, int *bad_sectors)
{
@@ -846,7 +876,7 @@ static inline bool is_rdev_broken(struct md_rdev *rdev)
static inline void rdev_dec_pending(struct md_rdev *rdev, struct mddev *mddev)
{
int faulty = test_bit(Faulty, &rdev->flags);
- if (atomic_dec_and_test(&rdev->nr_pending) && faulty) {
+ if (nr_pending_dec_and_test(rdev) && faulty) {
set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
md_wakeup_thread(mddev->thread);
}
diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c
index be8ac24f50b6..962cebbba6cd 100644
--- a/drivers/md/raid1.c
+++ b/drivers/md/raid1.c
@@ -581,7 +581,7 @@ static void update_read_sectors(struct r1conf *conf, int disk,
{
struct raid1_info *info = &conf->mirrors[disk];
- atomic_inc(&info->rdev->nr_pending);
+ nr_pending_inc(info->rdev);
if (info->next_seq_sect != this_sector)
info->seq_start = this_sector;
info->next_seq_sect = this_sector + len;
@@ -784,7 +784,7 @@ static int choose_best_rdev(struct r1conf *conf, struct r1bio *r1_bio)
if (ctl.readable_disks++ == 1)
set_bit(R1BIO_FailFast, &r1_bio->state);
- pending = atomic_read(&rdev->nr_pending);
+ pending = nr_pending_read(rdev);
dist = abs(r1_bio->sector - conf->mirrors[disk].head_position);
/* Don't change to another disk for sequential reads */
@@ -1495,7 +1495,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
write_behind = true;
if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
blocked_rdev = rdev;
break;
}
@@ -1506,7 +1506,7 @@ static void raid1_write_request(struct mddev *mddev, struct bio *bio,
continue;
}
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (test_bit(WriteErrorSeen, &rdev->flags)) {
sector_t first_bad;
int bad_sectors;
@@ -1879,7 +1879,7 @@ static bool raid1_remove_conf(struct r1conf *conf, int disk)
struct md_rdev *rdev = info->rdev;
if (!rdev || test_bit(In_sync, &rdev->flags) ||
- atomic_read(&rdev->nr_pending))
+ nr_pending_is_not_zero(rdev))
return false;
/* Only remove non-faulty devices if recovery is not possible. */
@@ -1987,7 +1987,7 @@ static int raid1_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
struct md_rdev *repl =
conf->mirrors[conf->raid_disks + number].rdev;
freeze_array(conf, 0);
- if (atomic_read(&repl->nr_pending)) {
+ if (nr_pending_is_not_zero(repl)) {
/* It means that some queued IO of retry_list
* hold repl. Thus, we cannot set replacement
* as NULL, avoiding rdev NULL pointer
@@ -2403,7 +2403,7 @@ static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio)
(!test_bit(Faulty, &rdev->flags) &&
rdev->recovery_offset >= sect + s)) &&
rdev_has_badblock(rdev, sect, s) == 0) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (sync_page_io(rdev, sect, s<<9,
conf->tmppage, REQ_OP_READ, false))
success = 1;
@@ -2433,7 +2433,7 @@ static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio)
rdev = conf->mirrors[d].rdev;
if (rdev &&
!test_bit(Faulty, &rdev->flags)) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
r1_sync_page_io(rdev, sect, s,
conf->tmppage, REQ_OP_WRITE);
rdev_dec_pending(rdev, mddev);
@@ -2447,7 +2447,7 @@ static void fix_read_error(struct r1conf *conf, struct r1bio *r1_bio)
rdev = conf->mirrors[d].rdev;
if (rdev &&
!test_bit(Faulty, &rdev->flags)) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (r1_sync_page_io(rdev, sect, s,
conf->tmppage, REQ_OP_READ)) {
atomic_add(s, &rdev->corrected_errors);
@@ -2909,7 +2909,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr,
}
}
if (rdev && bio->bi_end_io) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
bio->bi_iter.bi_sector = sector_nr + rdev->data_offset;
bio_set_dev(bio, rdev->bdev);
if (test_bit(FailFast, &rdev->flags))
diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c
index a4556d2e46bf..2b1238befd8a 100644
--- a/drivers/md/raid10.c
+++ b/drivers/md/raid10.c
@@ -808,7 +808,7 @@ static struct md_rdev *read_balance(struct r10conf *conf,
nonrot = bdev_nonrot(rdev->bdev);
has_nonrot_disk |= nonrot;
- pending = atomic_read(&rdev->nr_pending);
+ pending = nr_pending_read(rdev);
if (min_pending > pending && nonrot) {
min_pending = pending;
best_pending_slot = slot;
@@ -849,7 +849,7 @@ static struct md_rdev *read_balance(struct r10conf *conf,
}
if (slot >= 0) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
r10_bio->read_slot = slot;
} else
rdev = NULL;
@@ -1296,12 +1296,12 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
rdev = conf->mirrors[i].rdev;
rrdev = conf->mirrors[i].replacement;
if (rdev && unlikely(test_bit(Blocked, &rdev->flags))) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
blocked_rdev = rdev;
break;
}
if (rrdev && unlikely(test_bit(Blocked, &rrdev->flags))) {
- atomic_inc(&rrdev->nr_pending);
+ nr_pending_inc(rrdev);
blocked_rdev = rrdev;
break;
}
@@ -1322,7 +1322,7 @@ static void wait_blocked_dev(struct mddev *mddev, struct r10bio *r10_bio)
* Mustn't write here until the bad block
* is acknowledged
*/
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
set_bit(BlockedBadBlocks, &rdev->flags);
blocked_rdev = rdev;
break;
@@ -1467,11 +1467,11 @@ static void raid10_write_request(struct mddev *mddev, struct bio *bio,
}
if (rdev) {
r10_bio->devs[i].bio = bio;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
}
if (rrdev) {
r10_bio->devs[i].repl_bio = bio;
- atomic_inc(&rrdev->nr_pending);
+ nr_pending_inc(rrdev);
}
}
@@ -1731,11 +1731,11 @@ static int raid10_handle_discard(struct mddev *mddev, struct bio *bio)
if (rdev) {
r10_bio->devs[disk].bio = bio;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
}
if (rrdev) {
r10_bio->devs[disk].repl_bio = bio;
- atomic_inc(&rrdev->nr_pending);
+ nr_pending_inc(rrdev);
}
}
@@ -2155,7 +2155,7 @@ static int raid10_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
return 0;
if (test_bit(In_sync, &rdev->flags) ||
- atomic_read(&rdev->nr_pending)) {
+ nr_pending_is_not_zero(rdev)) {
err = -EBUSY;
goto abort;
}
@@ -2394,7 +2394,7 @@ static void sync_request_write(struct mddev *mddev, struct r10bio *r10_bio)
bio_copy_data(tbio, fbio);
- atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+ nr_pending_inc(conf->mirrors[d].rdev);
atomic_inc(&r10_bio->remaining);
md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(tbio));
@@ -2552,12 +2552,12 @@ static void recovery_request_write(struct mddev *mddev, struct r10bio *r10_bio)
*/
d = r10_bio->devs[1].devnum;
if (wbio->bi_end_io) {
- atomic_inc(&conf->mirrors[d].rdev->nr_pending);
+ nr_pending_inc(conf->mirrors[d].rdev);
md_sync_acct(conf->mirrors[d].rdev->bdev, bio_sectors(wbio));
submit_bio_noacct(wbio);
}
if (wbio2) {
- atomic_inc(&conf->mirrors[d].replacement->nr_pending);
+ nr_pending_inc(conf->mirrors[d].replacement);
md_sync_acct(conf->mirrors[d].replacement->bdev,
bio_sectors(wbio2));
submit_bio_noacct(wbio2);
@@ -2633,7 +2633,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
rdev_has_badblock(rdev,
r10_bio->devs[sl].addr + sect,
s) == 0) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
success = sync_page_io(rdev,
r10_bio->devs[sl].addr +
sect,
@@ -2682,7 +2682,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
!test_bit(In_sync, &rdev->flags))
continue;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (r10_sync_page_io(rdev,
r10_bio->devs[sl].addr +
sect,
@@ -2714,7 +2714,7 @@ static void fix_read_error(struct r10conf *conf, struct mddev *mddev, struct r10
!test_bit(In_sync, &rdev->flags))
continue;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
switch (r10_sync_page_io(rdev,
r10_bio->devs[sl].addr +
sect,
@@ -3342,9 +3342,9 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
continue;
}
if (mrdev)
- atomic_inc(&mrdev->nr_pending);
+ nr_pending_inc(mrdev);
if (mreplace)
- atomic_inc(&mreplace->nr_pending);
+ nr_pending_inc(mreplace);
r10_bio = raid10_alloc_init_r10buf(conf);
r10_bio->state = 0;
@@ -3413,7 +3413,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
bio->bi_iter.bi_sector = from_addr +
rdev->data_offset;
bio_set_dev(bio, rdev->bdev);
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
/* and we write to 'i' (if not in_sync) */
for (k=0; k<conf->copies; k++)
@@ -3601,7 +3601,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
continue;
}
}
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
atomic_inc(&r10_bio->remaining);
bio->bi_next = biolist;
biolist = bio;
@@ -3617,7 +3617,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr,
if (rdev == NULL || test_bit(Faulty, &rdev->flags))
continue;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
/* Need to set up for writing to the replacement */
bio = r10_bio->devs[i].repl_bio;
@@ -4906,7 +4906,7 @@ static void reshape_request_write(struct mddev *mddev, struct r10bio *r10_bio)
if (!rdev || test_bit(Faulty, &rdev->flags))
continue;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
md_sync_acct_bio(b, r10_bio->sectors);
atomic_inc(&r10_bio->remaining);
b->bi_next = NULL;
@@ -4986,7 +4986,7 @@ static int handle_reshape_read_error(struct mddev *mddev,
goto failed;
addr = r10b->devs[slot].addr + idx * PAGE_SIZE;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
success = sync_page_io(rdev,
addr,
s << 9,
diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c
index 874874fe4fa1..1253466666e4 100644
--- a/drivers/md/raid5-cache.c
+++ b/drivers/md/raid5-cache.c
@@ -1892,7 +1892,7 @@ r5l_recovery_replay_one_stripe(struct r5conf *conf,
/* in case device is broken */
rdev = conf->disks[disk_index].rdev;
if (rdev) {
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
sync_page_io(rdev, sh->sector, PAGE_SIZE,
sh->dev[disk_index].page, REQ_OP_WRITE,
false);
@@ -1900,7 +1900,7 @@ r5l_recovery_replay_one_stripe(struct r5conf *conf,
}
rrdev = conf->disks[disk_index].replacement;
if (rrdev) {
- atomic_inc(&rrdev->nr_pending);
+ nr_pending_inc(rrdev);
sync_page_io(rrdev, sh->sector, PAGE_SIZE,
sh->dev[disk_index].page, REQ_OP_WRITE,
false);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 3b04d8b526b1..25da83eaa2ef 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -1199,11 +1199,11 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
if (rdev && test_bit(Faulty, &rdev->flags))
rdev = NULL;
if (rdev)
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (rrdev && test_bit(Faulty, &rrdev->flags))
rrdev = NULL;
if (rrdev)
- atomic_inc(&rrdev->nr_pending);
+ nr_pending_inc(rrdev);
/* We have already checked bad blocks for reads. Now
* need to check for writes. We never accept write errors
@@ -1231,7 +1231,7 @@ static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
* will dec nr_pending, we must
* increment it first.
*/
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
md_wait_for_blocked_rdev(rdev, conf->mddev);
} else {
/* Acknowledged bad block - skip the write */
@@ -3628,7 +3628,7 @@ handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
if (rdev && test_bit(In_sync, &rdev->flags) &&
!test_bit(Faulty, &rdev->flags))
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
else
rdev = NULL;
if (rdev) {
@@ -4729,7 +4729,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
set_bit(BlockedBadBlocks,
&rdev->flags);
s->blocked_rdev = rdev;
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
}
}
clear_bit(R5_Insync, &dev->flags);
@@ -4767,7 +4767,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
clear_bit(R5_Insync, &dev->flags);
if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
s->handle_bad_blocks = 1;
- atomic_inc(&rdev2->nr_pending);
+ nr_pending_inc(rdev2);
} else
clear_bit(R5_WriteError, &dev->flags);
}
@@ -4778,7 +4778,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
s->handle_bad_blocks = 1;
- atomic_inc(&rdev2->nr_pending);
+ nr_pending_inc(rdev2);
} else
clear_bit(R5_MadeGood, &dev->flags);
}
@@ -4787,7 +4787,7 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
s->handle_bad_blocks = 1;
- atomic_inc(&rdev2->nr_pending);
+ nr_pending_inc(rdev2);
} else
clear_bit(R5_MadeGoodRepl, &dev->flags);
}
@@ -5478,7 +5478,7 @@ static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
return 0;
}
- atomic_inc(&rdev->nr_pending);
+ nr_pending_inc(rdev);
if (rdev_has_badblock(rdev, sector, bio_sectors(raid_bio))) {
rdev_dec_pending(rdev, mddev);
@@ -8157,7 +8157,7 @@ static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
clear_bit(In_sync, &rdev->flags);
if (test_bit(In_sync, &rdev->flags) ||
- atomic_read(&rdev->nr_pending)) {
+ nr_pending_is_not_zero(rdev)) {
err = -EBUSY;
goto abort;
}
--
2.34.1