[Devel] [PATCH RH8 12/61] ploop: Introduce defer_pios()
Kirill Tkhai
ktkhai at virtuozzo.com
Fri May 14 18:55:42 MSK 2021
Signed-off-by: Kirill Tkhai <ktkhai at virtuozzo.com>
---
drivers/md/dm-ploop-cmd.c | 10 +++--
drivers/md/dm-ploop-map.c | 83 +++++++++++++++++++++---------------------
drivers/md/dm-ploop-target.c | 2 +
drivers/md/dm-ploop.h | 9 +++--
4 files changed, 53 insertions(+), 51 deletions(-)
diff --git a/drivers/md/dm-ploop-cmd.c b/drivers/md/dm-ploop-cmd.c
index 07e69fc0ed01..449258c40a66 100644
--- a/drivers/md/dm-ploop-cmd.c
+++ b/drivers/md/dm-ploop-cmd.c
@@ -1467,8 +1467,8 @@ static int ploop_push_backup_write(struct ploop *ploop, char *uuid,
unsigned int cluster, unsigned int nr)
{
unsigned int i, nr_bat_entries = ploop->nr_bat_entries;
- struct bio_list bio_list = BIO_EMPTY_LIST;
struct push_backup *pb = ploop->pb;
+ LIST_HEAD(pio_list);
struct pio *h;
bool has_more = false;
@@ -1489,7 +1489,7 @@ static int ploop_push_backup_write(struct ploop *ploop, char *uuid,
cluster + nr - 1);
if (!h)
break;
- unlink_postponed_backup_endio(ploop, &bio_list, h);
+ unlink_postponed_backup_endio(ploop, &pio_list, h);
}
has_more = !RB_EMPTY_ROOT(&pb->rb_root);
@@ -1499,8 +1499,8 @@ static int ploop_push_backup_write(struct ploop *ploop, char *uuid,
pb->deadline_jiffies = S64_MAX;
spin_unlock_irq(&ploop->pb_lock);
- if (!bio_list_empty(&bio_list)) {
- defer_bios(ploop, NULL, &bio_list);
+ if (!list_empty(&pio_list)) {
+ defer_pios(ploop, NULL, &pio_list);
if (has_more)
mod_timer(&pb->deadline_timer, pb->timeout_in_jiffies + 1);
}
@@ -1514,7 +1514,7 @@ static bool ploop_has_pending_activity(struct ploop *ploop)
spin_lock_irq(&ploop->deferred_lock);
has = ploop->deferred_cmd;
- has |= !bio_list_empty(&ploop->deferred_bios);
+ has |= !list_empty(&ploop->deferred_pios);
has |= !bio_list_empty(&ploop->discard_bios);
has |= !bio_list_empty(&ploop->delta_cow_action_list);
spin_unlock_irq(&ploop->deferred_lock);
diff --git a/drivers/md/dm-ploop-map.c b/drivers/md/dm-ploop-map.c
index 39dc0a07f19f..97d237fb085a 100644
--- a/drivers/md/dm-ploop-map.c
+++ b/drivers/md/dm-ploop-map.c
@@ -129,15 +129,15 @@ static int ploop_bio_cluster(struct ploop *ploop, struct bio *bio,
return 0;
}
-void defer_bios(struct ploop *ploop, struct bio *bio, struct bio_list *bl)
+void defer_pios(struct ploop *ploop, struct pio *pio, struct list_head *pio_list)
{
unsigned long flags;
spin_lock_irqsave(&ploop->deferred_lock, flags);
- if (bio)
- bio_list_add(&ploop->deferred_bios, bio);
- if (bl)
- bio_list_merge(&ploop->deferred_bios, bl);
+ if (pio)
+ list_add_tail(&pio->list, &ploop->deferred_pios);
+ if (pio_list)
+ list_splice_tail_init(pio_list, &ploop->deferred_pios);
spin_unlock_irqrestore(&ploop->deferred_lock, flags);
queue_work(ploop->wq, &ploop->worker);
@@ -187,6 +187,7 @@ static void queue_discard_index_wb(struct ploop *ploop, struct bio *bio)
/* This 1)defers looking suitable discard bios and 2)ends the rest of them. */
static int ploop_map_discard(struct ploop *ploop, struct bio *bio)
{
+ struct pio *pio = bio_to_endio_hook(bio);
bool supported = false;
unsigned int cluster;
unsigned long flags;
@@ -203,7 +204,7 @@ static int ploop_map_discard(struct ploop *ploop, struct bio *bio)
}
if (supported) {
- defer_bios(ploop, bio, NULL);
+ defer_pios(ploop, pio, NULL);
} else {
bio->bi_status = BLK_STS_NOTSUPP;
bio_endio(bio);
@@ -331,19 +332,14 @@ static void link_endio_hook(struct ploop *ploop, struct pio *new, struct rb_root
* to deferred_list.
*/
static void unlink_endio_hook(struct ploop *ploop, struct rb_root *root,
- struct pio *h, struct bio_list *bio_list)
+ struct pio *h, struct list_head *pio_list)
{
- struct bio *bio;
- struct pio *pio;
-
BUG_ON(RB_EMPTY_NODE(&h->node));
rb_erase(&h->node, root);
RB_CLEAR_NODE(&h->node);
- while ((pio = pio_list_pop(&h->endio_list)) != NULL) {
- bio = dm_bio_from_per_bio_data(pio, sizeof(struct pio));
- bio_list_add(bio_list, bio);
- }
+
+ list_splice_tail_init(&h->endio_list, pio_list);
}
static void add_cluster_lk(struct ploop *ploop, struct pio *h, u32 cluster)
@@ -356,14 +352,14 @@ static void add_cluster_lk(struct ploop *ploop, struct pio *h, u32 cluster)
}
static void del_cluster_lk(struct ploop *ploop, struct pio *h)
{
- struct bio_list bio_list = BIO_EMPTY_LIST;
+ LIST_HEAD(pio_list);
unsigned long flags;
bool queue = false;
spin_lock_irqsave(&ploop->deferred_lock, flags);
- unlink_endio_hook(ploop, &ploop->exclusive_bios_rbtree, h, &bio_list);
- if (!bio_list_empty(&bio_list)) {
- bio_list_merge(&ploop->deferred_bios, &bio_list);
+ unlink_endio_hook(ploop, &ploop->exclusive_bios_rbtree, h, &pio_list);
+ if (!list_empty(&pio_list)) {
+ list_splice_tail(&pio_list, &ploop->deferred_pios);
queue = true;
}
spin_unlock_irqrestore(&ploop->deferred_lock, flags);
@@ -388,8 +384,8 @@ static void maybe_link_submitting_bio(struct ploop *ploop, struct bio *bio,
}
static void maybe_unlink_completed_bio(struct ploop *ploop, struct bio *bio)
{
- struct bio_list bio_list = BIO_EMPTY_LIST;
struct pio *h = bio_to_endio_hook(bio);
+ LIST_HEAD(pio_list);
unsigned long flags;
bool queue = false;
@@ -397,9 +393,9 @@ static void maybe_unlink_completed_bio(struct ploop *ploop, struct bio *bio)
return;
spin_lock_irqsave(&ploop->deferred_lock, flags);
- unlink_endio_hook(ploop, &ploop->inflight_bios_rbtree, h, &bio_list);
- if (!bio_list_empty(&bio_list)) {
- bio_list_merge(&ploop->deferred_bios, &bio_list);
+ unlink_endio_hook(ploop, &ploop->inflight_bios_rbtree, h, &pio_list);
+ if (!list_empty(&pio_list)) {
+ list_splice_tail(&pio_list, &ploop->deferred_pios);
queue = true;
}
spin_unlock_irqrestore(&ploop->deferred_lock, flags);
@@ -1104,7 +1100,8 @@ static void queue_or_fail(struct ploop *ploop, int err, void *data)
bio->bi_status = errno_to_blk_status(err);
bio_endio(bio);
} else {
- defer_bios(ploop, bio, NULL);
+ struct pio *pio = bio_to_endio_hook(bio);
+ defer_pios(ploop, pio, NULL);
}
}
@@ -1248,6 +1245,7 @@ static bool locate_new_cluster_and_attach_bio(struct ploop *ploop,
unsigned int *dst_cluster,
struct bio *bio)
{
+ struct pio *pio = bio_to_endio_hook(bio);
bool bat_update_prepared = false;
bool attached = false;
unsigned int page_nr;
@@ -1265,7 +1263,7 @@ static bool locate_new_cluster_and_attach_bio(struct ploop *ploop,
if (piwb->page_nr != page_nr || piwb->type != PIWB_TYPE_ALLOC) {
/* Another BAT page wb is in process */
- defer_bios(ploop, bio, NULL);
+ defer_pios(ploop, pio, NULL);
goto out;
}
@@ -1281,7 +1279,7 @@ static bool locate_new_cluster_and_attach_bio(struct ploop *ploop,
* batch? Delay submitting. Good thing, that cluster allocation
* has already made, and it goes in the batch.
*/
- defer_bios(ploop, bio, NULL);
+ defer_pios(ploop, pio, NULL);
}
out:
return attached;
@@ -1418,13 +1416,16 @@ void ploop_submit_index_wb_sync(struct ploop *ploop,
wait_for_completion(&piwb->comp);
}
-static void process_deferred_bios(struct ploop *ploop, struct bio_list *bios,
+static void process_deferred_pios(struct ploop *ploop, struct list_head *pios,
struct ploop_index_wb *piwb)
{
struct bio *bio;
+ struct pio *pio;
- while ((bio = bio_list_pop(bios)))
+ while ((pio = pio_list_pop(pios)) != NULL) {
+ bio = dm_bio_from_per_bio_data(pio, sizeof(*pio));
process_one_deferred_bio(ploop, bio, piwb);
+ }
}
static int process_one_discard_bio(struct ploop *ploop, struct bio *bio,
@@ -1518,28 +1519,26 @@ static void process_discard_bios(struct ploop *ploop, struct bio_list *bios,
/* Remove from tree bio and endio bio chain */
void unlink_postponed_backup_endio(struct ploop *ploop,
- struct bio_list *bio_list, struct pio *h)
+ struct list_head *pio_list, struct pio *h)
{
struct push_backup *pb = ploop->pb;
- struct bio *bio;
- /* Remove from tree and queue attached bios */
- unlink_endio_hook(ploop, &pb->rb_root, h, bio_list);
+ /* Remove from tree and queue attached pios */
+ unlink_endio_hook(ploop, &pb->rb_root, h, pio_list);
/* Unlink from pb->pending */
list_del_init(&h->list);
/* Queue relared bio itself */
- bio = dm_bio_from_per_bio_data(h, sizeof(*h));
- bio_list_add(bio_list, bio);
+ list_add_tail(&h->list, pio_list);
}
void cleanup_backup(struct ploop *ploop)
{
- struct bio_list bio_list = BIO_EMPTY_LIST;
struct push_backup *pb = ploop->pb;
struct pio *h;
struct rb_node *node;
+ LIST_HEAD(pio_list);
spin_lock_irq(&ploop->pb_lock);
/* Take bat_rwlock for visability in ploop_map() */
@@ -1549,12 +1548,12 @@ void cleanup_backup(struct ploop *ploop)
while ((node = pb->rb_root.rb_node) != NULL) {
h = rb_entry(node, struct pio, node);
- unlink_postponed_backup_endio(ploop, &bio_list, h);
+ unlink_postponed_backup_endio(ploop, &pio_list, h);
}
spin_unlock_irq(&ploop->pb_lock);
- if (!bio_list_empty(&bio_list))
- defer_bios(ploop, NULL, &bio_list);
+ if (!list_empty(&pio_list))
+ defer_pios(ploop, NULL, &pio_list);
del_timer_sync(&pb->deadline_timer);
}
@@ -1586,9 +1585,9 @@ static void check_services_timeout(struct ploop *ploop)
void do_ploop_work(struct work_struct *ws)
{
struct ploop *ploop = container_of(ws, struct ploop, worker);
- struct bio_list deferred_bios = BIO_EMPTY_LIST;
struct bio_list discard_bios = BIO_EMPTY_LIST;
struct ploop_index_wb piwb;
+ LIST_HEAD(deferred_pios);
/*
* In piwb we collect inquires of indexes updates, which are
@@ -1606,13 +1605,12 @@ void do_ploop_work(struct work_struct *ws)
process_deferred_cmd(ploop, &piwb);
process_delta_wb(ploop, &piwb);
- bio_list_merge(&deferred_bios, &ploop->deferred_bios);
+ list_splice_init(&ploop->deferred_pios, &deferred_pios);
bio_list_merge(&discard_bios, &ploop->discard_bios);
- bio_list_init(&ploop->deferred_bios);
bio_list_init(&ploop->discard_bios);
spin_unlock_irq(&ploop->deferred_lock);
- process_deferred_bios(ploop, &deferred_bios, &piwb);
+ process_deferred_pios(ploop, &deferred_pios, &piwb);
process_discard_bios(ploop, &discard_bios, &piwb);
if (piwb.page_nr != PAGE_NR_NONE) {
@@ -1648,6 +1646,7 @@ void do_ploop_fsync_work(struct work_struct *ws)
*/
int ploop_map(struct dm_target *ti, struct bio *bio)
{
+ struct pio *pio = bio_to_endio_hook(bio);
struct ploop *ploop = ti->private;
unsigned int cluster;
unsigned long flags;
@@ -1660,7 +1659,7 @@ int ploop_map(struct dm_target *ti, struct bio *bio)
if (op_is_discard(bio->bi_opf))
return ploop_map_discard(ploop, bio);
- defer_bios(ploop, bio, NULL);
+ defer_pios(ploop, pio, NULL);
return DM_MAPIO_SUBMITTED;
}
diff --git a/drivers/md/dm-ploop-target.c b/drivers/md/dm-ploop-target.c
index 4b02b8a2da4b..94df7fa3cad1 100644
--- a/drivers/md/dm-ploop-target.c
+++ b/drivers/md/dm-ploop-target.c
@@ -309,7 +309,7 @@ static int ploop_ctr(struct dm_target *ti, unsigned int argc, char **argv)
spin_lock_init(&ploop->deferred_lock);
spin_lock_init(&ploop->pb_lock);
- bio_list_init(&ploop->deferred_bios);
+ INIT_LIST_HEAD(&ploop->deferred_pios);
bio_list_init(&ploop->flush_bios);
bio_list_init(&ploop->discard_bios);
INIT_LIST_HEAD(&ploop->cluster_lk_list);
diff --git a/drivers/md/dm-ploop.h b/drivers/md/dm-ploop.h
index 421fcc153bd9..6ec1cd1b8d5f 100644
--- a/drivers/md/dm-ploop.h
+++ b/drivers/md/dm-ploop.h
@@ -202,7 +202,7 @@ struct ploop {
unsigned int inflight_bios_ref_index:1;
spinlock_t deferred_lock;
- struct bio_list deferred_bios;
+ struct list_head deferred_pios;
struct bio_list flush_bios;
struct bio_list discard_bios;
@@ -236,6 +236,9 @@ struct pio {
/* List of pios, which will be queued from this pio end */
struct list_head endio_list;
+ struct bvec_iter bi_iter;
+ struct bio_vec *bi_io_vec;
+
unsigned int cluster;
#define PLOOP_END_IO_NONE 0
@@ -507,7 +510,7 @@ extern bool try_update_bat_entry(struct ploop *ploop, unsigned int cluster,
extern int convert_bat_entries(u32 *bat_entries, u32 count);
extern int ploop_add_delta(struct ploop *ploop, u32 level, struct file *file, bool is_raw);
-extern void defer_bios(struct ploop *ploop, struct bio *bio, struct bio_list *bio_list);
+extern void defer_pios(struct ploop *ploop, struct pio *pio, struct list_head *pio_list);
extern void do_ploop_work(struct work_struct *ws);
extern void do_ploop_fsync_work(struct work_struct *ws);
extern void process_deferred_cmd(struct ploop *ploop,
@@ -517,7 +520,7 @@ extern int ploop_endio(struct dm_target *ti, struct bio *bio, blk_status_t *err)
extern int ploop_inflight_bios_ref_switch(struct ploop *ploop, bool killable);
extern struct pio *find_lk_of_cluster(struct ploop *ploop, u32 cluster);
extern void unlink_postponed_backup_endio(struct ploop *ploop,
- struct bio_list *bio_list,
+ struct list_head *pio_list,
struct pio *h);
extern int rw_page_sync(unsigned rw, struct file *file,
u64 index, struct page *page);
More information about the Devel
mailing list