[Devel] [PATCH vz9 v1 43/63] dm-ploop: use lockless lists for chained cow updates list

[Alexander Atanasov]

reduce locking for cow pios chained to bat updates

https://virtuozzo.atlassian.net/browse/VSTOR-91821
Signed-off-by: Alexander Atanasov <alexander.atanasov at virtuozzo.com>
---
 drivers/md/dm-ploop-map.c | 19 ++++++++++---------
 drivers/md/dm-ploop.h     |  2 +-
 2 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/drivers/md/dm-ploop-map.c b/drivers/md/dm-ploop-map.c
index dcdbc509c54e..809ae3ee71ec 100644
--- a/drivers/md/dm-ploop-map.c
+++ b/drivers/md/dm-ploop-map.c
@@ -63,7 +63,8 @@ void ploop_index_wb_init(struct ploop_index_wb *piwb, struct ploop *ploop)
 	piwb->bat_page = NULL;
 	piwb->bi_status = 0;
 	INIT_LIST_HEAD(&piwb->ready_data_pios);
-	INIT_LIST_HEAD(&piwb->cow_list);
+	init_llist_head(&piwb->cow_llist);
+
 	/* For ploop_bat_write_complete() */
 	atomic_set(&piwb->count, 1);
 	piwb->completed = false;
@@ -901,7 +902,8 @@ static void ploop_bat_write_complete(struct pio *pio, void *piwb_ptr,
 	struct pio *data_pio;
 	unsigned long flags;
 	LIST_HEAD(lready_pios);
-	LIST_HEAD(lcow_pios);
+	struct llist_node *pos, *t;
+	struct llist_node *ll_cow_pios;
 	int completed = atomic_read(&piwb->count) == 1;
 
 	if (completed) {
@@ -926,9 +928,7 @@ static void ploop_bat_write_complete(struct pio *pio, void *piwb_ptr,
 	list_splice_init(&piwb->ready_data_pios, &lready_pios);
 	spin_unlock_irqrestore(&piwb->lock, flags);
 
-	spin_lock_irqsave(&ploop->deferred_lock, flags);
-	list_splice_init(&piwb->cow_list, &lcow_pios);
-	spin_unlock_irqrestore(&ploop->deferred_lock, flags);
+	ll_cow_pios = llist_reverse_order(llist_del_all(&piwb->cow_llist));
 
 	/*
 	 * End pending data bios.
@@ -940,7 +940,9 @@ static void ploop_bat_write_complete(struct pio *pio, void *piwb_ptr,
 		ploop_pio_endio(data_pio);
 	}
 
-	while ((aux_pio = ploop_pio_list_pop(&lcow_pios))) {
+	llist_for_each_safe(pos, t, ll_cow_pios) {
+		aux_pio = list_entry((struct list_head *)pos, typeof(*aux_pio), list);
+		INIT_LIST_HEAD(&aux_pio->list);
 		cow = aux_pio->endio_cb_data;
 		ploop_complete_cow(cow, bi_status);
 	}
@@ -1508,9 +1510,8 @@ static void ploop_submit_cow_index_wb(struct ploop_cow *cow)
 #endif
 	/* Prevent double clearing of holes_bitmap bit on complete_cow() */
 	cow->dst_clu = BAT_ENTRY_NONE;
-	spin_lock_irq(&ploop->deferred_lock);
-	list_add_tail(&cow->aux_pio->list, &piwb->cow_list);
-	spin_unlock_irq(&ploop->deferred_lock);
+
+	llist_add((struct llist_node *)(&cow->aux_pio->list), &piwb->cow_llist);
 out:
 	return;
 err_resource:
diff --git a/drivers/md/dm-ploop.h b/drivers/md/dm-ploop.h
index 224f4915d9f1..3f80d1750ffb 100644
--- a/drivers/md/dm-ploop.h
+++ b/drivers/md/dm-ploop.h
@@ -102,7 +102,7 @@ struct ploop_index_wb {
 	struct pio *pio;
 	struct page *bat_page;
 	struct list_head ready_data_pios;
-	struct list_head cow_list;
+	struct llist_head cow_llist;
 	atomic_t count;
 	bool completed;
 	blk_status_t bi_status;
-- 
2.43.0