[Devel] [PATCH rh7 8/9] ploop: fix barriers for PLOOP_E_RELOC_NULLIFY

[Maxim Patlasov]

The last step of processing of RELOC_A request is
nullifying BAT block. We smartly noticed, that flush
needed after that, but fsync is not enough:

> 	/*
> 	 * Lately we think we does sync of nullified blocks at format
> 	 * driver by image fsync before header update.
> 	 * But we write this data directly into underlying device
> 	 * bypassing EXT4 by usage of extent map tree
> 	 * (see dio_submit()). So fsync of EXT4 image doesnt help us.
> 	 * We need to force sync of nullified blocks.
> 	 */
> 	set_bit(PLOOP_REQ_FORCE_FUA, &preq->state);
> 	top_delta->io.ops->submit(&top_delta->io, preq, preq->req_rw,
> 				  &sbl, preq->iblock, 1<<plo->cluster_log);

Unfortunately, the way how we handle FORCE_FUA in dio_submit
(sending last bio with REQ_FUA bit set) is not safe: firstly because
we decided that ploop shouldn't strongly rely on the assumption of
equivalence of REQ_FUA and post-FLUSH; and secondly because dio_submit
cannot ensure that that last bio marked as REQ_FUA won't be actually
processed before others.

To fix this problem the patch makes explicit ->issue_flush to flush
nullified block.

Signed-off-by: Maxim Patlasov <mpatlasov at virtuozzo.com>
---
 drivers/block/ploop/dev.c       |   11 ++++++++++-
 drivers/block/ploop/io_direct.c |    3 ++-
 drivers/block/ploop/map.c       |    4 +++-
 include/linux/ploop/ploop.h     |    1 +
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/drivers/block/ploop/dev.c b/drivers/block/ploop/dev.c
index 557ddba..2b60dfa 100644
--- a/drivers/block/ploop/dev.c
+++ b/drivers/block/ploop/dev.c
@@ -1305,6 +1305,8 @@ static void ploop_complete_request(struct ploop_request * preq)
 	}
 	preq->bl.tail = NULL;
 
+	WARN_ON(!preq->error && test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state));
+
 	if (test_bit(PLOOP_REQ_RELOC_A, &preq->state) ||
 	    test_bit(PLOOP_REQ_RELOC_S, &preq->state)) {
 		if (preq->error)
@@ -2429,6 +2431,13 @@ static void ploop_req_state_process(struct ploop_request * preq)
 		preq->eng_io = NULL;
 	}
 
+	if (test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state)) {
+		preq->eng_io->ops->issue_flush(preq->eng_io, preq);
+		clear_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state);
+		preq->eng_io = NULL;
+		goto out;
+	}
+
 restart:
 	BUG_ON(test_bit(PLOOP_REQ_POST_SUBMIT, &preq->state));
 	__TRACE("ST %p %u %lu\n", preq, preq->req_cluster, preq->eng_state);
@@ -2705,7 +2714,7 @@ restart:
 	default:
 		BUG();
 	}
-
+out:
 	if (release_ioc) {
 		struct io_context * ioc = current->io_context;
 		current->io_context = saved_ioc;
diff --git a/drivers/block/ploop/io_direct.c b/drivers/block/ploop/io_direct.c
index 94936c7..c4d0f63 100644
--- a/drivers/block/ploop/io_direct.c
+++ b/drivers/block/ploop/io_direct.c
@@ -413,6 +413,7 @@ try_again:
 
 		preq->iblock = iblk;
 		preq->eng_io = io;
+		BUG_ON(test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state));
 		set_bit(PLOOP_REQ_POST_SUBMIT, &preq->state);
 		dio_submit_pad(io, preq, sbl, size, em);
 		err = 0;
@@ -1819,7 +1820,7 @@ static void dio_issue_flush(struct ploop_io * io, struct ploop_request *preq)
 
 	atomic_inc(&preq->io_count);
 	ploop_acc_ff_out(io->plo, preq->req_rw | bio->bi_rw);
-	submit_bio(preq->req_rw, bio);
+	submit_bio(WRITE_FLUSH, bio);
 	ploop_complete_io_request(preq);
 }
 
diff --git a/drivers/block/ploop/map.c b/drivers/block/ploop/map.c
index f87fb08..915a216 100644
--- a/drivers/block/ploop/map.c
+++ b/drivers/block/ploop/map.c
@@ -1077,7 +1077,9 @@ static void map_wb_complete_post_process(struct ploop_map *map,
 	 * (see dio_submit()). So fsync of EXT4 image doesnt help us.
 	 * We need to force sync of nullified blocks.
 	 */
-	set_bit(PLOOP_REQ_FORCE_FUA, &preq->state);
+	preq->eng_io = &top_delta->io;
+	BUG_ON(test_bit(PLOOP_REQ_POST_SUBMIT, &preq->state));
+	set_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state);
 	top_delta->io.ops->submit(&top_delta->io, preq, preq->req_rw,
 				  &sbl, preq->iblock, 1<<plo->cluster_log);
 }
diff --git a/include/linux/ploop/ploop.h b/include/linux/ploop/ploop.h
index af222f1..920daf7 100644
--- a/include/linux/ploop/ploop.h
+++ b/include/linux/ploop/ploop.h
@@ -479,6 +479,7 @@ enum
 	PLOOP_REQ_POST_SUBMIT, /* preq needs post_submit processing */
 	PLOOP_REQ_PUSH_BACKUP, /* preq was ACKed by userspace push_backup */
 	PLOOP_REQ_FSYNC_DONE,  /* fsync_thread() performed f_op->fsync() */
+	PLOOP_REQ_ISSUE_FLUSH, /* preq needs ->issue_flush before completing */
 };
 
 #define PLOOP_REQ_MERGE_FL (1 << PLOOP_REQ_MERGE)