[Devel] [PATCH RHEL7 COMMIT] ploop: fix barriers for PLOOP_E_RELOC_NULLIFY

[Konstantin Khorenko]

The commit is pushed to "branch-rh7-3.10.0-327.18.2.vz7.14.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh7-3.10.0-327.18.2.vz7.14.20
------>
commit 1da7d3f275f9dbc96dc389af3aa18819e112b9cb
Author: Maxim Patlasov <mpatlasov at virtuozzo.com>
Date:   Mon Jun 27 11:10:26 2016 +0400

    ploop: fix barriers for PLOOP_E_RELOC_NULLIFY
    
    The last step of processing of RELOC_A request is
    nullifying BAT block. We smartly noticed, that flush
    needed after that, but fsync is not enough:
    
    > 	/*
    > 	 * Lately we think we does sync of nullified blocks at format
    > 	 * driver by image fsync before header update.
    > 	 * But we write this data directly into underlying device
    > 	 * bypassing EXT4 by usage of extent map tree
    > 	 * (see dio_submit()). So fsync of EXT4 image doesnt help us.
    > 	 * We need to force sync of nullified blocks.
    > 	 */
    > 	set_bit(PLOOP_REQ_FORCE_FUA, &preq->state);
    > 	top_delta->io.ops->submit(&top_delta->io, preq, preq->req_rw,
    > 				  &sbl, preq->iblock, 1<<plo->cluster_log);
    
    Unfortunately, the way how we handle FORCE_FUA in dio_submit
    (sending last bio with REQ_FUA bit set) is not safe: firstly because
    we decided that ploop shouldn't strongly rely on the assumption of
    equivalence of REQ_FUA and post-FLUSH; and secondly because dio_submit
    cannot ensure that that last bio marked as REQ_FUA won't be actually
    processed before others.
    
    To fix this problem the patch makes explicit ->issue_flush to flush
    nullified block.
    
    Signed-off-by: Maxim Patlasov <mpatlasov at virtuozzo.com>
    Acked-by: Dmitry Monakhov <dmonakhov at virtuozzo.com>
---
 drivers/block/ploop/dev.c       | 11 ++++++++++-
 drivers/block/ploop/io_direct.c |  3 ++-
 drivers/block/ploop/map.c       |  4 +++-
 include/linux/ploop/ploop.h     |  1 +
 4 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/drivers/block/ploop/dev.c b/drivers/block/ploop/dev.c
index 557ddba..2b60dfa 100644
--- a/drivers/block/ploop/dev.c
+++ b/drivers/block/ploop/dev.c
@@ -1305,6 +1305,8 @@ static void ploop_complete_request(struct ploop_request * preq)
 	}
 	preq->bl.tail = NULL;
 
+	WARN_ON(!preq->error && test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state));
+
 	if (test_bit(PLOOP_REQ_RELOC_A, &preq->state) ||
 	    test_bit(PLOOP_REQ_RELOC_S, &preq->state)) {
 		if (preq->error)
@@ -2429,6 +2431,13 @@ static void ploop_req_state_process(struct ploop_request * preq)
 		preq->eng_io = NULL;
 	}
 
+	if (test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state)) {
+		preq->eng_io->ops->issue_flush(preq->eng_io, preq);
+		clear_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state);
+		preq->eng_io = NULL;
+		goto out;
+	}
+
 restart:
 	BUG_ON(test_bit(PLOOP_REQ_POST_SUBMIT, &preq->state));
 	__TRACE("ST %p %u %lu\n", preq, preq->req_cluster, preq->eng_state);
@@ -2705,7 +2714,7 @@ restart:
 	default:
 		BUG();
 	}
-
+out:
 	if (release_ioc) {
 		struct io_context * ioc = current->io_context;
 		current->io_context = saved_ioc;
diff --git a/drivers/block/ploop/io_direct.c b/drivers/block/ploop/io_direct.c
index ca8c145..189aa1a 100644
--- a/drivers/block/ploop/io_direct.c
+++ b/drivers/block/ploop/io_direct.c
@@ -414,6 +414,7 @@ try_again:
 
 		preq->iblock = iblk;
 		preq->eng_io = io;
+		BUG_ON(test_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state));
 		set_bit(PLOOP_REQ_POST_SUBMIT, &preq->state);
 		dio_submit_pad(io, preq, sbl, size, em);
 		err = 0;
@@ -1824,7 +1825,7 @@ static void dio_issue_flush(struct ploop_io * io, struct ploop_request *preq)
 
 	atomic_inc(&preq->io_count);
 	ploop_acc_ff_out(io->plo, preq->req_rw | bio->bi_rw);
-	submit_bio(preq->req_rw, bio);
+	submit_bio(WRITE_FLUSH, bio);
 	ploop_complete_io_request(preq);
 }
 
diff --git a/drivers/block/ploop/map.c b/drivers/block/ploop/map.c
index f87fb08..915a216 100644
--- a/drivers/block/ploop/map.c
+++ b/drivers/block/ploop/map.c
@@ -1077,7 +1077,9 @@ static void map_wb_complete_post_process(struct ploop_map *map,
 	 * (see dio_submit()). So fsync of EXT4 image doesnt help us.
 	 * We need to force sync of nullified blocks.
 	 */
-	set_bit(PLOOP_REQ_FORCE_FUA, &preq->state);
+	preq->eng_io = &top_delta->io;
+	BUG_ON(test_bit(PLOOP_REQ_POST_SUBMIT, &preq->state));
+	set_bit(PLOOP_REQ_ISSUE_FLUSH, &preq->state);
 	top_delta->io.ops->submit(&top_delta->io, preq, preq->req_rw,
 				  &sbl, preq->iblock, 1<<plo->cluster_log);
 }
diff --git a/include/linux/ploop/ploop.h b/include/linux/ploop/ploop.h
index af222f1..920daf7 100644
--- a/include/linux/ploop/ploop.h
+++ b/include/linux/ploop/ploop.h
@@ -479,6 +479,7 @@ enum
 	PLOOP_REQ_POST_SUBMIT, /* preq needs post_submit processing */
 	PLOOP_REQ_PUSH_BACKUP, /* preq was ACKed by userspace push_backup */
 	PLOOP_REQ_FSYNC_DONE,  /* fsync_thread() performed f_op->fsync() */
+	PLOOP_REQ_ISSUE_FLUSH, /* preq needs ->issue_flush before completing */
 };
 
 #define PLOOP_REQ_MERGE_FL (1 << PLOOP_REQ_MERGE)