[Devel] [RH7 PATCH 3/6] ploop: add delayed flush support

[Maxim Patlasov]

On 06/23/2016 10:25 AM, Dmitry Monakhov wrote:
> dio_submit and dio_submit_pad may produce several bios. This makes
> processing of REQ_FUA complicated because in order to preserve correctness
> we have to TAG each bio with FUA flag which is suboptimal.
> Obviously there is a room for optimization here: once all bios was acknowledged
> by lower layer we may issue empty barrier aka ->issue_flush().
> post_submit call back is the place where we all bios completed already.
>
> b1:FUA, b2:FUA, b3:FUA =>  b1,b2,b3,wait_for_bios,bX:FLUSH
>
> This allow us to remove all this REQ_FORCE_{FLUSH,FUA} crap and

It seems we can remove REQ_FORCE_{FLUSH,FUA} right now. Only RELOC_A|S 
needs it and we can fix them in a simple and straightforward way -- 
essentially your 5th patch must be enough enough.

Btw, this patch doesn't disable the logic of passing FUA from incoming 
bio-s to outgoing (commit c2247f3745). Was it by a miss or deliberately?

>
> Signed-off-by: Dmitry Monakhov <dmonakhov at openvz.org>
> ---
>   drivers/block/ploop/io_direct.c | 48 +++++++++++++++++------------------------
>   include/linux/ploop/ploop.h     |  2 ++
>   2 files changed, 22 insertions(+), 28 deletions(-)
>
> diff --git a/drivers/block/ploop/io_direct.c b/drivers/block/ploop/io_direct.c
> index 195d318..752a9c3e 100644
> --- a/drivers/block/ploop/io_direct.c
> +++ b/drivers/block/ploop/io_direct.c
> @@ -82,31 +82,13 @@ dio_submit(struct ploop_io *io, struct ploop_request * preq,
>   	sector_t sec, nsec;
>   	int err;
>   	struct bio_list_walk bw;
> -	int preflush;
> -	int postfua = 0;
> +	int preflush = !!(rw & REQ_FLUSH);
> +	int postflush = !!(rw & REQ_FUA);
>   	int write = !!(rw & REQ_WRITE);
>   
>   	trace_submit(preq);
>   
> -	preflush = !!(rw & REQ_FLUSH);
> -
> -	if (test_and_clear_bit(PLOOP_REQ_FORCE_FLUSH, &preq->state))
> -		preflush = 1;
> -
> -	if (test_and_clear_bit(PLOOP_REQ_FORCE_FUA, &preq->state))
> -		postfua = 1;
> -
> -	if (!postfua && ploop_req_delay_fua_possible(rw, preq)) {
> -
> -		/* Mark req that delayed flush required */
> -		set_bit(PLOOP_REQ_FORCE_FLUSH, &preq->state);
> -	} else if (rw & REQ_FUA) {
> -		postfua = 1;
> -	}
> -
>   	rw &= ~(REQ_FLUSH | REQ_FUA);
> -
> -
>   	bio_list_init(&bl);
>   
>   	if (iblk == PLOOP_ZERO_INDEX)
> @@ -237,13 +219,14 @@ flush_bio:
>   			rw2 |= REQ_FLUSH;
>   			preflush = 0;
>   		}
> -		if (unlikely(postfua && !bl.head))
> -			rw2 |= REQ_FUA;
> -
>   		ploop_acc_ff_out(preq->plo, rw2 | b->bi_rw);
>   		submit_bio(rw2, b);
>   	}
> -
> +	/* TODO: minor optimization is possible for single bio case */
> +	if (postflush) {
> +		set_bit(PLOOP_REQ_DEL_FLUSH, &preq->state);
> +		ploop_add_post_submit(io, preq);
> +	}
>   	ploop_complete_io_request(preq);
>   	return;
>   
> @@ -523,9 +506,10 @@ dio_convert_extent(struct ploop_io *io, struct ploop_request * preq)
>   					      (loff_t)sec << 9, clu_siz);
>   
>   	/* highly unlikely case: FUA coming to a block not provisioned yet */
> -	if (!err && force_sync)
> +	if (!err && force_sync) {
> +		clear_bit(PLOOP_REQ_DEL_FLUSH, &preq->state);
>   		err = io->ops->sync(io);
> -
> +	}
>   	if (!force_sync) {
>   		spin_lock_irq(&plo->lock);
>   		io->io_count++;
> @@ -546,7 +530,12 @@ dio_post_submit(struct ploop_io *io, struct ploop_request * preq)
>   	if (test_and_clear_bit(PLOOP_REQ_DEL_CONV, &preq->state))
>   		dio_convert_extent(io, preq);
>   
> +	if (test_and_clear_bit(PLOOP_REQ_DEL_FLUSH, &preq->state)) {
> +		io->ops->issue_flush(io, preq);
> +		return 1;
> +	}
>   	return 0;
> +
>   }
>   
>   /* Submit the whole cluster. If preq contains only partial data
> @@ -562,7 +551,6 @@ dio_submit_pad(struct ploop_io *io, struct ploop_request * preq,
>   	sector_t sec, end_sec, nsec, start, end;
>   	struct bio_list_walk bw;
>   	int err;
> -
>   	bio_list_init(&bl);
>   
>   	/* sec..end_sec is the range which we are going to write */
> @@ -694,7 +682,11 @@ flush_bio:
>   		ploop_acc_ff_out(preq->plo, rw | b->bi_rw);
>   		submit_bio(rw, b);
>   	}
> -
> +	/* TODO: minor optimization is possible for single bio case */
> +	if (preq->req_rw &  REQ_FUA) {
> +		set_bit(PLOOP_REQ_DEL_FLUSH, &preq->state);
> +		ploop_add_post_submit(io, preq);
> +	}
>   	ploop_complete_io_request(preq);
>   	return;
>   
> diff --git a/include/linux/ploop/ploop.h b/include/linux/ploop/ploop.h
> index 4c52a40..5076f16 100644
> --- a/include/linux/ploop/ploop.h
> +++ b/include/linux/ploop/ploop.h
> @@ -472,6 +472,7 @@ enum
>   	PLOOP_REQ_PUSH_BACKUP, /* preq was ACKed by userspace push_backup */
>   	PLOOP_REQ_ALLOW_READS, /* READs are allowed for given req_cluster */
>   	PLOOP_REQ_DEL_CONV,    /* post_submit: conversion required */
> +	PLOOP_REQ_DEL_FLUSH,   /* post_submit: REQ_FLUSH required */
>   	PLOOP_REQ_FSYNC_DONE,  /* fsync_thread() performed f_op->fsync() */
>   };
>   
> @@ -482,6 +483,7 @@ enum
>   #define PLOOP_REQ_ZERO_FL (1 << PLOOP_REQ_ZERO)
>   #define PLOOP_REQ_POST_SUBMIT_FL (1 << PLOOP_REQ_POST_SUBMIT)
>   #define PLOOP_REQ_DEL_CONV_FL (1 << PLOOP_REQ_DEL_CONV)
> +#define PLOOP_REQ_DEL_FLUSH_FL (1 << PLOOP_REQ_DEL_FLUSH)
>   
>   enum
>   {