[CRIU] [RFC PATCH 1/7] lazy-pages: extend the page_read with ability to read remote pages

[Pavel Emelyanov]

On 11/14/2016 06:37 PM, Mike Rapoport wrote:
> Currently lazy-pages daemon uses either pr->read_pages or get_remote_pages
> to get actual page data from local images or remote server. From now on,
> page_read will be completely responsible for getting the page data.
> 
> Signed-off-by: Mike Rapoport <rppt at linux.vnet.ibm.com>
> ---
>  criu/include/pagemap.h |  7 ++++++-
>  criu/pagemap.c         | 32 +++++++++++++++++++++++++++-----
>  criu/uffd.c            | 11 +++++------
>  3 files changed, 38 insertions(+), 12 deletions(-)
> 
> diff --git a/criu/include/pagemap.h b/criu/include/pagemap.h
> index 711e169..e4a4ecd 100644
> --- a/criu/include/pagemap.h
> +++ b/criu/include/pagemap.h
> @@ -58,6 +58,9 @@ struct page_read {
>  	void (*reset)(struct page_read *pr);
>  	int (*sync)(struct page_read *pr);
>  
> +	int (*maybe_read_page)(struct page_read *pr, unsigned long vaddr,
> +			       unsigned long len, void *buf, unsigned flags);
> +
>  	/* Private data of reader */
>  	struct cr_img *pmi;
>  	struct cr_img *pi;
> @@ -72,7 +75,8 @@ struct page_read {
>  
>  	struct iovec bunch;		/* record consequent neighbour
>  					   iovecs to punch together */
> -	unsigned id; /* for logging */
> +	unsigned id;			/* for logging */
> +	int pid;			/* PID of the process */
>  
>  	PagemapEntry **pmes;
>  	int nr_pmes;
> @@ -90,6 +94,7 @@ struct page_read {
>  
>  #define PR_TYPE_MASK	0x3
>  #define PR_MOD		0x4	/* Will need to modify */
> +#define PR_REMOTE	0x8
>  
>  /*
>   * -1 -- error
> diff --git a/criu/pagemap.c b/criu/pagemap.c
> index 1112515..728b031 100644
> --- a/criu/pagemap.c
> +++ b/criu/pagemap.c
> @@ -10,6 +10,7 @@
>  #include "cr_options.h"
>  #include "servicefd.h"
>  #include "pagemap.h"
> +#include "page-xfer.h"
>  
>  #include "xmalloc.h"
>  #include "protobuf.h"
> @@ -377,8 +378,8 @@ static int enqueue_async_page(struct page_read *pr, unsigned long vaddr,
>  	return 0;
>  }
>  
> -static int maybe_read_page(struct page_read *pr, unsigned long vaddr,
> -			   unsigned long len, void *buf, unsigned flags)
> +static int maybe_read_page_local(struct page_read *pr, unsigned long vaddr,
> +				 unsigned long len, void *buf, unsigned flags)
>  {
>  	int ret;
>  
> @@ -392,6 +393,19 @@ static int maybe_read_page(struct page_read *pr, unsigned long vaddr,
>  	return ret;
>  }
>  
> +static int maybe_read_page_remote(struct page_read *pr, unsigned long vaddr,
> +				  unsigned long len, void *buf, unsigned flags)
> +{
> +	int ret;
> +
> +	if (flags & PR_ASYNC)
> +		ret = -1;	/* not yet supported */
> +	else
> +		ret = get_remote_pages(pr->pid, vaddr, len / PAGE_SIZE, buf);
> +
> +	return ret;
> +}
> +
>  static int read_pagemap_page(struct page_read *pr, unsigned long vaddr, int nr,
>  			     void *buf, unsigned flags)
>  {
> @@ -407,7 +421,7 @@ static int read_pagemap_page(struct page_read *pr, unsigned long vaddr, int nr,
>  		/* zero mappings should be skipped by get_pagemap */
>  		BUG();
>  	} else {
> -		if (maybe_read_page(pr, vaddr, len, buf, flags) < 0)
> +		if (pr->maybe_read_page(pr, vaddr, len, buf, flags) < 0)
>  			return -1;
>  	}
>  
> @@ -605,6 +619,7 @@ int open_page_read_at(int dfd, int pid, struct page_read *pr, int pr_flags)
>  {
>  	int flags, i_typ;
>  	static unsigned ids = 1;
> +	bool remote = pr_flags & PR_REMOTE;
>  
>  	if (opts.auto_dedup)
>  		pr_flags |= PR_MOD;
> @@ -668,9 +683,16 @@ int open_page_read_at(int dfd, int pid, struct page_read *pr, int pr_flags)

Here between these lines we have call to try_open_parent(), but parent
page_read-s are always local, so the PR_REMOTE flag should (probably)
be dropped. Should it?

>  	pr->reset = reset_pagemap;
>  	pr->sync = process_async_reads;
>  	pr->id = ids++;
> +	pr->pid = pid;
> +
> +	if (remote)
> +		pr->maybe_read_page = maybe_read_page_remote;
> +	else
> +		pr->maybe_read_page = maybe_read_page_local;
>  
> -	pr_debug("Opened page read %u (parent %u)\n",
> -			pr->id, pr->parent ? pr->parent->id : 0);
> +	pr_debug("Opened %s page read %u (parent %u)\n",
> +		 remote ? "remote" : "local", pr->id,
> +		 pr->parent ? pr->parent->id : 0);
>  
>  	return 1;
>  }
> diff --git a/criu/uffd.c b/criu/uffd.c
> index 06857fd..3eddf85 100644
> --- a/criu/uffd.c
> +++ b/criu/uffd.c
> @@ -452,6 +452,7 @@ static int ud_open(int client, struct lazy_pages_info **_lpi)
>  	struct lazy_pages_info *lpi;
>  	int ret = -1;
>  	int uffd_flags;
> +	int pr_flags = PR_TASK;
>  
>  	lpi = lpi_init();
>  	if (!lpi)
> @@ -485,7 +486,9 @@ static int ud_open(int client, struct lazy_pages_info **_lpi)
>  	uffd_flags = fcntl(lpi->lpfd.fd, F_GETFD, NULL);
>  	pr_debug("uffd_flags are 0x%x\n", uffd_flags);
>  
> -	ret = open_page_read(lpi->pid, &lpi->pr, PR_TASK);
> +	if (opts.use_page_server)
> +		pr_flags |= PR_REMOTE;
> +	ret = open_page_read(lpi->pid, &lpi->pr, pr_flags);
>  	if (ret <= 0) {
>  		ret = -1;
>  		goto out;
> @@ -584,11 +587,7 @@ static int uffd_handle_pages(struct lazy_pages_info *lpi, __u64 address, int nr)
>  	if (ret == 0 || pagemap_zero(lpi->pr.pe))
>  		return uffd_zero(lpi, address, nr);
>  
> -	if (opts.use_page_server)
> -		ret = get_remote_pages(lpi->pid, address, nr, lpi->buf);
> -	else
> -		ret = lpi->pr.read_pages(&lpi->pr, address, nr, lpi->buf, 0);
> -
> +	ret = lpi->pr.read_pages(&lpi->pr, address, nr, lpi->buf, 0);
>  	if (ret <= 0) {
>  		pr_err("%d: failed reading pages at %llx\n", lpi->pid, address);
>  		return ret;
>