[CRIU] [PATCH RFC 8/8] criu: lazy-pages: enable remoting of lazy pages

[Mike Rapoport]

On Fri, May 27, 2016 at 10:38:00PM +0300, Pavel Emelyanov wrote:
> On 05/21/2016 01:49 PM, Mike Rapoport wrote:
> > The remote lazy pages variant can be run as follows:
> > 
> > src# criu dump -t <pid> --lazy-pages --port 9876 -D /tmp/1 &
> 
> This thing starts dump and lazy page server that flushes pages remotely.
> 
> > src# while ! sudo fuser 9876/tcp ; do sleep 1; done
> > src# scp -r /tmp/1/ dst:/tmp/
> > 
> > dst# criu lazy-pages --lazy-addr /tmp/uffd.sock --page-server \
> >                      --address dst --port 9876 -D /tmp/1 &
> 
> This will start lazy pages client that would connect to dump side
> and ... request for pages?

Yep, lazy-pages daemon will connect to the dump side and forward page fault
requests there.
 
> > dst# criu restore --lazy-pages --lazy-addr /tmp/uffd.sock -D /tmp/1
> 
> One more comment inline :)
> 
> > Signed-off-by: Mike Rapoport <rppt at linux.vnet.ibm.com>
> > ---
> >  criu/cr-dump.c   | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
> >  criu/page-read.c |  2 +-
> >  criu/uffd.c      |  9 ++++++++-
> >  3 files changed, 61 insertions(+), 4 deletions(-)
> > 
> > diff --git a/criu/cr-dump.c b/criu/cr-dump.c
> > index 1a551b4..faca73b 100644
> > --- a/criu/cr-dump.c
> > +++ b/criu/cr-dump.c
> > @@ -1298,7 +1298,7 @@ static int dump_one_task(struct pstree_item *item)
> >  		}
> >  	}
> >  
> > -	ret = parasite_dump_pages_seized(parasite_ctl, &vmas, false);
> > +	ret = parasite_dump_pages_seized(parasite_ctl, &vmas, opts.lazy_pages);
> >  	if (ret)
> >  		goto err_cure;
> >  
> > @@ -1338,7 +1338,10 @@ static int dump_one_task(struct pstree_item *item)
> >  		goto err;
> >  	}
> >  
> > -	ret = parasite_cure_seized(parasite_ctl);
> > +	if (opts.lazy_pages)
> > +		ret = parasite_cure_remote(parasite_ctl);
> > +	else
> > +		ret = parasite_cure_seized(parasite_ctl);
> >  	if (ret) {
> >  		pr_err("Can't cure (pid: %d) from parasite\n", pid);
> >  		goto err;
> > @@ -1525,6 +1528,49 @@ err:
> >  	return cr_pre_dump_finish(ret);
> >  }
> >  
> > +static int cr_lazy_mem_dump(void)
> > +{
> > +	struct pstree_item *item;
> > +	int ret = 0;
> > +
> > +	pr_info("Lazy pages: pre-dumping memory\n");
> > +	for_each_pstree_item(item) {
> > +		struct parasite_ctl *ctl = item->parasite_ctl;
> > +		struct page_xfer xfer;
> > +
> > +		timing_start(TIME_MEMWRITE);
> > +		ret = open_page_xfer(&xfer, CR_FD_PAGEMAP, ctl->pid.virt);
> > +		if (ret < 0)
> > +			goto err;
> > +
> > +		ret = page_xfer_dump_pages(&xfer, ctl->mem_pp, 0, false);
> 
> If I got the set right :) 

Hmm, not quite ;-)

> this place just flushes all the lazy memory into
> socket ignoring the requests from restore side and that's it. No?

This place flushes all the memory *except* lazy pages into the images.
The buffers that are marked with PPB_LAZY are not  written.
 
> > +		xfer.close(&xfer);
> > +
> > +		if (ret)
> > +			goto err;
> > +
> > +		timing_stop(TIME_MEMWRITE);
> > +	}
> > +
> > +	pr_info("Starting lazy pages server\n");
> > +	ret = cr_page_server(false, -1);
> > +
> > +	for_each_pstree_item(item) {
> > +		struct parasite_ctl *ctl = item->parasite_ctl;
> > +		destroy_page_pipe(ctl->mem_pp);
> > +		parasite_cure_local(ctl);
> > +	}
> > +
> > +err:
> > +	if (ret)
> > +		pr_err("Lazy pages transfer FAILED.\n");
> > +	else
> > +		pr_info("Lazy pages transfer finished successfully\n");
> > +
> > +	return ret;
> > +}
> > +
> >  static int cr_dump_finish(int ret)
> >  {
> >  	int post_dump_ret = 0;
> > @@ -1583,6 +1629,10 @@ static int cr_dump_finish(int ret)
> >  		network_unlock();
> >  		delete_link_remaps();
> >  	}
> > +
> > +	if (opts.lazy_pages)
> > +		ret = cr_lazy_mem_dump();
> > +
> >  	pstree_switch_state(root_item,
> >  			    (ret || post_dump_ret) ?
> >  			    TASK_ALIVE : opts.final_state);
> > diff --git a/criu/page-read.c b/criu/page-read.c
> > index e5ec76a..203b170 100644
> > --- a/criu/page-read.c
> > +++ b/criu/page-read.c
> > @@ -92,7 +92,7 @@ static void skip_pagemap_pages(struct page_read *pr, unsigned long len)
> >  		return;
> >  
> >  	pr_debug("\tpr%u Skip %lu bytes from page-dump\n", pr->id, len);
> > -	if (!pr->pe->in_parent)
> > +	if (!pr->pe->in_parent && !opts.lazy_pages)
> >  		lseek(img_raw_fd(pr->pi), len, SEEK_CUR);
> >  	pr->cvaddr += len;
> >  }
> > diff --git a/criu/uffd.c b/criu/uffd.c
> > index ef1ff89..d140d38 100644
> > --- a/criu/uffd.c
> > +++ b/criu/uffd.c
> > @@ -33,6 +33,7 @@
> >  #include "xmalloc.h"
> >  #include "syscall-codes.h"
> >  #include "restorer.h"
> > +#include "page-xfer.h"
> >  
> >  #undef  LOG_PREFIX
> >  #define LOG_PREFIX "lazy-pages: "
> > @@ -363,7 +364,10 @@ static int uffd_copy_page(struct lazy_pages_info *lpi, __u64 address,
> >  	struct uffdio_copy uffdio_copy;
> >  	int rc;
> >  
> > -	rc = get_page(lpi, address, dest);
> > +	if (opts.use_page_server)
> > +		rc = get_remote_pages(lpi->pid, address, 1, dest);
> > +	else
> > +		rc = get_page(lpi, address, dest);
> >  	if (rc <= 0)
> >  		return rc;
> >  
> > @@ -861,6 +865,9 @@ int cr_lazy_pages()
> >  	if (prepare_uffds(epollfd))
> >  		return -1;
> >  
> > +	if (connect_to_page_server())
> > +		return -1;
> > +
> >  	ret = handle_requests(epollfd, events);
> >  	lpi_hash_fini();
> >  
> > 
>