[CRIU] [PATCH RFC 8/8] criu: lazy-pages: enable remoting of lazy pages

[Pavel Emelyanov]

On 05/29/2016 09:58 AM, Mike Rapoport wrote:
> On Fri, May 27, 2016 at 10:38:00PM +0300, Pavel Emelyanov wrote:
>> On 05/21/2016 01:49 PM, Mike Rapoport wrote:
>>> The remote lazy pages variant can be run as follows:
>>>
>>> src# criu dump -t <pid> --lazy-pages --port 9876 -D /tmp/1 &
>>
>> This thing starts dump and lazy page server that flushes pages remotely.
>>
>>> src# while ! sudo fuser 9876/tcp ; do sleep 1; done
>>> src# scp -r /tmp/1/ dst:/tmp/
>>>
>>> dst# criu lazy-pages --lazy-addr /tmp/uffd.sock --page-server \
>>>                      --address dst --port 9876 -D /tmp/1 &
>>
>> This will start lazy pages client that would connect to dump side
>> and ... request for pages?
> 
> Yep, lazy-pages daemon will connect to the dump side and forward page fault
> requests there.
>  
>>> dst# criu restore --lazy-pages --lazy-addr /tmp/uffd.sock -D /tmp/1
>>
>> One more comment inline :)
>>
>>> Signed-off-by: Mike Rapoport <rppt at linux.vnet.ibm.com>
>>> ---
>>>  criu/cr-dump.c   | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++--
>>>  criu/page-read.c |  2 +-
>>>  criu/uffd.c      |  9 ++++++++-
>>>  3 files changed, 61 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/criu/cr-dump.c b/criu/cr-dump.c
>>> index 1a551b4..faca73b 100644
>>> --- a/criu/cr-dump.c
>>> +++ b/criu/cr-dump.c
>>> @@ -1298,7 +1298,7 @@ static int dump_one_task(struct pstree_item *item)
>>>  		}
>>>  	}
>>>  
>>> -	ret = parasite_dump_pages_seized(parasite_ctl, &vmas, false);
>>> +	ret = parasite_dump_pages_seized(parasite_ctl, &vmas, opts.lazy_pages);
>>>  	if (ret)
>>>  		goto err_cure;
>>>  
>>> @@ -1338,7 +1338,10 @@ static int dump_one_task(struct pstree_item *item)
>>>  		goto err;
>>>  	}
>>>  
>>> -	ret = parasite_cure_seized(parasite_ctl);
>>> +	if (opts.lazy_pages)
>>> +		ret = parasite_cure_remote(parasite_ctl);
>>> +	else
>>> +		ret = parasite_cure_seized(parasite_ctl);
>>>  	if (ret) {
>>>  		pr_err("Can't cure (pid: %d) from parasite\n", pid);
>>>  		goto err;
>>> @@ -1525,6 +1528,49 @@ err:
>>>  	return cr_pre_dump_finish(ret);
>>>  }
>>>  
>>> +static int cr_lazy_mem_dump(void)
>>> +{
>>> +	struct pstree_item *item;
>>> +	int ret = 0;
>>> +
>>> +	pr_info("Lazy pages: pre-dumping memory\n");
>>> +	for_each_pstree_item(item) {
>>> +		struct parasite_ctl *ctl = item->parasite_ctl;
>>> +		struct page_xfer xfer;
>>> +
>>> +		timing_start(TIME_MEMWRITE);
>>> +		ret = open_page_xfer(&xfer, CR_FD_PAGEMAP, ctl->pid.virt);
>>> +		if (ret < 0)
>>> +			goto err;
>>> +
>>> +		ret = page_xfer_dump_pages(&xfer, ctl->mem_pp, 0, false);
>>
>> If I got the set right :) 
> 
> Hmm, not quite ;-)
> 
>> this place just flushes all the lazy memory into
>> socket ignoring the requests from restore side and that's it. No?
> 
> This place flushes all the memory *except* lazy pages into the images.
> The buffers that are marked with PPB_LAZY are not  written.

Ah! Indeed. The last 'false', yes. So here we flush all the memory __but__ the
lazy one. OK, the question is -- why not write one using existing --page-server
dump?

And one more comment regarding this below.

>>> +		xfer.close(&xfer);
>>> +
>>> +		if (ret)
>>> +			goto err;
>>> +
>>> +		timing_stop(TIME_MEMWRITE);
>>> +	}
>>> +
>>> +	pr_info("Starting lazy pages server\n");
>>> +	ret = cr_page_server(false, -1);
>>> +
>>> +	for_each_pstree_item(item) {
>>> +		struct parasite_ctl *ctl = item->parasite_ctl;
>>> +		destroy_page_pipe(ctl->mem_pp);
>>> +		parasite_cure_local(ctl);
>>> +	}
>>> +
>>> +err:
>>> +	if (ret)
>>> +		pr_err("Lazy pages transfer FAILED.\n");
>>> +	else
>>> +		pr_info("Lazy pages transfer finished successfully\n");
>>> +
>>> +	return ret;
>>> +}
>>> +
>>>  static int cr_dump_finish(int ret)
>>>  {
>>>  	int post_dump_ret = 0;
>>> @@ -1583,6 +1629,10 @@ static int cr_dump_finish(int ret)
>>>  		network_unlock();
>>>  		delete_link_remaps();
>>>  	}
>>> +
>>> +	if (opts.lazy_pages)
>>> +		ret = cr_lazy_mem_dump();
>>> +
>>>  	pstree_switch_state(root_item,
>>>  			    (ret || post_dump_ret) ?
>>>  			    TASK_ALIVE : opts.final_state);
>>> diff --git a/criu/page-read.c b/criu/page-read.c
>>> index e5ec76a..203b170 100644
>>> --- a/criu/page-read.c
>>> +++ b/criu/page-read.c
>>> @@ -92,7 +92,7 @@ static void skip_pagemap_pages(struct page_read *pr, unsigned long len)
>>>  		return;
>>>  
>>>  	pr_debug("\tpr%u Skip %lu bytes from page-dump\n", pr->id, len);
>>> -	if (!pr->pe->in_parent)
>>> +	if (!pr->pe->in_parent && !opts.lazy_pages)
>>>  		lseek(img_raw_fd(pr->pi), len, SEEK_CUR);
>>>  	pr->cvaddr += len;
>>>  }
>>> diff --git a/criu/uffd.c b/criu/uffd.c
>>> index ef1ff89..d140d38 100644
>>> --- a/criu/uffd.c
>>> +++ b/criu/uffd.c
>>> @@ -33,6 +33,7 @@
>>>  #include "xmalloc.h"
>>>  #include "syscall-codes.h"
>>>  #include "restorer.h"
>>> +#include "page-xfer.h"
>>>  
>>>  #undef  LOG_PREFIX
>>>  #define LOG_PREFIX "lazy-pages: "
>>> @@ -363,7 +364,10 @@ static int uffd_copy_page(struct lazy_pages_info *lpi, __u64 address,
>>>  	struct uffdio_copy uffdio_copy;
>>>  	int rc;
>>>  
>>> -	rc = get_page(lpi, address, dest);
>>> +	if (opts.use_page_server)
>>> +		rc = get_remote_pages(lpi->pid, address, 1, dest);

Would you make the respective page_read handler for this case?

>>> +	else
>>> +		rc = get_page(lpi, address, dest);
>>>  	if (rc <= 0)
>>>  		return rc;
>>>  
>>> @@ -861,6 +865,9 @@ int cr_lazy_pages()
>>>  	if (prepare_uffds(epollfd))
>>>  		return -1;
>>>  
>>> +	if (connect_to_page_server())
>>> +		return -1;
>>> +
>>>  	ret = handle_requests(epollfd, events);
>>>  	lpi_hash_fini();
>>>  
>>>
>>
> 
> .
>