<div dir="ltr"><div><div> criu/img-cache.c                | 153 +++++++++<br></div><div> criu/img-proxy.c                |  89 +++++</div><div> criu/img-remote-proto.c         | 741 ++++++++++++++++++++++++++++++++++++++++</div><div> criu/include/img-remote-proto.h |  86 +++++</div><div> 4 files changed, 1069 insertions(+)</div><div><br></div><div>    The current patch brings the implementation of the image proxy and image cache.</div><div>    These components are necessary to perform in-memory live migration of processes</div><div>    using CRIU. The image proxy receives images from CRIU Dump/Pre-Dump (through</div><div>    UNIX sockets) and forwards them to the image cache (through a TCP socket). The</div><div>    image cache caches image in memory and sends them to CRIU Restore (through</div><div>    UNIX sockets) when requested.</div><div>    </div><div>    Signed-off-by: Rodrigo Bruno &lt;<a href="mailto:rbruno@gsd.inesc-id.pt">rbruno@gsd.inesc-id.pt</a>&gt;</div><div>    Signed-off-by: Katerina Koukiou &lt;<a href="mailto:k.koukiou@gmail.com">k.koukiou@gmail.com</a>&gt;</div><div><br></div><div>diff --git a/criu/img-cache.c b/criu/img-cache.c</div><div>new file mode 100644</div><div>index 0000000..9111fbd</div><div>--- /dev/null</div><div>+++ b/criu/img-cache.c</div><div>@@ -0,0 +1,153 @@</div><div>+#include &lt;unistd.h&gt;</div><div>+</div><div>+#include &quot;img-remote-proto.h&quot;</div><div>+#include &quot;criu-log.h&quot;</div><div>+#include &lt;pthread.h&gt;</div><div>+#include &lt;sys/socket.h&gt;</div><div>+#include &lt;netinet/in.h&gt;</div><div>+#include &lt;fcntl.h&gt;</div><div>+#include &quot;cr_options.h&quot;</div><div>+</div><div>+static struct rimage *wait_for_image(struct wthread *wt)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *result;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (!strncmp(wt-&gt;path, RESTORE_FINISH, sizeof(RESTORE_FINISH))) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>finished = true;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>shutdown(local_req_fd, SHUT_RD);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>result = get_rimg_by_name(wt-&gt;snapshot_id, wt-&gt;path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (result != NULL &amp;&amp; result-&gt;size &gt; 0)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return result;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* The file does not exist and we do not expect new files */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (finished &amp;&amp; !is_receiving())</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* NOTE: at this point, when the thread wakes up, either the image is</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span> * already in memory or it will never come (the dump is finished).</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span> */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>sem_wait(&amp;(wt-&gt;wakeup_sem));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>result = get_rimg_by_name(wt-&gt;snapshot_id, wt-&gt;path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (result != NULL &amp;&amp; result-&gt;size &gt; 0)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return result;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>else</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+}</div><div>+</div><div>+/* The image cache creates a thread that calls this function. It waits for remote</div><div>+ * images from the image-cache.</div><div>+ */</div><div>+void *accept_remote_image_connections(void *port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int fd = *((int *) port);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_in cli_addr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>socklen_t clilen = sizeof(cli_addr);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char snapshot_id_buf[PATHLEN], path_buf[PATHLEN];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>uint64_t size, ret;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int flags, proxy_fd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>proxy_fd = accept(fd, (struct sockaddr *) &amp;cli_addr, &amp;clilen);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (proxy_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to accept remote image connection from image proxy&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>ret = read_remote_header(proxy_fd, snapshot_id_buf, path_buf, &amp;flags, &amp;size);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (ret &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to receive remote header from image proxy&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>/* This means that the no more images are coming. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>else if (!ret) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_info(&quot;Image Proxy connection closed.\n&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>finished = true;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>unlock_workers();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_info(&quot;Received %s request for %s:%s\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>flags == O_RDONLY ? &quot;read&quot; :</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>flags == O_APPEND ? &quot;append&quot; : &quot;write&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>path_buf, snapshot_id_buf);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>rimg = prepare_remote_image(path_buf, snapshot_id_buf, flags);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>prepare_recv_rimg();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (!size)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>ret = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>else</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>ret = recv_image(proxy_fd, rimg, size, flags, false);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (ret &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to receive %s:%s from image proxy&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>finalize_recv_rimg(NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>} else if (ret != size) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to receive %s:%s from image proxy (received %lu bytes, expected %lu bytes)&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;path, rimg-&gt;snapshot_id, ret, size);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>finalize_recv_rimg(NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>finalize_recv_rimg(rimg);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_info(&quot;Finished receiving %s:%s (received %lu bytes)\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>rimg-&gt;path, rimg-&gt;snapshot_id, ret);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+}</div><div>+</div><div>+int image_cache(bool background, char *local_cache_path, unsigned short cache_write_port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_t local_req_thr, remote_req_thr;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pr_info(&quot;Proxy to Cache Port %d, CRIU to Cache Path %s\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>cache_write_port, local_cache_path);</div><div>+</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (opts.ps_socket != -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>proxy_to_cache_fd = opts.ps_socket;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_info(&quot;Re-using ps socket %d\n&quot;, proxy_to_cache_fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>} else {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>proxy_to_cache_fd = setup_TCP_server_socket(cache_write_port);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (proxy_to_cache_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to open proxy to cache TCP socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>local_req_fd = setup_UNIX_server_socket(local_cache_path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (local_req_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open cache to proxy UNIX socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (init_daemon(background, wait_for_image)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to initialize daemon&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_create(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>&amp;remote_req_thr,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>NULL, accept_remote_image_connections,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>(void *) &amp;proxy_to_cache_fd)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to create remote requests thread&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_create(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>&amp;local_req_thr,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>NULL,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>accept_local_image_connections,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>(void *) &amp;local_req_fd)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to create local requests thread&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_join(remote_req_thr, NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_join(local_req_thr, NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>join_workers();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pr_info(&quot;Finished image cache.&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return 0;</div><div>+}</div><div>diff --git a/criu/img-proxy.c b/criu/img-proxy.c</div><div>new file mode 100644</div><div>index 0000000..cb46745</div><div>--- /dev/null</div><div>+++ b/criu/img-proxy.c</div><div>@@ -0,0 +1,89 @@</div><div>+#include &lt;unistd.h&gt;</div><div>+</div><div>+#include &quot;img-remote.h&quot;</div><div>+#include &quot;img-remote-proto.h&quot;</div><div>+#include &quot;criu-log.h&quot;</div><div>+#include &lt;pthread.h&gt;</div><div>+#include &lt;fcntl.h&gt;</div><div>+#include &lt;sys/socket.h&gt;</div><div>+#include &quot;cr_options.h&quot;</div><div>+</div><div>+static struct rimage *wait_for_image(struct wthread *wt)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return get_rimg_by_name(wt-&gt;snapshot_id, wt-&gt;path);</div><div>+}</div><div>+</div><div>+uint64_t forward_image(struct rimage *rimg)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>uint64_t ret;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int fd = proxy_to_cache_fd;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;(rimg-&gt;in_use));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (write_remote_header(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>fd, rimg-&gt;snapshot_id, rimg-&gt;path, O_APPEND, rimg-&gt;size) &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Error writing header for %s:%s&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ret = send_image(fd, rimg, O_APPEND, false);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (ret &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to send %s:%s to image cache&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>} else if (ret != rimg-&gt;size) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to send %s:%s to image proxy (sent %lu bytes, expected %lu bytes&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    rimg-&gt;path, rimg-&gt;snapshot_id, ret, rimg-&gt;size);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pr_info(&quot;Finished forwarding %s:%s (sent %lu bytes)\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    rimg-&gt;path, rimg-&gt;snapshot_id, rimg-&gt;size);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret;</div><div>+}</div><div>+</div><div>+int image_proxy(bool background, char *local_proxy_path, char *fwd_host, unsigned short fwd_port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_t local_req_thr;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pr_info(&quot;CRIU to Proxy Path: %s, Cache Address %s:%hu\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>local_proxy_path, fwd_host, fwd_port);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>local_req_fd = setup_UNIX_server_socket(local_proxy_path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (local_req_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open CRIU to proxy UNIX socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (opts.ps_socket != -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>proxy_to_cache_fd = opts.ps_socket;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_info(&quot;Re-using ps socket %d\n&quot;, proxy_to_cache_fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>} else {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>proxy_to_cache_fd = setup_TCP_client_socket(fwd_host, fwd_port);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (proxy_to_cache_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to open proxy to cache TCP socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (init_daemon(background, wait_for_image))</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_create(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    &amp;local_req_thr,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    NULL,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    accept_local_image_connections,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    (void *) &amp;local_req_fd)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to create local requests thread&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_join(local_req_thr, NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>join_workers();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pr_info(&quot;Finished image proxy.&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return 0;</div><div>+}</div><div>diff --git a/criu/img-remote-proto.c b/criu/img-remote-proto.c</div><div>new file mode 100644</div><div>index 0000000..2dcb4ad</div><div>--- /dev/null</div><div>+++ b/criu/img-remote-proto.c</div><div>@@ -0,0 +1,741 @@</div><div>+#include &lt;unistd.h&gt;</div><div>+#include &lt;stdlib.h&gt;</div><div>+</div><div>+#include &lt;semaphore.h&gt;</div><div>+#include &lt;sys/socket.h&gt;</div><div>+#include &lt;netinet/in.h&gt;</div><div>+#include &lt;netdb.h&gt;</div><div>+#include &quot;sys/un.h&quot;</div><div>+#include &lt;pthread.h&gt;</div><div>+#include &lt;fcntl.h&gt;</div><div>+#include &lt;sys/file.h&gt;</div><div>+</div><div>+#include &quot;img-remote-proto.h&quot;</div><div>+#include &quot;criu-log.h&quot;</div><div>+#include &quot;common/compiler.h&quot;</div><div>+</div><div>+#include &quot;protobuf.h&quot;</div><div>+#include &quot;images/remote-image.pb-c.h&quot;</div><div>+#include &quot;image.h&quot;</div><div>+</div><div>+LIST_HEAD(rimg_head);</div><div>+pthread_mutex_t rimg_lock;</div><div>+</div><div>+pthread_mutex_t proxy_to_cache_lock;</div><div>+</div><div>+LIST_HEAD(workers_head);</div><div>+pthread_mutex_t workers_lock;</div><div>+sem_t workers_semph;</div><div>+</div><div>+struct rimage * (*wait_for_image) (struct wthread *wt);</div><div>+</div><div>+bool finished = false;</div><div>+int writing = 0;</div><div>+int forwarding = 0;</div><div>+int proxy_to_cache_fd;</div><div>+int local_req_fd;</div><div>+</div><div>+struct rimage *get_rimg_by_name(const char *snapshot_id, const char *path)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg = NULL;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_for_each_entry(rimg, &amp;rimg_head, l) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (!strncmp(rimg-&gt;path, path, PATHLEN) &amp;&amp;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    !strncmp(rimg-&gt;snapshot_id, snapshot_id, PATHLEN)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return rimg;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return NULL;</div><div>+}</div><div>+</div><div>+static struct wthread *get_wt_by_name(const char *snapshot_id, const char *path)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wt = NULL;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_for_each_entry(wt, &amp;workers_head, l) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (!strncmp(wt-&gt;path, path, PATHLEN) &amp;&amp;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>   !strncmp(wt-&gt;snapshot_id, snapshot_id, PATHLEN)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pthread_mutex_unlock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return wt;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return NULL;</div><div>+}</div><div>+</div><div>+static int init_sync_structures(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_mutex_init(&amp;rimg_lock, NULL) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Remote image list mutex init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_mutex_init(&amp;proxy_to_cache_lock, NULL) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Remote image connection mutex init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_mutex_init(&amp;workers_lock, NULL) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Workers mutex init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sem_init(&amp;workers_semph, 0, 0) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Workers semaphore init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return 0;</div><div>+}</div><div>+</div><div>+void prepare_recv_rimg(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>writing++;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+}</div><div>+</div><div>+void finalize_recv_rimg(struct rimage *rimg)</div><div>+{</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (rimg)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>list_add_tail(&amp;(rimg-&gt;l), &amp;rimg_head);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>writing--;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* Wake thread waiting for this image. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (rimg) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>struct wthread *wt = get_wt_by_name(rimg-&gt;snapshot_id, rimg-&gt;path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (wt)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>sem_post(&amp;(wt-&gt;wakeup_sem));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+}</div><div>+</div><div>+bool is_receiving(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int ret;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ret = writing;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret &gt; 0;</div><div>+}</div><div>+</div><div>+static void prepare_fwd_rimg(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>forwarding++;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+}</div><div>+</div><div>+static void finalize_fwd_rimg(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>forwarding--;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+}</div><div>+</div><div>+static bool is_forwarding(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int ret;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ret = forwarding;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret &gt; 0;</div><div>+}</div><div>+</div><div>+/* This function is called when no more images are coming. Threads still waiting</div><div>+ * for images will be awaken to send a ENOENT (no such file) to the requester.</div><div>+ */</div><div>+void unlock_workers(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wt = NULL;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_for_each_entry(wt, &amp;workers_head, l)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>sem_post(&amp;(wt-&gt;wakeup_sem));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;workers_lock);</div><div>+}</div><div>+</div><div>+int init_daemon(bool background, struct rimage *(*wfi)(struct wthread*))</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (background) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (daemon(1, 0) == -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Can&#39;t run service server in the background&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>wait_for_image = wfi;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return init_sync_structures();</div><div>+}</div><div>+</div><div>+int setup_TCP_server_socket(int port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_in serv_addr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int sockopt = 1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int sockfd = socket(AF_INET, SOCK_STREAM, 0);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sockfd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>bzero((char *) &amp;serv_addr, sizeof(serv_addr));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>serv_addr.sin_family = AF_INET;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>serv_addr.sin_addr.s_addr = INADDR_ANY;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>serv_addr.sin_port = htons(port);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (setsockopt(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    sockfd, SOL_SOCKET, SO_REUSEADDR, &amp;sockopt, sizeof(sockopt)) == -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to set SO_REUSEADDR&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (bind(sockfd, (struct sockaddr *) &amp;serv_addr, sizeof(serv_addr)) &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to bind image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (listen(sockfd, DEFAULT_LISTEN)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to listen image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return sockfd;</div><div>+}</div><div>+</div><div>+int setup_TCP_client_socket(char *hostname, int port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int sockfd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_in serv_addr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct hostent *server;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>sockfd = socket(AF_INET, SOCK_STREAM, 0);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sockfd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open remote image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>server = gethostbyname(hostname);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (server == NULL) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to get host by name (%s)&quot;, hostname);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>bzero((char *) &amp;serv_addr, sizeof(serv_addr));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>serv_addr.sin_family = AF_INET;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>bcopy((char *) server-&gt;h_addr,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>      (char *) &amp;serv_addr.sin_addr.s_addr,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>      server-&gt;h_length);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>serv_addr.sin_port = htons(port);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (connect(sockfd, (struct sockaddr *) &amp;serv_addr, sizeof(serv_addr)) &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to connect to remote %s&quot;, hostname);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return sockfd;</div><div>+}</div><div>+</div><div>+int setup_UNIX_server_socket(char *path)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_un addr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int sockfd = socket(AF_UNIX, SOCK_STREAM, 0);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sockfd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>memset(&amp;addr, 0, sizeof(addr));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>addr.sun_family = AF_UNIX;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>unlink(path);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (bind(sockfd, (struct sockaddr *)&amp;addr, sizeof(addr)) == -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to bind image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (listen(sockfd, 50) == -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to listen image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return sockfd;</div><div>+}</div><div>+</div><div>+int setup_UNIX_client_socket(char *path)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_un addr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int sockfd = socket(AF_UNIX, SOCK_STREAM, 0);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sockfd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to open local image socket&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>memset(&amp;addr, 0, sizeof(addr));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>addr.sun_family = AF_UNIX;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>strncpy(addr.sun_path, path, sizeof(addr.sun_path)-1);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (connect(sockfd, (struct sockaddr *)&amp;addr, sizeof(addr)) &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to connect to local socket: %s&quot;, path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return sockfd;</div><div>+}</div><div>+</div><div>+uint64_t pb_write_obj(int fd, void *obj, int type)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct cr_img img;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>img._x.fd = fd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>bfd_setraw(&amp;img._x);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return pb_write_one(&amp;img, obj, type);</div><div>+}</div><div>+</div><div>+uint64_t pb_read_obj(int fd, void **pobj, int type)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct cr_img img;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>img._x.fd = fd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>bfd_setraw(&amp;img._x);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return do_pb_read_one(&amp;img, pobj, type, true);</div><div>+}</div><div>+</div><div>+uint64_t write_header(int fd, char *snapshot_id, char *path, int flags)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>LocalImageEntry li = LOCAL_IMAGE_ENTRY__INIT;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span><a href="http://li.name">li.name</a> = path;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>li.snapshot_id = snapshot_id;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>li.open_mode = flags;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return pb_write_obj(fd, &amp;li, PB_LOCAL_IMAGE);</div><div>+}</div><div>+</div><div>+uint64_t write_reply_header(int fd, int error)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>LocalImageReplyEntry lir = LOCAL_IMAGE_REPLY_ENTRY__INIT;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>lir.error = error;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return pb_write_obj(fd, &amp;lir, PB_LOCAL_IMAGE_REPLY);</div><div>+}</div><div>+</div><div>+uint64_t write_remote_header(int fd, char *snapshot_id, char *path, int flags, uint64_t size)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>RemoteImageEntry ri = REMOTE_IMAGE_ENTRY__INIT;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span><a href="http://ri.name">ri.name</a> = path;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ri.snapshot_id = snapshot_id;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ri.open_mode = flags;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>ri.size = size;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return pb_write_obj(fd, &amp;ri, PB_REMOTE_IMAGE);</div><div>+}</div><div>+</div><div>+uint64_t read_header(int fd, char *snapshot_id, char *path, int *flags)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>LocalImageEntry *li;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int ret = pb_read_obj(fd, (void **)&amp;li, PB_LOCAL_IMAGE);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (ret &gt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>strncpy(snapshot_id, li-&gt;snapshot_id, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>strncpy(path, li-&gt;name, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>*flags = li-&gt;open_mode;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>free(li);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret;</div><div>+}</div><div>+</div><div>+uint64_t read_reply_header(int fd, int *error)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>LocalImageReplyEntry *lir;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int ret = pb_read_obj(fd, (void **)&amp;lir, PB_LOCAL_IMAGE_REPLY);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (ret &gt; 0)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>*error = lir-&gt;error;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>free(lir);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret;</div><div>+}</div><div>+</div><div>+uint64_t read_remote_header(int fd, char *snapshot_id, char *path, int *flags, uint64_t *size)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>RemoteImageEntry *ri;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int ret = pb_read_obj(fd, (void **)&amp;ri, PB_REMOTE_IMAGE);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (ret &gt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>strncpy(snapshot_id, ri-&gt;snapshot_id, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>strncpy(path, ri-&gt;name, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>*flags = ri-&gt;open_mode;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>*size = ri-&gt;size;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>free(ri);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return ret;</div><div>+}</div><div>+</div><div>+static struct wthread *new_worker(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wt = malloc(sizeof(struct wthread));</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (!wt) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to allocate worker thread structure&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (sem_init(&amp;(wt-&gt;wakeup_sem), 0, 0) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Workers semaphore init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return wt;</div><div>+}</div><div>+</div><div>+static void add_worker(struct wthread *wt)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_add_tail(&amp;(wt-&gt;l), &amp;workers_head);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;workers_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>sem_post(&amp;workers_semph);</div><div>+}</div><div>+</div><div>+void join_workers(void)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wthread = NULL;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (! list_empty(&amp;workers_head)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>wthread = list_entry(workers_head.next, struct wthread, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pthread_join(wthread-&gt;tid, NULL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>list_del(&amp;(wthread-&gt;l));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>free(wthread);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+}</div><div>+</div><div>+static struct rimage *new_remote_image(char *path, char *snapshot_id)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg = malloc(sizeof(struct rimage));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rbuf *buf = malloc(sizeof(struct rbuf));</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (rimg == NULL) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to allocate remote_image structures&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (buf == NULL) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Unable to allocate remote_buffer structures&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>strncpy(rimg-&gt;path, path, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>strncpy(rimg-&gt;snapshot_id, snapshot_id, PATHLEN);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;size = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>buf-&gt;nbytes = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>INIT_LIST_HEAD(&amp;(rimg-&gt;buf_head));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_add_tail(&amp;(buf-&gt;l), &amp;(rimg-&gt;buf_head));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;curr_sent_buf = list_entry(rimg-&gt;buf_head.next, struct rbuf, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;curr_sent_bytes = 0;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (pthread_mutex_init(&amp;(rimg-&gt;in_use), NULL) != 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_perror(&quot;Remote image in_use mutex init failed&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return rimg;</div><div>+}</div><div>+</div><div>+/* Clears a remote image struct for reusing it. */</div><div>+static struct rimage *clear_remote_image(struct rimage *rimg)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;(rimg-&gt;in_use));</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (!list_is_singular(&amp;(rimg-&gt;buf_head))) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>struct rbuf *buf = list_entry(rimg-&gt;buf_head.prev, struct rbuf, l);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>list_del(rimg-&gt;buf_head.prev);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>free(buf);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_entry(rimg-&gt;buf_head.next, struct rbuf, l)-&gt;nbytes = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;size = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;curr_sent_buf = list_entry(rimg-&gt;buf_head.next, struct rbuf, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>rimg-&gt;curr_sent_bytes = 0;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return rimg;</div><div>+}</div><div>+</div><div>+struct rimage *prepare_remote_image(char *path, char *snapshot_id, int open_mode)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg = get_rimg_by_name(snapshot_id, path);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* There is no record of such image, create a new one. */</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (rimg == NULL)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return new_remote_image(path, snapshot_id);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_lock(&amp;rimg_lock);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>list_del(&amp;(rimg-&gt;l));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_unlock(&amp;rimg_lock);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* There is already an image record. Simply return it for appending. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (open_mode == O_APPEND)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return rimg;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* There is already an image record. Clear it for writing. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>else</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>return clear_remote_image(rimg);</div><div>+}</div><div>+</div><div>+void *process_local_read(struct wthread *wt)</div><div>+{</div><div>+    <span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg = NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>uint64_t ret;</div><div>+        /* TODO - split wait_for_image</div><div>+        * in cache - improve the parent stuf</div><div>+        * in proxy - do not wait for anything, return no file</div><div>+        */</div><div>+        rimg = wait_for_image(wt);</div><div>+        if (!rimg) {<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>    </div><div>+                pr_info(&quot;No image %s:%s.\n&quot;, wt-&gt;path, wt-&gt;snapshot_id);</div><div>+                if (write_reply_header(wt-&gt;fd, ENOENT) &lt; 0)</div><div>+                        pr_perror(&quot;Error writing reply header for unexisting image&quot;);</div><div>+                close(wt-&gt;fd);</div><div>+                return NULL;</div><div>+        } else {</div><div>+                if (write_reply_header(wt-&gt;fd, 0) &lt; 0) {</div><div>+                    pr_perror(&quot;Error writing reply header for %s:%s&quot;,</div><div>+                        wt-&gt;path, wt-&gt;snapshot_id);</div><div>+                    close(wt-&gt;fd);</div><div>+                    return NULL;</div><div>+                }</div><div>+        }</div><div>+</div><div>+        pthread_mutex_lock(&amp;(rimg-&gt;in_use));</div><div>+        ret = send_image(wt-&gt;fd, rimg, wt-&gt;flags, true);</div><div>+        if (ret &lt; 0)</div><div>+                pr_perror(&quot;Unable to send %s:%s to CRIU (sent %lu bytes)&quot;,</div><div>+                    rimg-&gt;path, rimg-&gt;snapshot_id, ret);</div><div>+        else</div><div>+                pr_info(&quot;Finished sending %s:%s to CRIU (sent %lu bytes)\n&quot;,</div><div>+                    rimg-&gt;path, rimg-&gt;snapshot_id, ret);</div><div>+        pthread_mutex_unlock(&amp;(rimg-&gt;in_use));</div><div>+        return NULL;</div><div>+}</div><div>+</div><div>+static void *process_local_image_connection(void *ptr)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wt = (struct wthread *) ptr;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rimage *rimg = NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>uint64_t ret;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* NOTE: the code inside this if is shared for both cache and proxy. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (wt-&gt;flags == O_RDONLY)</div><div>+            return process_local_read(wt);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* NOTE: IMAGE PROXY ONLY. The image cache receives write connections</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span> * through TCP (see accept_remote_image_connections).</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span> */</div><div>+        rimg = prepare_remote_image(wt-&gt;path, wt-&gt;snapshot_id, wt-&gt;flags);</div><div>+        ret = recv_image(wt-&gt;fd, rimg, 0, wt-&gt;flags, true);</div><div>+        if (ret &lt; 0) {</div><div>+                pr_perror(&quot;Unable to receive %s:%s to CRIU (received %lu bytes)&quot;,</div><div>+                    rimg-&gt;path, rimg-&gt;snapshot_id, ret);</div><div>+                finalize_recv_rimg(NULL);</div><div>+                return NULL;</div><div>+        }</div><div>+        finalize_recv_rimg(rimg);</div><div>+        pr_info(&quot;Finished receiving %s:%s (received %lu bytes)\n&quot;,</div><div>+                rimg-&gt;path, rimg-&gt;snapshot_id, ret);</div><div>+</div><div>+</div><div>+        if (!strncmp(rimg-&gt;path, DUMP_FINISH, sizeof(DUMP_FINISH))) {</div><div>+                finished = true;</div><div>+                shutdown(local_req_fd, SHUT_RD);</div><div>+        } else {</div><div>+                pthread_mutex_lock(&amp;proxy_to_cache_lock);</div><div>+                ret = forward_image(rimg);</div><div>+                pthread_mutex_unlock(&amp;proxy_to_cache_lock);                        </div><div>+        }</div><div>+</div><div>+        finalize_fwd_rimg();</div><div>+        if (ret &lt; 0) {</div><div>+            pr_perror(&quot;Unable to forward %s:%s to Image Cache&quot;,</div><div>+                    rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+</div><div>+            return NULL;</div><div>+        }</div><div>+</div><div>+        if (finished &amp;&amp; !is_forwarding() &amp;&amp; !is_receiving()) {</div><div>+            pr_info(&quot;Closing connection to Image Cache.\n&quot;);</div><div>+            close(proxy_to_cache_fd);</div><div>+            unlock_workers();</div><div>+        }</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>return NULL;</div><div>+}</div><div>+</div><div>+</div><div>+void *accept_local_image_connections(void *port)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int fd = *((int *) port);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int cli_fd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct sockaddr_in cli_addr;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>socklen_t clilen = sizeof(cli_addr);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_t tid;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct wthread *wt;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>cli_fd = accept(fd, (struct sockaddr *) &amp;cli_addr, &amp;clilen);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (cli_fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (!finished)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>pr_err(&quot;Unable to accept local image connection&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>close(cli_fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>wt = new_worker();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>wt-&gt;fd = cli_fd;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (read_header(wt-&gt;fd, wt-&gt;snapshot_id, wt-&gt;path, &amp;(wt-&gt;flags)) &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Error reading local image header&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>pr_info(&quot;Received %s request for %s:%s\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    wt-&gt;flags == O_RDONLY ? &quot;read&quot; :</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>wt-&gt;flags == O_APPEND ? &quot;append&quot; : &quot;write&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    wt-&gt;path, wt-&gt;snapshot_id);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>/* These function calls are used to avoid other threads from</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span> * thinking that there are no more images are coming.</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span> */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (wt-&gt;flags != O_RDONLY) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>prepare_recv_rimg();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>prepare_fwd_rimg();</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>/* We need to flock the last pid file to avoid stealing pids</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span> * from restore.</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span> */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>int fd = open_proc_rw(PROC_GEN, LAST_PID_PATH);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (fd &lt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Can&#39;t open %s&quot;, LAST_PID_PATH);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (flock(fd, LOCK_EX)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Can&#39;t lock %s&quot;, LAST_PID_PATH);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (pthread_create(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    &amp;tid, NULL, process_local_image_connection, (void *) wt)) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Unable to create worker thread&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (flock(fd, LOCK_UN))</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Can&#39;t unlock %s&quot;, LAST_PID_PATH);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>close(fd);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>wt-&gt;tid = tid;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>add_worker(wt);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+}</div><div>+</div><div>+/* Note: size is a limit on how much we want to read from the socket. Zero means</div><div>+ * read until the socket is closed.</div><div>+ */</div><div>+uint64_t recv_image(int fd, struct rimage *rimg, uint64_t size, int flags, bool close_fd)</div><div>+{</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rbuf *curr_buf = NULL;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int n;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (flags == O_APPEND)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>curr_buf = list_entry(rimg-&gt;buf_head.prev, struct rbuf, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>else</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>curr_buf = list_entry(rimg-&gt;buf_head.next, struct rbuf, l);</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>n = read(fd,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span> curr_buf-&gt;buffer + curr_buf-&gt;nbytes,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span> size ?</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>     min((int) (size - rimg-&gt;size), BUF_SIZE - curr_buf-&gt;nbytes) :</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>     BUF_SIZE - curr_buf-&gt;nbytes);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (n == 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (close_fd)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return rimg-&gt;size;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>} else if (n &gt; 0) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>curr_buf-&gt;nbytes += n;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>rimg-&gt;size += n;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (curr_buf-&gt;nbytes == BUF_SIZE) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>  struct rbuf *buf = malloc(sizeof(struct rbuf));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>if (buf == NULL) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                        </span>pr_perror(&quot;Unable to allocate remote_buffer structures&quot;);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                        </span>if (close_fd)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                                </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>buf-&gt;nbytes = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>list_add_tail(&amp;(buf-&gt;l), &amp;(rimg-&gt;buf_head));</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>curr_buf = buf;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (size &amp;&amp; rimg-&gt;size == size) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>if (close_fd)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                        </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>return rimg-&gt;size;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>} else {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Read on %s:%s socket failed&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (close_fd)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+}</div><div>+</div><div>+uint64_t send_image(int fd, struct rimage *rimg, int flags, bool close_fd)</div><div>+{</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int n, nblocks = 0;</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>if (flags != O_APPEND) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>rimg-&gt;curr_sent_buf = list_entry(rimg-&gt;buf_head.next, struct rbuf, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>rimg-&gt;curr_sent_bytes = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>while (1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>n = send(</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    fd,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    rimg-&gt;curr_sent_buf-&gt;buffer + rimg-&gt;curr_sent_bytes,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    min(BUF_SIZE, rimg-&gt;curr_sent_buf-&gt;nbytes) - rimg-&gt;curr_sent_bytes,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>    MSG_NOSIGNAL);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>if (n &gt; -1) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>rimg-&gt;curr_sent_bytes += n;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>if (rimg-&gt;curr_sent_bytes == BUF_SIZE) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;curr_sent_buf =</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>    list_entry(rimg-&gt;curr_sent_buf-&gt;l.next, struct rbuf, l);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>nblocks++;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;curr_sent_bytes = 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>} else if (rimg-&gt;curr_sent_bytes == rimg-&gt;curr_sent_buf-&gt;nbytes) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>if (close_fd)</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                        </span>close(fd);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>       return nblocks*BUF_SIZE + rimg-&gt;curr_sent_buf-&gt;nbytes;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>} else if (errno == EPIPE || errno == ECONNRESET) {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_warn(&quot;Connection for %s:%s was closed early than expected\n&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return 0;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>} else {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>pr_perror(&quot;Write on %s:%s socket failed&quot;,</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                                </span>rimg-&gt;path, rimg-&gt;snapshot_id);</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                        </span>return -1;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">                </span>}</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>}</div><div>+</div><div>+}</div><div>diff --git a/criu/include/img-remote-proto.h b/criu/include/img-remote-proto.h</div><div>new file mode 100644</div><div>index 0000000..51ee2cd</div><div>--- /dev/null</div><div>+++ b/criu/include/img-remote-proto.h</div><div>@@ -0,0 +1,86 @@</div><div>+#ifndef IMAGE_REMOTE_PVT_H</div><div>+#define<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>IMAGE_REMOTE_PVT_H</div><div>+</div><div>+#include &lt;stdbool.h&gt;</div><div>+#include &lt;stdint.h&gt;</div><div>+#include &quot;common/list.h&quot;</div><div>+#include &quot;img-remote.h&quot;</div><div>+#include &lt;pthread.h&gt;</div><div>+#include &lt;semaphore.h&gt;</div><div>+</div><div>+#define DEFAULT_LISTEN 50</div><div>+#define PAGESIZE 4096</div><div>+#define BUF_SIZE PAGESIZE</div><div>+</div><div>+struct rbuf {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char buffer[BUF_SIZE];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int nbytes; /* How many bytes are in the buffer. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct list_head l;</div><div>+};</div><div>+</div><div>+struct rimage {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char path[PATHLEN];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char snapshot_id[PATHLEN];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct list_head l;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct list_head buf_head;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* Used to track already sent buffers when the image is appended. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct rbuf *curr_sent_buf;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* Similar to the previous field. Number of bytes sent in &#39;curr_sent_buf&#39;. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int curr_sent_bytes;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>uint64_t size; /* number of bytes */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_mutex_t in_use; /* Only one operation at a time, per image. */</div><div>+};</div><div>+</div><div>+struct wthread {</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>pthread_t tid;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>struct list_head l;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* Client fd. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int fd;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* The path and snapshot_id identify the request handled by this thread. */</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char path[PATHLEN];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>char snapshot_id[PATHLEN];</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>int flags;</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>/* This semph is used to wake this thread if the image is in memory.*/</div><div>+<span class="gmail-Apple-tab-span" style="white-space:pre">        </span>sem_t wakeup_sem;</div><div>+};</div><div>+</div><div>+/* This variable is used to indicate when the dump is finished. */</div><div>+extern bool finished;</div><div>+/* This is the proxy to cache TCP socket FD. */</div><div>+extern int proxy_to_cache_fd;</div><div>+/* This the unix socket used to fulfill local requests. */</div><div>+extern int local_req_fd;</div><div>+</div><div>+int init_daemon(bool background, struct rimage *(*wfi)(struct wthread*));</div><div>+</div><div>+void join_workers(void);</div><div>+void unlock_workers(void);</div><div>+</div><div>+void prepare_recv_rimg(void);</div><div>+void finalize_recv_rimg(struct rimage *rimg);</div><div>+struct rimage *prepare_remote_image(char *path, char *namesapce, int flags);</div><div>+struct rimage *get_rimg_by_name(const char *snapshot_id, const char *path);</div><div>+bool is_receiving(void);</div><div>+</div><div>+void *accept_local_image_connections(void *ptr);</div><div>+void *accept_remote_image_connections(void *ptr);</div><div>+</div><div>+uint64_t forward_image(struct rimage *rimg);</div><div>+uint64_t send_image(int fd, struct rimage *rimg, int flags, bool image_check);</div><div>+uint64_t recv_image(int fd, struct rimage *rimg, uint64_t size, int flags, bool image_check);</div><div>+</div><div>+uint64_t pb_write_obj(int fd, void *obj, int type);</div><div>+uint64_t pb_read_obj(int fd, void **obj, int type);</div><div>+</div><div>+uint64_t write_header(int fd, char *snapshot_id, char *path, int open_mode);</div><div>+uint64_t read_header(int fd, char *snapshot_id, char *path, int *open_mode);</div><div>+uint64_t write_reply_header(int fd, int error);</div><div>+uint64_t read_reply_header(int fd, int *error);</div><div>+uint64_t read_remote_header(int fd, char *snapshot_id, char *path, int *open_mode, uint64_t *size);</div><div>+uint64_t write_remote_header(int fd, char *snapshot_id, char *path, int open_mode, uint64_t size);</div><div>+</div><div>+int setup_TCP_server_socket(int port);</div><div>+int setup_TCP_client_socket(char *hostname, int port);</div><div>+int setup_UNIX_client_socket(char *path);</div><div>+int setup_UNIX_server_socket(char *path);</div><div>+#endif</div></div><div><br></div></div>