[CRIU] [PATCH cr 2/9] tcp: unset TCP_REPAIR at the last moment after unlocking network (v2)

[Andrey Vagin]

TCP_REPAIR should be droppet when a network is unlocked.
A network should be unlocked at the last moment, because
after this moment restore must not failed, otherwise a state of
a tcp connection can be changed and a state of one side in our image
will be invalid.

v2: use xremalloc instead of mmap and remmap

Signed-off-by: Andrey Vagin <avagin at openvz.org>
---
 cr-restore.c       |    9 ++++++++-
 include/restorer.h |    3 +++
 include/sk-inet.h  |    6 ++++++
 restorer.c         |   20 ++++++++++++++++++--
 sk-tcp.c           |   50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 5 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/cr-restore.c b/cr-restore.c
index 0489a9c..d95c428 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -1305,7 +1305,8 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core, struct list_head *tgt_v
 					      restore_task_vma_len +
 					      restore_thread_vma_len +
 					      self_vmas_len + vmas_len +
-					      SHMEMS_SIZE + TASK_ENTRIES_SIZE);
+					      SHMEMS_SIZE + TASK_ENTRIES_SIZE +
+					      rst_tcp_socks_size);
 	if (exec_mem_hint == -1) {
 		pr_err("No suitable area for task_restore bootstrap (%ldK)\n",
 		       restore_task_vma_len + restore_thread_vma_len);
@@ -1381,6 +1382,12 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core, struct list_head *tgt_v
 	if (!task_args->tgt_vmas)
 		goto err;
 
+	mem += vmas_len;
+	if (rst_tcp_socks_remap(mem))
+		goto err;
+	task_args->rst_tcp_socks = mem;
+	task_args->rst_tcp_socks_size = rst_tcp_socks_size;
+
 	/*
 	 * Arguments for task restoration.
 	 */
diff --git a/include/restorer.h b/include/restorer.h
index 274459b..3b9fbcf 100644
--- a/include/restorer.h
+++ b/include/restorer.h
@@ -110,6 +110,9 @@ struct task_restore_core_args {
 	bool				has_futex;
 	u64				futex_rla;
 	u32				futex_rla_len;
+
+	int				*rst_tcp_socks;
+	int				rst_tcp_socks_size;
 } __aligned(sizeof(long));
 
 struct pt_regs {
diff --git a/include/sk-inet.h b/include/sk-inet.h
index 0fe550c..29c2d22 100644
--- a/include/sk-inet.h
+++ b/include/sk-inet.h
@@ -3,6 +3,9 @@
 
 #include <netinet/tcp.h>
 
+#include "sockets.h"
+#include "files.h"
+#include "list.h"
 #include "protobuf.h"
 #include "../protobuf/sk-inet.pb-c.h"
 
@@ -62,4 +65,7 @@ struct cr_options;
 void show_tcp_stream(int fd, struct cr_options *);
 
 int check_tcp_repair(void);
+
+extern int rst_tcp_socks_size;
+extern int rst_tcp_socks_remap(void *addr);
 #endif
diff --git a/restorer.c b/restorer.c
index 5cee0b1..d516bed 100644
--- a/restorer.c
+++ b/restorer.c
@@ -18,6 +18,7 @@
 #include "restorer-log.h"
 #include "util.h"
 #include "image.h"
+#include "sk-inet.h"
 
 #include "crtools.h"
 #include "lock.h"
@@ -281,6 +282,19 @@ static u64 restore_mapping(const VmaEntry *vma_entry)
 	return addr;
 }
 
+static void rst_tcp_socks_all(int *arr, int size)
+{
+	int i;
+
+	if (size == 0)
+		return;
+
+	for (i =0; arr[i] >= 0; i++)
+		tcp_repair_off(arr[i]);
+
+	sys_munmap(arr, size);
+}
+
 /*
  * The main routine to restore task via sigreturn.
  * This one is very special, we never return there
@@ -631,10 +645,12 @@ long __export_restore_task(struct task_restore_core_args *args)
 
 	futex_dec_and_wake(&args->task_entries->nr_in_progress);
 
-	sys_close(args->logfd);
-
 	futex_wait_while(&args->task_entries->start, CR_STATE_RESTORE_SIGCHLD);
 
+	rst_tcp_socks_all(args->rst_tcp_socks, args->rst_tcp_socks_size);
+
+	sys_close(args->logfd);
+
 	/*
 	 * The code that prepared the itimers makes shure the
 	 * code below doesn't fail due to bad timing values.
diff --git a/sk-tcp.c b/sk-tcp.c
index 4a9cf73..647d061 100644
--- a/sk-tcp.c
+++ b/sk-tcp.c
@@ -3,6 +3,8 @@
 #include <linux/sockios.h>
 #include <unistd.h>
 #include <stdlib.h>
+#include <sys/mman.h>
+#include <string.h>
 
 #include "crtools.h"
 #include "util.h"
@@ -473,6 +475,51 @@ err:
 	return -1;
 }
 
+/*
+ * rst_tcp_socks contains sockets in repair mode,
+ * which will be off in restorer before resuming.
+ */
+static int *rst_tcp_socks = NULL;
+static int rst_tcp_socks_num = 0;
+int rst_tcp_socks_size = 0;
+
+int rst_tcp_socks_remap(void *addr)
+{
+	void *ret;
+	if (!rst_tcp_socks) {
+		BUG_ON(rst_tcp_socks_size);
+		return 0;
+	}
+
+	rst_tcp_socks[rst_tcp_socks_num] = -1;
+
+	ret = mmap(addr, rst_tcp_socks_size, PROT_READ | PROT_WRITE,
+			MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0);
+
+	if (ret != addr) {
+		pr_perror("mmap() failed\n");
+		return -1;
+	}
+
+	memcpy(addr, rst_tcp_socks, rst_tcp_socks_size);
+
+	return 0;
+}
+
+static int rst_tcp_socks_add(int fd)
+{
+	/* + 2 = ( new one + guard (-1) ) */
+	if ((rst_tcp_socks_num + 2) * sizeof(int) > rst_tcp_socks_size) {
+		rst_tcp_socks_size += PAGE_SIZE;
+		rst_tcp_socks = xrealloc(rst_tcp_socks, rst_tcp_socks_size);
+		if (rst_tcp_socks == NULL)
+			return -1;
+	}
+
+	rst_tcp_socks[rst_tcp_socks_num++] = fd;
+	return 0;
+}
+
 int restore_one_tcp(int fd, struct inet_sk_info *ii)
 {
 	pr_info("Restoring TCP connection\n");
@@ -480,6 +527,9 @@ int restore_one_tcp(int fd, struct inet_sk_info *ii)
 	if (tcp_repair_on(fd))
 		return -1;
 
+	if (rst_tcp_socks_add(fd))
+		return -1;
+
 	nf_unlock_connection_info(ii);
 
 	if (restore_tcp_conn_state(fd, ii))
-- 
1.7.1