[CRIU] [PATCH 2/3] rst: Fix rst_tcp_sock memory management

[Pavel Emelyanov]

In current scheme we grow an array with realloc()-s then
memcpy() the result into rst_mem. I propose to get rid
or realloc-s (we already have objects for the data we
need to keep) and memcpy-s (and put objects directly
into rst_mem at the end).

Signed-off-by: Pavel Emelyanov <xemul at parallels.com>
---
 cr-restore.c      | 16 ++++------------
 include/sk-inet.h | 29 +++++++++++++++--------------
 sk-inet.c         |  6 +++---
 sk-tcp.c          | 33 +++++++++++++++++----------------
 4 files changed, 39 insertions(+), 45 deletions(-)

diff --git a/cr-restore.c b/cr-restore.c
index 67fde2a..f1e96af 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -2648,9 +2648,6 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	struct vma_area *vma;
 	unsigned long tgt_vmas;
 
-	void *tcp_socks_mem;
-	unsigned long tcp_socks;
-
 	void *timerfd_mem;
 	unsigned long timerfd_mem_cpos;
 
@@ -2718,17 +2715,12 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	}
 
 	/*
-	 * Copy tcp sockets fds to rst memory -- restorer will
-	 * turn repair off before going sigreturn
+	 * Get all the tcp sockets fds into rst memory -- restorer
+	 * will turn repair off before going sigreturn
 	 */
-
-	tcp_socks = rst_mem_cpos(RM_PRIVATE);
-	tcp_socks_mem = rst_mem_alloc(rst_tcp_socks_len(), RM_PRIVATE);
-	if (!tcp_socks_mem)
+	if (rst_tcp_socks_prep())
 		goto err_nv;
 
-	memcpy(tcp_socks_mem, rst_tcp_socks, rst_tcp_socks_len());
-
 	/*
 	 * Copy timerfd params for restorer args, we need to proceed
 	 * timer setting at the very late.
@@ -2887,7 +2879,7 @@ static int sigreturn_restore(pid_t pid, CoreEntry *core)
 	remap_array(posix_timers, posix_timers_nr, posix_timers_cpos);
 	remap_array(timerfd,	  rst_timerfd_nr, timerfd_mem_cpos);
 	remap_array(siginfo,	  siginfo_nr, siginfo_cpos);
-	remap_array(tcp_socks,	  rst_tcp_socks_nr, tcp_socks);
+	remap_array(tcp_socks,	  rst_tcp_socks_nr, rst_tcp_socks_cpos);
 	remap_array(rings,	  mm->n_aios, aio_rings);
 	remap_array(rlims,	  rlims_nr, rlims_cpos);
 	remap_array(helpers,	  n_helpers, helpers_pos);
diff --git a/include/sk-inet.h b/include/sk-inet.h
index 22553bc..1a071e0 100644
--- a/include/sk-inet.h
+++ b/include/sk-inet.h
@@ -40,25 +40,18 @@ struct inet_sk_info {
 	InetSkEntry *ie;
 	struct file_desc d;
 	struct inet_port *port;
+	/*
+	 * This is an fd by which the socket is opened.
+	 * It will be carried down to restorer code to
+	 * repair-off the socket at the very end.
+	 */
+	int sk_fd;
 	struct list_head rlist;
 };
 
 extern int inet_bind(int sk, struct inet_sk_info *);
 extern int inet_connect(int sk, struct inet_sk_info *);
 
-struct rst_tcp_sock {
-	int	sk;
-	bool	reuseaddr;
-};
-
-extern struct rst_tcp_sock *rst_tcp_socks;
-extern int rst_tcp_socks_nr;
-
-static inline unsigned long rst_tcp_socks_len(void)
-{
-	return rst_tcp_socks_nr * sizeof(struct rst_tcp_sock);
-}
-
 static inline void tcp_repair_off(int fd)
 {
 	int aux = 0, ret;
@@ -78,7 +71,15 @@ extern int restore_one_tcp(int sk, struct inet_sk_info *si);
 #define SK_EST_PARAM	"tcp-established"
 
 extern int check_tcp(void);
-extern int rst_tcp_socks_add(int fd, bool reuseaddr);
 extern mutex_t *inet_get_reuseaddr_lock(struct inet_sk_info *ii);
 
+int rst_tcp_socks_prep(void);
+extern unsigned long rst_tcp_socks_cpos;
+extern unsigned int rst_tcp_socks_nr;
+
+struct rst_tcp_sock {
+	int	sk;
+	bool	reuseaddr;
+};
+
 #endif /* __CR_SK_INET_H__ */
diff --git a/sk-inet.c b/sk-inet.c
index b961741..bce24e3 100644
--- a/sk-inet.c
+++ b/sk-inet.c
@@ -450,9 +450,9 @@ static int post_open_inet_sk(struct file_desc *d, int sk)
 	 * after unlocking connections.
 	 */
 	if (tcp_connection(ii->ie)) {
-		if (rst_tcp_socks_add(sk, ii->ie->opts->reuseaddr))
-			return -1;
-
+		pr_debug("Schedule %d socket for repair off\n", sk);
+		BUG_ON(ii->sk_fd != -1);
+		ii->sk_fd = sk;
 		return 0;
 	}
 
diff --git a/sk-tcp.c b/sk-tcp.c
index 85eaafa..2b8e278 100644
--- a/sk-tcp.c
+++ b/sk-tcp.c
@@ -21,6 +21,7 @@
 #include "config.h"
 #include "cr-show.h"
 #include "kerndat.h"
+#include "rst-malloc.h"
 
 #include "protobuf.h"
 #include "protobuf/tcp-stream.pb-c.h"
@@ -653,26 +654,25 @@ err:
 	return -1;
 }
 
-/*
- * rst_tcp_socks contains sockets in repair mode,
- * which will be off in restorer before resuming.
- */
-struct rst_tcp_sock *rst_tcp_socks = NULL;
-int rst_tcp_socks_nr = 0;
+unsigned long rst_tcp_socks_cpos;
+unsigned int rst_tcp_socks_nr = 0;
 
-int rst_tcp_socks_add(int fd, bool reuseaddr)
+int rst_tcp_socks_prep(void)
 {
-	struct rst_tcp_sock *cur;
+	struct inet_sk_info *ii;
 
-	rst_tcp_socks_nr++;
-	rst_tcp_socks = xrealloc(rst_tcp_socks, rst_tcp_socks_len());
-	if (!rst_tcp_socks)
-		return -1;
+	rst_tcp_socks_cpos = rst_mem_cpos(RM_PRIVATE);
+	list_for_each_entry(ii, &rst_tcp_repair_sockets, rlist) {
+		struct rst_tcp_sock *rs;
 
-	pr_debug("Schedule %d socket for repair off\n", fd);
-	cur = &rst_tcp_socks[rst_tcp_socks_nr - 1];
-	cur->sk = fd;
-	cur->reuseaddr = reuseaddr;
+		rs = rst_mem_alloc(sizeof(*rs), RM_PRIVATE);
+		if (!rs)
+			return -1;
+
+		rs->sk = ii->sk_fd;
+		rs->reuseaddr = ii->ie->opts->reuseaddr;
+		rst_tcp_socks_nr++;
+	}
 
 	return 0;
 }
@@ -693,6 +693,7 @@ int restore_one_tcp(int fd, struct inet_sk_info *ii)
 void tcp_locked_conn_add(struct inet_sk_info *ii)
 {
 	list_add_tail(&ii->rlist, &rst_tcp_repair_sockets);
+	ii->sk_fd = -1;
 }
 
 void rst_unlock_tcp_connections(void)
-- 
1.9.3