[PATCH] inet: tcp -- Find size of max memory allowed to restore TCP data

[Cyrill Gorcunov]

The maximal size which may be used in the kernel for sending TCP data
on restore is varies depending on how many memory installed on the
system, moreover the memory allocated for "read queue" is bigger than
used for "write queue". Thus when we checkpointed a big slab of data
we need to figure out which size is allowed for sending data on restore.

For this we read /proc/sys/net/ipv4/tcp_[wmem|rmem] on restore and calculate
the size needed, then we simply chop data to segements and send it
in a loop.

Typical output on restore is something like

 | (00.013001)  30110: TCP queue memory limits are 2097152:3145728

https://bugzilla.openvz.org/show_bug.cgi?id=2751

Reported-by: Andrey Vagin <avagin at openvz.org>
Signed-off-by: Cyrill Gorcunov <gorcunov at openvz.org>
---
 cr-restore.c      |  8 ++++++++
 include/sk-inet.h |  2 ++
 sk-tcp.c          | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
 3 files changed, 66 insertions(+), 5 deletions(-)

diff --git a/cr-restore.c b/cr-restore.c
index ddee815..de5caca 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -1610,6 +1610,14 @@ int cr_restore_tasks(void)
 	if (crtools_prepare_shared() < 0)
 		return -1;
 
+	/*
+	 * Read TCP sysctls before anything else,
+	 * since the limits we're interested in are
+	 * not available inside namespaces.
+	 */
+	if (tcp_read_sysctl_limits())
+		return -1;
+
 	return restore_root_task(root_item);
 }
 
diff --git a/include/sk-inet.h b/include/sk-inet.h
index a3dff73..030c15a 100644
--- a/include/sk-inet.h
+++ b/include/sk-inet.h
@@ -79,4 +79,6 @@ int restore_one_tcp(int sk, struct inet_sk_info *si);
 int check_tcp(void);
 extern int rst_tcp_socks_add(int fd, bool reuseaddr);
 
+extern int tcp_read_sysctl_limits(void);
+
 #endif /* __CR_SK_INET_H__ */
diff --git a/sk-tcp.c b/sk-tcp.c
index 0d4fcfc..26ca1ac 100644
--- a/sk-tcp.c
+++ b/sk-tcp.c
@@ -21,6 +21,7 @@
 #include "xmalloc.h"
 #include "config.h"
 #include "cr-show.h"
+#include "sysctl.h"
 
 #include "protobuf.h"
 #include "protobuf/tcp-stream.pb-c.h"
@@ -57,6 +58,47 @@ enum {
 static LIST_HEAD(cpt_tcp_repair_sockets);
 static LIST_HEAD(rst_tcp_repair_sockets);
 
+/*
+ * Strictly speaking, if there is a machine with huge amount
+ * of memory, we're allowed to send up to 4M and read up to
+ * 6M of tcp data at once. But we will figure out precise size
+ * of a limit a bit later when restore starts.
+ *
+ * Meanwhile set it up to 2M and 3M, which is safe enough to
+ * proceed without errors.
+ */
+static int max_wshare = 2U << 20;
+static int max_rshare = 3U << 20;
+
+int tcp_read_sysctl_limits(void)
+{
+	u32 vect[2][3] = { };
+	int ret;
+
+	struct sysctl_req req[] = {
+		{ "net/ipv4/tcp_wmem", &vect[0], CTL_U32A(ARRAY_SIZE(vect[0])) },
+		{ "net/ipv4/tcp_rmem", &vect[1], CTL_U32A(ARRAY_SIZE(vect[1])) },
+		{ },
+	};
+
+	/*
+	 * Lets figure out which exactly amount of memory is
+	 * availabe for send/read queues on restore.
+	 */
+	ret = sysctl_op(req, CTL_READ);
+	if (ret)
+		return ret;
+
+	max_wshare = min(max_wshare, (int)vect[0][2]);
+	max_rshare = min(max_rshare, (int)vect[1][2]);
+
+	if (max_wshare < 128 || max_rshare < 128)
+		pr_warn("The memory limits for TCP queues are suspiciously small\n");
+
+	pr_debug("TCP queue memory limits are %d:%d\n", max_wshare, max_rshare);
+	return 0;
+}
+
 static int tcp_repair_on(int fd)
 {
 	int ret, aux = 1;
@@ -444,6 +486,7 @@ static int restore_tcp_seqs(int sk, TcpStreamEntry *tse)
 static int send_tcp_queue(int sk, int queue, u32 len, int imgfd)
 {
 	int ret, err = -1;
+	int off, max;
 	char *buf;
 
 	pr_debug("\tRestoring TCP %d queue data %u bytes\n", queue, len);
@@ -460,11 +503,19 @@ static int send_tcp_queue(int sk, int queue, u32 len, int imgfd)
 	if (read_img_buf(imgfd, buf, len) < 0)
 		goto err;
 
-	ret = send(sk, buf, len, 0);
-	if (ret != len) {
-		pr_perror("Can't restore %d queue data (%d), want %d",
-				queue, ret, len);
-		goto err;
+	max = (queue == TCP_SEND_QUEUE) ? max_wshare : max_rshare;
+	off = 0;
+	while (len) {
+		int chunk = len > max ? max : len;
+
+		ret = send(sk, buf + off, chunk, 0);
+		if (ret != chunk) {
+			pr_perror("Can't restore %d queue data (%d), want (%d:%d)",
+				  queue, ret, chunk, len);
+			goto err;
+		}
+		off += chunk;
+		len -= chunk;
 	}
 
 	err = 0;
-- 
1.8.3.1


--Md/poaVZ8hnGTzuv--