[CRIU] [PATCH 21/24] parasite: restore parasite state via rt_sigreturn

Andrey Vagin avagin at openvz.org
Wed May 22 16:08:22 EDT 2013


This patch reduces a window, when a crtools can kill a dumped process,
because if a parasite in a deamon mode can restore the state of the
process, if crtools detached unexpectedly.

All threads are synchronized on the exit point from sys_rt_sigreturn,
for that crtools traces all syscalls.

Signed-off-by: Andrey Vagin <avagin at openvz.org>
---
 parasite-syscall.c | 158 +++++++++++++++++++++++++++++++++--------------------
 pie/parasite.c     |  50 +++++++++++++----
 2 files changed, 137 insertions(+), 71 deletions(-)

diff --git a/parasite-syscall.c b/parasite-syscall.c
index 32711c5..32af1b0 100644
--- a/parasite-syscall.c
+++ b/parasite-syscall.c
@@ -679,13 +679,14 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
 
 	for (i = 1; i < item->nr_threads; i++) {
 		pid_t tid = item->threads[i].real;
+		user_regs_struct_t *regs_orig = &ctl->threads[i].regs_orig;
 
 		ctl->threads[i].tid = tid;
 		ctl->nr_threads++;
 
 		args->real = tid;
-
-		ret = ptrace(PTRACE_GETREGS, tid, NULL, &ctl->threads[i].regs_orig);
+		args->sigframe = ctl->threads[i].rsigframe;
+		ret = ptrace(PTRACE_GETREGS, tid, NULL, regs_orig);
 		if (ret) {
 			pr_perror("Can't obtain registers (pid: %d)", tid);
 			goto err;
@@ -698,7 +699,7 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
 			goto err;
 		}
 
-		ret = get_task_regs(tid, ctl->threads[i].regs_orig, item->core[i]);
+		ret = get_task_regs(tid, *regs_orig, item->core[i]);
 		if (ret) {
 			pr_err("Can't obtain regs for thread %d\n", tid);
 			goto err;
@@ -716,79 +717,118 @@ err:
 	return -1 ;
 }
 
-int parasite_fini_threads_seized(struct parasite_ctl *ctl)
+static int parasite_fini_seized(struct parasite_ctl *ctl)
 {
-	struct parasite_init_args *args;
-	int ret = 0, i, status;
-
-	args = parasite_args(ctl, struct parasite_init_args);
+	int status, ret = 0, i, nr = 0, nr_dmnz = 0;
 
-	for (i = 1; i < ctl->nr_threads; i++) {
-		pid_t tid = ctl->threads[i].tid;
+	/* Start to trace syscalls for each thread */
+	for (i = 0; i < ctl->nr_threads; i++) {
+		pid_t pid = ctl->threads[i].tid;
 
 		if (!ctl->threads[i].daemonized)
 			break;
 
-		args->real = tid;
-		ret = parasite_execute_daemon_by_pid(PARASITE_CMD_FINI_THREAD, ctl, tid);
-		/*
-		 * Note the thread's fini() can be called even when not
-		 * all threads were init()'ed, say we're rolling back from
-		 * error happened while we were init()'ing some thread, thus
-		 * -ENOENT will be returned but we should continie for the
-		 * rest of threads set.
-		 *
-		 * Strictly speaking we always init() threads in sequence thus
-		 * we could simply break the loop once first -ENOENT returned
-		 * but I prefer to be on a safe side even if some future changes
-		 * would change the code logic.
-		 */
-		if (ret && ret != -ENOENT) {
-			pr_err("Can't fini thread in parasite %d\n", tid);
-			break;
-		} else if (ret == -ENOENT)
-			continue;
+		ptrace(PTRACE_INTERRUPT, pid, NULL, NULL);
 
-		pr_debug("Waiting for %d to trap\n", tid);
-		if (wait4(tid, &status, __WALL, NULL) != tid) {
-			pr_perror("Waited pid mismatch (pid: %d)", tid);
-			break;
+		pr_debug("Waiting for %d to trap\n", pid);
+		if (wait4(pid, &status, __WALL, NULL) != pid) {
+			pr_perror("Waited pid mismatch (pid: %d)", pid);
+			return -1;
 		}
 
-		pr_debug("Daemon %d exited trapping\n", tid);
+		pr_debug("Daemon %d exited trapping\n", pid);
 		if (!WIFSTOPPED(status)) {
-			pr_err("Task is still running (pid: %d)\n", tid);
-			break;
+			pr_err("Task is still running (pid: %d)\n", pid);
+			return -1;
 		}
 
-		if (ptrace(PTRACE_SETREGS, tid, NULL, &ctl->threads[i].regs_orig)) {
-			pr_perror("Can't restore registers (pid: %d)", tid);
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
 			return -1;
 		}
+
+		nr_dmnz++;
 	}
 
-	return ret;
-}
+	ret = __parasite_execute_daemon_by_pid(PARASITE_CMD_FINI, ctl, ctl->pid.real, false);
+	if (ret)
+		return -1;
 
-static int parasite_fini_seized(struct parasite_ctl *ctl)
-{
-	struct parasite_init_args *args;
-	int status, ret = 0;
+	/* Stop all threads on the enter point in sys_rt_sigreturn */
+	while (1) {
+		user_regs_struct_t regs;
+		pid_t pid;
 
-	args = parasite_args(ctl, struct parasite_init_args);
-	args->real = ctl->pid.real;
+		pid = wait4(-1, &status, __WALL, NULL);
+		if (pid < 0) {
+			pr_perror("wait4 failed");
+			return -1;
+		}
+
+		pr_debug("%d was trapped\n", pid);
+		if (!WIFSTOPPED(status)) {
+			pr_err("%d\n", status);
+			return -1;
+		}
+		ret = ptrace(PTRACE_GETREGS, pid, NULL, &regs);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
 
-	args->real = ctl->pid.real;
-	__parasite_execute_daemon_by_pid(PARASITE_CMD_FINI, ctl, ctl->pid.real, false);
+		pr_debug("%d is going to execute the syscall %lx\n", pid, regs.orig_ax);
+		if (regs.orig_ax == __NR_rt_sigreturn) {
+			nr++;
+			pr_debug("%d was stopped\n", pid);
+			if (nr == nr_dmnz)
+				break;
+			continue;
+		}
 
-	if (wait4(ctl->pid.real, &status, __WALL, NULL) != ctl->pid.real) {
-		pr_perror("Waited pid mismatch (pid: %d)", ctl->pid.real);
-		ret = -1;
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
 	}
 
-	if (!WIFSTOPPED(status)) {
-		pr_err("Task is still running (pid: %d)\n", ctl->pid.real);
-		ret = -1;
+	/* Stop all threads on the exit point from sys_rt_sigreturn */
+	for (i = 0; i < ctl->nr_threads; i++) {
+		pid_t pid = ctl->threads[i].tid;
+		user_regs_struct_t regs;
+
+		if (!ctl->threads[i].daemonized)
+			break;
+
+		ctl->threads[i].use_sig_blocked = false;
+
+		ret = ptrace(PTRACE_SYSCALL, pid, NULL, NULL);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
+
+		if (wait4(pid, &status, __WALL, NULL) != pid) {
+			pr_perror("wait4 failed");
+			return -1;
+		}
+
+		pr_debug("Trap %d\n", pid);
+		if (!WIFSTOPPED(status)) {
+			pr_err("%d\n", status);
+			return -1;
+		}
+		ret = ptrace(PTRACE_GETREGS, pid, NULL, &regs);
+		if (ret) {
+			pr_perror("ptrace");
+			return -1;
+		}
+
+		if (regs.orig_ax != __NR_rt_sigreturn) {
+			pr_debug("%d: syscall mismatch %lx (%x)\n",
+					pid, regs.orig_ax, __NR_rt_sigreturn);
+		}
 	}
 
 	return ret;
@@ -798,10 +838,8 @@ int parasite_cure_remote(struct parasite_ctl *ctl)
 {
 	int ret = 0;
 
-	if (ctl->parasite_ip) {
-		ret = parasite_fini_threads_seized(ctl);
+	if (ctl->parasite_ip)
 		parasite_fini_seized(ctl);
-	}
 
 	ctl->tsock = -1;
 
@@ -986,9 +1024,6 @@ struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item,
 
 		thread->sigframe = ctl->local_map + parasite_size + ctl->args_size + i * RESTORE_STACK_SIGFRAME;
 		thread->rsigframe = ctl->remote_map + parasite_size + ctl->args_size + i * RESTORE_STACK_SIGFRAME;
-
-		if (construct_sigframe(thread->sigframe, thread->rsigframe, item->core[i]))
-			goto err_restore;
 	}
 
 	pr_info("Putting parasite blob into %p->%p\n", ctl->local_map, ctl->remote_map);
@@ -1035,6 +1070,9 @@ struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item,
 				&thread->sig_blocked, sizeof(k_rtsigset_t));
 			item->core[i]->thread_core->has_blk_sigset = true;
 		}
+
+		if (construct_sigframe(thread->sigframe, thread->rsigframe, item->core[i]))
+			goto err_restore;
 	}
 
 	return ctl;
diff --git a/pie/parasite.c b/pie/parasite.c
index e2f70f8..762d863 100644
--- a/pie/parasite.c
+++ b/pie/parasite.c
@@ -13,7 +13,9 @@
 
 #include <string.h>
 
+#include "asm/types.h"
 #include "asm/parasite.h"
+#include "asm/restorer.h"
 
 static int tsock = -1;
 
@@ -260,9 +262,15 @@ static int init_thread(struct parasite_init_args *args)
 
 static int fini_thread(struct tid_state_s *s)
 {
-	if (s->use_sig_blocked)
-		return sys_sigprocmask(SIG_SETMASK, &s->sig_blocked,
-				       NULL, sizeof(k_rtsigset_t));
+	unsigned long new_sp;
+
+	new_sp = (long)s->sigframe + SIGFRAME_OFFSET;
+	pr_debug("%ld: new_sp=%lx ip %lx\n", sys_gettid(),
+		 new_sp, s->sigframe->uc.uc_mcontext.rip);
+
+	ARCH_RT_SIGRETURN(new_sp);
+
+	BUG();
 
 	return 0;
 }
@@ -434,13 +442,6 @@ static int parasite_cfg_log(struct parasite_log_args *args)
 	return ret;
 }
 
-static int fini(struct tid_state_s *s)
-{
-	log_set_fd(-1);
-
-	return fini_thread(s);
-}
-
 static int __parasite_daemon_reply_ack(unsigned int id, unsigned int cmd, int err)
 {
 	struct ctl_msg m;
@@ -546,6 +547,30 @@ static int __parasite_execute_thread(struct ctl_msg *m)
 	return s->ret;
 }
 
+static int fini(struct tid_state_s *s)
+{
+	unsigned long new_sp;
+	int i;
+
+	for (i = 1; i < next_tid_state; i++) {
+		struct ctl_msg m = {.cmd = PARASITE_CMD_FINI_THREAD, .id = tid_state[i].real};
+		__parasite_execute_thread(&m);
+	}
+
+	new_sp = (long)s->sigframe + SIGFRAME_OFFSET;
+	pr_debug("%ld: new_sp=%lx ip %lx\n", sys_gettid(),
+		  new_sp, s->sigframe->uc.uc_mcontext.rip);
+
+	sys_close(tsock);
+	log_set_fd(-1);
+
+	ARCH_RT_SIGRETURN(new_sp);
+
+	BUG();
+
+	return -1;
+}
+
 static unsigned long noinline __used
 __parasite_daemon_thread_leader(void *args, struct tid_state_s *s, unsigned long oldstack)
 {
@@ -556,7 +581,7 @@ __parasite_daemon_thread_leader(void *args, struct tid_state_s *s, unsigned long
 
 	/* Reply we're alive */
 	if (__parasite_daemon_reply_ack(s->real, PARASITE_CMD_DAEMONIZE, 0))
-		return oldstack;
+		goto out;
 
 	while (1) {
 		if (__parasite_daemon_wait_msg(&m))
@@ -613,6 +638,9 @@ __parasite_daemon_thread_leader(void *args, struct tid_state_s *s, unsigned long
 			break;
 	}
 
+out:
+	fini(&tid_state[0]);
+
 	return oldstack;
 }
 
-- 
1.8.2



More information about the CRIU mailing list