[CRIU] [PATCH 10/13] restore: collect signals from zombies (v2)

[Andrey Vagin]

Each zombie sends SIGCHLD to parent. crtools restores all pending
signals, so all other signals should be collected.

Here is a problems, that signals SIGCHLD can be merged, but crtools
should be sure, that all signals are collected.

For that a zombie locks a global zombie_lock, which is released by
parent.

This operation should be done between CR_STATE_RESTORE and
CR_STATE_RESTORE_SIGCHLD. Two new states CR_STATE_RESTORE_ZOMBIES
and CR_STATE_RESTORE_PREP_ZOMBIES were added.

CR_STATE_RESTORE_PREP_ZOMBIES is used for waiting when all tasks
installes a special sighandler.

CR_STATE_RESTORE_ZOMBIES is used for waiting all zombies.

v2: clean up
v3: rework synchronization

Signed-off-by: Andrey Vagin <avagin at openvz.org>
---
 cr-restore.c       | 28 ++++++++++++++++++++++++++--
 include/restorer.h |  4 ++++
 pie/restorer.c     |  9 +++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/cr-restore.c b/cr-restore.c
index bdba840..e4757bf 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -694,9 +694,12 @@ static int restore_one_zombie(int pid, int exit_code)
 	pr_info("Restoring zombie with %d code\n", exit_code);
 
 	if (task_entries != NULL) {
+		atomic_inc(&task_entries->nr_zombies);
+
 		restore_finish_stage(CR_STATE_RESTORE);
 		zombie_prepare_signals();
-		restore_finish_stage(CR_STATE_RESTORE_SIGCHLD);
+		restore_finish_stage(CR_STATE_RESTORE_PREP_ZOMBIES);
+		mutex_lock(&task_entries->zombie_lock);
 	}
 
 	if (exit_code & 0x7f) {
@@ -1122,8 +1125,11 @@ static inline int stage_participants(int next_stage)
 	case CR_STATE_RESTORE_PGID:
 		return task_entries->nr_tasks;
 	case CR_STATE_RESTORE:
-	case CR_STATE_RESTORE_SIGCHLD:
+	case CR_STATE_RESTORE_ZOMBIES:
+	case CR_STATE_RESTORE_PREP_ZOMBIES:
 		return task_entries->nr_threads;
+	case CR_STATE_RESTORE_SIGCHLD:
+		return task_entries->nr_threads - atomic_get(&task_entries->nr_zombies);
 	}
 
 	BUG();
@@ -1204,6 +1210,22 @@ static int restore_root_task(struct pstree_item *init, struct cr_options *opts)
 	if (ret < 0)
 		goto out;
 
+	pr_info("Prepare for collecting SIGCHLD signals from zombies\n");
+	ret = restore_switch_stage(CR_STATE_RESTORE_PREP_ZOMBIES);
+	if (ret < 0)
+		goto out;
+
+	pr_info("Collect SIGCHLD signals from zombies\n");
+	ret = restore_switch_stage(CR_STATE_RESTORE_ZOMBIES);
+	if (ret < 0)
+		goto out;
+
+	pr_info("Wait threads %d %d\n", task_entries->nr_threads, task_entries->nr_tasks);
+	futex_wait_while_gt(&task_entries->nr_in_progress, task_entries->nr_tasks);
+	ret = (int) futex_get(&task_entries->nr_in_progress);
+	if(ret < 0)
+		goto out;
+
 	pr_info("Wait until all tasks are restored\n");
 	ret = restore_switch_stage(CR_STATE_RESTORE_SIGCHLD);
 	if (ret < 0)
@@ -1255,6 +1277,8 @@ static int prepare_task_entries()
 	task_entries->nr_tasks = 0;
 	task_entries->nr_helpers = 0;
 	futex_set(&task_entries->start, CR_STATE_FORKING);
+	atomic_set(&task_entries->nr_zombies, 0);
+	mutex_init(&task_entries->zombie_lock);
 
 	return 0;
 }
diff --git a/include/restorer.h b/include/restorer.h
index 336e28d..a6f1868 100644
--- a/include/restorer.h
+++ b/include/restorer.h
@@ -162,14 +162,18 @@ enum {
 	CR_STATE_FORKING,
 	CR_STATE_RESTORE_PGID,
 	CR_STATE_RESTORE,
+	CR_STATE_RESTORE_PREP_ZOMBIES,
+	CR_STATE_RESTORE_ZOMBIES,
 	CR_STATE_RESTORE_SIGCHLD,
 	CR_STATE_COMPLETE
 };
 
 struct task_entries {
 	int nr_threads, nr_tasks, nr_helpers;
+	atomic_t nr_zombies;
 	futex_t nr_in_progress;
 	futex_t start;
+	mutex_t	zombie_lock;
 };
 
 static always_inline struct shmem_info *
diff --git a/pie/restorer.c b/pie/restorer.c
index 5530ac8..72929e2 100644
--- a/pie/restorer.c
+++ b/pie/restorer.c
@@ -45,6 +45,10 @@ extern void cr_restore_rt (void) asm ("__cr_restore_rt")
 
 static void sigzombie_handler(int signal, siginfo_t *siginfo, void *data)
 {
+	pr_debug("%ld: Collect a zombie with (pid %d, %d)\n",
+		sys_getpid(), siginfo->si_pid, siginfo->si_pid);
+	futex_dec_and_wake(&task_entries->nr_in_progress);
+	mutex_unlock(&task_entries->zombie_lock);
 }
 
 static void sigchld_handler(int signal, siginfo_t *siginfo, void *data)
@@ -219,6 +223,8 @@ long __export_restore_thread(struct thread_restore_args *args)
 	pr_info("%ld: Restored\n", sys_gettid());
 
 	restore_finish_stage(CR_STATE_RESTORE);
+	restore_finish_stage(CR_STATE_RESTORE_PREP_ZOMBIES);
+	restore_finish_stage(CR_STATE_RESTORE_ZOMBIES);
 
 	if (restore_signals(args->siginfo, args->siginfo_nr, 0))
 		goto core_restore_end;
@@ -741,6 +747,9 @@ long __export_restore_task(struct task_restore_core_args *args)
 	act.rt_sa_restorer = cr_restore_rt;
 	sys_sigaction(SIGCHLD, &act, NULL, sizeof(k_rtsigset_t));
 
+	restore_finish_stage(CR_STATE_RESTORE_PREP_ZOMBIES);
+	restore_finish_stage(CR_STATE_RESTORE_ZOMBIES);
+
 	sys_sigaction(SIGCHLD, &args->sigchld_act, NULL, sizeof(k_rtsigset_t));
 
 	ret = restore_signals(args->siginfo, args->siginfo_nr, 1);
-- 
1.7.11.7