[PATCH] parasite: Switch parasite to daemon mode
Cyrill Gorcunov
gorcunov at openvz.org
Thu Feb 28 07:19:19 EST 2013
Parasite daemon mode it quite tricky. One may consider it as consisting
of two parts
- daemon mode for thread leader
- daemon mode for regular threads
Thread leader daemon
--------------------
On initialization we pass a control socket to parasite
thread leader daemon code via old ptrace interface.
Once socket is created, the thread leader parasite
code switched to daemon mode and start listening for
commands received on control socket.
Every command consists of 4 fields
- @id -- command recipient (basically it's global pid seen in main crtools module)
- @cmd -- command to handle
- @ack -- to reply once handled
- @err -- error code to return after handling the command
The thread leader daemon grabs command and if @id belongs to one
of the regular thread -- the command passed to thread daemon to
handle. Once the command is handled, thread leader daemon replies
upward to crtools module the result, via control socket message.
Thread daemon
-------------
On initialization thread daemon starts waiting a command on futex.
The futex is triggered by thread leader daemon, noone else have
access to poke thread daemon. Once command is received and handled,
the result reported back to thread leader daemon.
Signed-off-by: Cyrill Gorcunov <gorcunov at openvz.org>
---
parasite-syscall.c | 178 ++++++++++++++++++++++++++++++++++----
pie/parasite.c | 250 +++++++++++++++++++++++++++++++++++++++++++++--------
2 files changed, 376 insertions(+), 52 deletions(-)
diff --git a/parasite-syscall.c b/parasite-syscall.c
index 39f06c4..88aad14 100644
--- a/parasite-syscall.c
+++ b/parasite-syscall.c
@@ -233,6 +233,89 @@ static int parasite_execute_trap(unsigned int cmd, struct parasite_ctl *ctl)
return parasite_execute_trap_by_pid(cmd, ctl, ctl->pid);
}
+static int __parasite_send_cmd(int ctlsock, struct ctl_msg *m)
+{
+ int ret;
+
+ ret = send(ctlsock, m, sizeof(*m), 0);
+ if (ret == -1) {
+ pr_perror("Failed to send command %d to daemon %d\n", m->cmd, m->id);
+ return -1;
+ } else if (ret != sizeof(*m)) {
+ pr_err("Message to daemon is trimmed (%d/%d)\n",
+ (int)sizeof(*m), ret);
+ return -1;
+ }
+
+ pr_debug("Sent msg to daemon %d %d %d %d\n", m->id, m->cmd, m->ack, m->err);
+ return 0;
+}
+
+static int parasite_wait_ack(int ctlsock, pid_t pid, unsigned int cmd, struct ctl_msg *m)
+{
+ int ret;
+
+ pr_debug("Wait for ack %d-%d on daemon socket\n", pid, cmd);
+
+ while (1) {
+ memzero(m, sizeof(*m));
+
+ ret = recv(ctlsock, m, sizeof(*m), MSG_WAITALL);
+ if (ret == -1) {
+ pr_perror("Failed to read ack from %d", pid);
+ return -1;
+ } else if (ret != sizeof(*m)) {
+ pr_err("Message reply from daemon is trimmed (%d/%d)\n",
+ (int)sizeof(*m), ret);
+ return -1;
+ }
+ pr_debug("Fetched ack: %d %d %d %d\n",
+ m->id, m->cmd, m->ack, m->err);
+
+ if (m->id != pid || m->cmd != cmd || m->ack != cmd) {
+ pr_err("Communication error, this is not "
+ "the ack we expected\n");
+ return -1;
+ }
+ return 0;
+ }
+
+ return -1;
+}
+
+static int __parasite_execute_daemon_by_pid(unsigned int cmd, struct parasite_ctl *ctl,
+ pid_t pid, bool wait_ack)
+{
+ struct ctl_msg m;
+
+ m = ctl_msg_cmd(pid, cmd);
+ if (__parasite_send_cmd(ctl->ctlsock, &m))
+ return -1;
+
+ if (wait_ack) {
+ if (parasite_wait_ack(ctl->ctlsock, pid, cmd, &m))
+ return -1;
+
+ if (m.err != 0) {
+ pr_err("Command %d for daemon %d failed with %d\n",
+ cmd, pid, m.err);
+ return -1;
+ }
+ }
+
+ return 0;
+}
+
+static int parasite_execute_daemon_by_pid(unsigned int cmd, struct parasite_ctl *ctl, pid_t pid)
+{
+ return __parasite_execute_daemon_by_pid(cmd, ctl, pid, true);
+}
+
+static int parasite_execute_daemon(unsigned int cmd, struct parasite_ctl *ctl)
+{
+ return parasite_execute_daemon_by_pid(cmd, ctl, ctl->pid);
+}
+
static int munmap_seized(struct parasite_ctl *ctl, void *addr, size_t length)
{
unsigned long x;
@@ -327,6 +410,39 @@ err:
return -1;
}
+static int parasite_daemonize(struct parasite_ctl *ctl, pid_t pid)
+{
+ user_regs_struct_t regs = ctl->regs_orig;
+ struct ctl_msg m = { };
+
+ *ctl->addr_cmd = PARASITE_CMD_DAEMONIZE;
+ parasite_setup_regs(ctl->parasite_ip, ®s);
+
+ if (ptrace(PTRACE_SETREGS, pid, NULL, ®s)) {
+ pr_perror("Can't set registers (pid: %d)", pid);
+ goto err;
+ }
+
+ if (ptrace(PTRACE_CONT, pid, NULL, NULL)) {
+ pr_perror("Can't continue (pid: %d)\n", pid);
+ goto err;
+ }
+
+ pr_info("Wait for parasite being daemonized...\n");
+
+ if (parasite_wait_ack(ctl->ctlsock, pid, PARASITE_CMD_DAEMONIZE, &m)) {
+ pr_err("Can't switch parasite %d to daemon mode %d\n",
+ pid, m.err);
+ goto err;
+ }
+
+ pr_info("Parasite %d has been switched to daemon mode\n", pid);
+ return 0;
+
+err:
+ return -1;
+}
+
static int parasite_init(struct parasite_ctl *ctl, pid_t pid, int nr_threads)
{
struct parasite_init_args *args;
@@ -392,7 +508,7 @@ int parasite_dump_thread_seized(struct parasite_ctl *ctl, struct pid *tid,
args = parasite_args(ctl, struct parasite_dump_thread);
args->id = tid->real;
- ret = parasite_execute_trap_by_pid(PARASITE_CMD_DUMP_THREAD, ctl, tid->real);
+ ret = parasite_execute_daemon_by_pid(PARASITE_CMD_DUMP_THREAD, ctl, tid->real);
memcpy(&core->thread_core->blk_sigset, &args->blocked, sizeof(args->blocked));
CORE_THREAD_ARCH_INFO(core)->clear_tid_addr = encode_pointer(args->tid_addr);
@@ -410,7 +526,7 @@ int parasite_dump_sigacts_seized(struct parasite_ctl *ctl, struct cr_fdset *cr_f
args = parasite_args(ctl, struct parasite_dump_sa_args);
- ret = parasite_execute_trap(PARASITE_CMD_DUMP_SIGACTS, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_DUMP_SIGACTS, ctl);
if (ret < 0)
return ret;
@@ -453,7 +569,7 @@ int parasite_dump_itimers_seized(struct parasite_ctl *ctl, struct cr_fdset *cr_f
args = parasite_args(ctl, struct parasite_dump_itimers_args);
- ret = parasite_execute_trap(PARASITE_CMD_DUMP_ITIMERS, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_DUMP_ITIMERS, ctl);
if (ret < 0)
return ret;
@@ -473,7 +589,7 @@ int parasite_dump_misc_seized(struct parasite_ctl *ctl, struct parasite_dump_mis
struct parasite_dump_misc *ma;
ma = parasite_args(ctl, struct parasite_dump_misc);
- if (parasite_execute_trap(PARASITE_CMD_DUMP_MISC, ctl) < 0)
+ if (parasite_execute_daemon(PARASITE_CMD_DUMP_MISC, ctl) < 0)
return -1;
*misc = *ma;
@@ -487,7 +603,7 @@ struct parasite_tty_args *parasite_dump_tty(struct parasite_ctl *ctl, int fd)
p = parasite_args(ctl, struct parasite_tty_args);
p->fd = fd;
- if (parasite_execute_trap(PARASITE_CMD_DUMP_TTY, ctl) < 0)
+ if (parasite_execute_daemon(PARASITE_CMD_DUMP_TTY, ctl) < 0)
return NULL;
return p;
@@ -498,7 +614,7 @@ int parasite_dump_creds(struct parasite_ctl *ctl, CredsEntry *ce)
struct parasite_dump_creds *pc;
pc = parasite_args(ctl, struct parasite_dump_creds);
- if (parasite_execute_trap(PARASITE_CMD_DUMP_CREDS, ctl) < 0)
+ if (parasite_execute_daemon(PARASITE_CMD_DUMP_CREDS, ctl) < 0)
return -1;
ce->secbits = pc->secbits;
@@ -535,7 +651,7 @@ int parasite_dump_pages_seized(struct parasite_ctl *ctl, struct list_head *vma_a
if (ret < 0)
goto out;
- ret = parasite_execute_trap(PARASITE_CMD_DUMPPAGES_INIT, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_DUMPPAGES_INIT, ctl);
if (ret < 0) {
pr_err("Dumping pages failed with %i\n", ret);
goto out;
@@ -573,7 +689,7 @@ int parasite_dump_pages_seized(struct parasite_ctl *ctl, struct list_head *vma_a
if (vma_area->vma.end > TASK_SIZE)
continue;
- ret = parasite_execute_trap(PARASITE_CMD_DUMPPAGES, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_DUMPPAGES, ctl);
if (ret) {
pr_err("Dumping pages failed with %d\n", ret);
goto out_fini;
@@ -596,7 +712,7 @@ int parasite_dump_pages_seized(struct parasite_ctl *ctl, struct list_head *vma_a
ret = 0;
out_fini:
- parasite_execute_trap(PARASITE_CMD_DUMPPAGES_FINI, ctl);
+ parasite_execute_daemon(PARASITE_CMD_DUMPPAGES_FINI, ctl);
out:
pr_info("----------------------------------------\n");
@@ -613,7 +729,7 @@ int parasite_drain_fds_seized(struct parasite_ctl *ctl,
args = parasite_args_s(ctl, size);
memcpy(args, dfds, size);
- ret = parasite_execute_trap(PARASITE_CMD_DRAIN_FDS, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_DRAIN_FDS, ctl);
if (ret) {
pr_err("Parasite failed to drain descriptors\n");
goto err;
@@ -633,7 +749,7 @@ int parasite_get_proc_fd_seized(struct parasite_ctl *ctl)
{
int ret = -1, fd;
- ret = parasite_execute_trap(PARASITE_CMD_GET_PROC_FD, ctl);
+ ret = parasite_execute_daemon(PARASITE_CMD_GET_PROC_FD, ctl);
if (ret) {
pr_err("Parasite failed to get proc fd\n");
return ret;
@@ -668,6 +784,9 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
item->threads[i].real);
break;
}
+
+ if (parasite_daemonize(ctl, item->threads[i].real))
+ break;
}
return ret;
@@ -676,7 +795,7 @@ int parasite_init_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
int parasite_fini_threads_seized(struct parasite_ctl *ctl, struct pstree_item *item)
{
struct parasite_init_args *args;
- int ret = 0, i;
+ int ret = 0, i, status;
args = parasite_args(ctl, struct parasite_init_args);
memzero(args, sizeof(*args));
@@ -686,8 +805,8 @@ int parasite_fini_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
continue;
args->id = item->threads[i].real;
- ret = parasite_execute_trap_by_pid(PARASITE_CMD_FINI_THREAD, ctl,
- item->threads[i].real);
+ ret = parasite_execute_daemon_by_pid(PARASITE_CMD_FINI_THREAD, ctl,
+ item->threads[i].real);
/*
* Note the thread's fini() can be called even when not
* all threads were init()'ed, say we're rolling back from
@@ -704,6 +823,19 @@ int parasite_fini_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
pr_err("Can't fini thread in parasite %d\n",
item->threads[i].real);
break;
+ } else if (ret == -ENOENT)
+ continue;
+
+ pr_debug("Waiting for %d to trap\n", item->threads[i].real);
+ if (wait4(item->threads[i].real, &status, __WALL, NULL) != item->threads[i].real) {
+ pr_perror("Waited pid mismatch (pid: %d)", item->threads[i].real);
+ break;
+ }
+
+ pr_debug("Daemon %d exited trapping\n", item->threads[i].real);
+ if (!WIFSTOPPED(status)) {
+ pr_err("Task is still running (pid: %d)\n", item->threads[i].real);
+ break;
}
}
@@ -713,12 +845,25 @@ int parasite_fini_threads_seized(struct parasite_ctl *ctl, struct pstree_item *i
static int parasite_fini_seized(struct parasite_ctl *ctl)
{
struct parasite_init_args *args;
+ int status, ret = 0;
args = parasite_args(ctl, struct parasite_init_args);
memzero(args, sizeof(*args));
args->id = ctl->pid;
- return parasite_execute_trap(PARASITE_CMD_FINI, ctl);
+ __parasite_execute_daemon_by_pid(PARASITE_CMD_FINI, ctl, ctl->pid, false);
+
+ if (wait4(ctl->pid, &status, __WALL, NULL) != ctl->pid) {
+ pr_perror("Waited pid mismatch (pid: %d)", ctl->pid);
+ ret = -1;
+ }
+
+ if (!WIFSTOPPED(status)) {
+ pr_err("Task is still running (pid: %d)\n", ctl->pid);
+ ret = -1;
+ }
+
+ return ret;
}
int parasite_cure_seized(struct parasite_ctl *ctl, struct pstree_item *item)
@@ -890,6 +1035,9 @@ struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item,
goto err_restore;
}
+ if (parasite_daemonize(ctl, pid))
+ goto err_restore;
+
ret = parasite_init_threads_seized(ctl, item);
if (ret)
goto err_restore;
diff --git a/pie/parasite.c b/pie/parasite.c
index e6246b3..f34a22e 100644
--- a/pie/parasite.c
+++ b/pie/parasite.c
@@ -437,11 +437,8 @@ static int init_thread(struct parasite_init_args *args)
return ret;
}
-static int fini_thread(struct parasite_init_args *args)
+static int fini_thread(struct tid_state_s *s)
{
- struct tid_state_s *s;
-
- s = find_thread_state(args->id);
if (!s)
return -ENOENT;
@@ -652,18 +649,221 @@ static int parasite_cfg_log(struct parasite_log_args *args)
return ret;
}
-static int fini(struct parasite_init_args *args)
+static int fini(struct tid_state_s *s)
{
+ brk_fini();
+ log_set_fd(-1);
+ sys_close(tsock);
+
+ return fini_thread(s);
+}
+
+static int __parasite_daemon_reply_ack(unsigned int id, unsigned int cmd, int err)
+{
+ struct ctl_msg m;
int ret;
- ret = fini_thread(args);
+ m = ctl_msg_ack(id, cmd, err);
+ ret = sys_sendto(ctlsock, &m, sizeof(m), 0, NULL, 0);
+ if (ret != sizeof(m)) {
+ pr_err("Sent only %d bytes while %d expected\n",
+ ret, (int)sizeof(m));
+ return -1;
+ }
- sys_munmap(tid_state, TID_STATE_SIZE(nr_tid_state));
- log_set_fd(-1);
- sys_close(tsock);
- brk_fini();
+ pr_debug("__sent ack msg: %d %d %d %d\n",
+ m.id, m.cmd, m.ack, m.err);
- return ret;
+ return 0;
+}
+
+static int __parasite_daemon_wait_msg(struct ctl_msg *m)
+{
+ int ret;
+
+ pr_debug("Daemon wais for command\n");
+
+ while (1) {
+ *m = (struct ctl_msg){ };
+ ret = sys_recvfrom(ctlsock, m, sizeof(*m), MSG_WAITALL, NULL, 0);
+ if (ret != sizeof(*m)) {
+ pr_err("Trimmed message received (%d/%d)\n",
+ (int)sizeof(*m), ret);
+ return 0;
+ }
+
+ pr_debug("__fetched msg: %d %d %d %d\n",
+ m->id, m->cmd, m->ack, m->err);
+ return 0;
+ }
+
+ return -1;
+}
+
+static int __parasite_daemon_thread_wait_cmd(struct tid_state_s *s)
+{
+ futex_wait_while_lt(&s->cmd, PARASITE_CMD_DAEMONIZED);
+ return futex_get(&s->cmd);
+}
+
+static void __parasite_daemon_thread_ack(struct tid_state_s *s, int ret)
+{
+ s->ret = ret;
+ futex_set(&s->ack, PARASITE_CMD_IDLE);
+ futex_set_and_wake(&s->cmd, PARASITE_CMD_IDLE);
+ futex_wait_until(&s->ack, PARASITE_CMD_ACK);
+}
+
+static unsigned long noinline __used
+__parasite_daemon_thread(void *args, struct tid_state_s *s, unsigned long oldstack)
+{
+ pr_debug("Running daemon thread %d\n", s->id);
+
+ /* Reply we're alive */
+ if (__parasite_daemon_reply_ack(s->id, PARASITE_CMD_DAEMONIZE, 0))
+ return oldstack;
+
+ while (1) {
+ int ret, cmd;
+
+ cmd = __parasite_daemon_thread_wait_cmd(s);
+
+ pr_debug("Command %d in daemon thread %d\n", cmd, s->id);
+
+ switch (cmd) {
+ case PARASITE_CMD_DUMP_THREAD:
+ ret = dump_thread(args);
+ break;
+ case PARASITE_CMD_FINI_THREAD:
+ __parasite_daemon_thread_ack(s, 0);
+ fini_thread(s);
+ return oldstack;
+ default:
+ pr_err("Unknown command in parasite daemon thread: %d\n", cmd);
+ ret = -1;
+ break;
+ }
+ __parasite_daemon_thread_ack(s, ret);
+ }
+
+ return oldstack;
+}
+
+static int __parasite_execute_thread(struct ctl_msg *m)
+{
+ struct tid_state_s *s = find_thread_state(m->id);
+ if (!s)
+ return -ENOENT;
+
+ pr_debug("Wake thread %d daemon with command %d\n", s->id, m->cmd);
+ futex_set_and_wake(&s->cmd, m->cmd);
+
+ pr_debug("Wait thread %d for PARASITE_CMD_IDLE\n", s->id);
+ futex_wait_until(&s->cmd, PARASITE_CMD_IDLE);
+
+ pr_debug("Wake thread %d daemon with " __stringify_1(PARASITE_CMD_ACK) "\n", s->id);
+ futex_set_and_wake(&s->ack, PARASITE_CMD_ACK);
+
+ return s->ret;
+}
+
+static unsigned long noinline __used
+__parasite_daemon_thread_leader(void *args, struct tid_state_s *s, unsigned long oldstack)
+{
+ struct ctl_msg m = { };
+ int ret = -1;
+
+ pr_debug("Running daemon thread leader %d\n", s->id);
+
+ /* Reply we're alive */
+ if (__parasite_daemon_reply_ack(s->id, PARASITE_CMD_DAEMONIZE, 0))
+ return oldstack;
+
+ while (1) {
+ if (__parasite_daemon_wait_msg(&m))
+ break;
+
+ switch (m.cmd) {
+ case PARASITE_CMD_FINI:
+ ret = fini(s);
+ sys_close(ctlsock);
+ /*
+ * No ACK here since we're getting out.
+ */
+ break;
+ case PARASITE_CMD_FINI_THREAD:
+ ret = __parasite_execute_thread(&m);
+ break;
+ case PARASITE_CMD_DUMP_THREAD:
+ ret = __parasite_execute_thread(&m);
+ break;
+ case PARASITE_CMD_DUMPPAGES_INIT:
+ ret = dump_pages_init();
+ break;
+ case PARASITE_CMD_DUMPPAGES_FINI:
+ ret = dump_pages_fini();
+ break;
+ case PARASITE_CMD_DUMPPAGES:
+ ret = dump_pages(args);
+ break;
+ case PARASITE_CMD_DUMP_SIGACTS:
+ ret = dump_sigact(args);
+ break;
+ case PARASITE_CMD_DUMP_ITIMERS:
+ ret = dump_itimers(args);
+ break;
+ case PARASITE_CMD_DUMP_MISC:
+ ret = dump_misc(args);
+ break;
+ case PARASITE_CMD_DUMP_CREDS:
+ ret = dump_creds(args);
+ break;
+ case PARASITE_CMD_DRAIN_FDS:
+ ret = drain_fds(args);
+ break;
+ case PARASITE_CMD_GET_PROC_FD:
+ ret = parasite_get_proc_fd();
+ break;
+ case PARASITE_CMD_DUMP_TTY:
+ ret = parasite_dump_tty(args);
+ break;
+ default:
+ pr_err("Unknown command in parasite daemon thread leader: %d\n", m.cmd);
+ ret = -1;
+ break;
+ }
+
+ if (__parasite_daemon_reply_ack(m.id, m.cmd, ret))
+ break;
+ }
+
+ return oldstack;
+}
+
+static int noinline parasite_daemon(struct parasite_init_args *args)
+{
+ struct tid_state_s *s;
+ unsigned long new_sp = 0;
+
+ s = find_thread_state(args->id);
+ if (!s)
+ return -ENOENT;
+
+ pr_info("Parasite entering daemon mode for %d\n", s->id);
+ new_sp = (unsigned long)(void *)&s->stack[PARASITE_STACK_SIZE - 8];
+
+ if (s->id == thread_leader->id)
+ call_daemon_thread(new_sp, args, s, __parasite_daemon_thread_leader);
+ else
+ call_daemon_thread(new_sp, args, s, __parasite_daemon_thread);
+
+ pr_info("Parasite leaving daemon mode for %d\n", s->id);
+
+ if (s == thread_leader)
+ sys_munmap(tid_state, TID_STATE_SIZE(nr_tid_state));
+
+ asm_trap();
+ return 0;
}
int __used parasite_service(unsigned int cmd, void *args)
@@ -675,34 +875,10 @@ int __used parasite_service(unsigned int cmd, void *args)
return init(args);
case PARASITE_CMD_INIT_THREAD:
return init_thread(args);
- case PARASITE_CMD_FINI:
- return fini(args);
- case PARASITE_CMD_FINI_THREAD:
- return fini_thread(args);
case PARASITE_CMD_CFG_LOG:
return parasite_cfg_log(args);
- case PARASITE_CMD_DUMPPAGES_INIT:
- return dump_pages_init();
- case PARASITE_CMD_DUMPPAGES_FINI:
- return dump_pages_fini();
- case PARASITE_CMD_DUMPPAGES:
- return dump_pages(args);
- case PARASITE_CMD_DUMP_SIGACTS:
- return dump_sigact(args);
- case PARASITE_CMD_DUMP_ITIMERS:
- return dump_itimers(args);
- case PARASITE_CMD_DUMP_MISC:
- return dump_misc(args);
- case PARASITE_CMD_DUMP_CREDS:
- return dump_creds(args);
- case PARASITE_CMD_DUMP_THREAD:
- return dump_thread(args);
- case PARASITE_CMD_DRAIN_FDS:
- return drain_fds(args);
- case PARASITE_CMD_GET_PROC_FD:
- return parasite_get_proc_fd();
- case PARASITE_CMD_DUMP_TTY:
- return parasite_dump_tty(args);
+ case PARASITE_CMD_DAEMONIZE:
+ return parasite_daemon(args);
}
pr_err("Unknown command to parasite: %d\n", cmd);
--
1.8.1.2
More information about the CRIU
mailing list