[CRIU] [PATCH v2] Add inherit fd support
Saied Kazemi
saied at google.com
Tue Dec 9 12:20:14 PST 2014
There are cases where a process's file descriptor cannot be restored
from the checkpoint images. For example, a pipe file descriptor with
one end in the checkpointed process and the other end in a separate
process (that was not part of the checkpointed process tree) cannot be
restored because after checkpoint the pipe will be broken.
There are also cases where the user wants to use a new file during
restore instead of the original file at checkpoint time. For example,
the user wants to change the log file of a process from /path/to/oldlog
to /path/to/newlog.
In these cases, criu's caller should set up a new file descriptor to be
inherited by the restored process and specify the file descriptor with the
--inherit-fd command line option. The argument of --inherit-fd has the
format fd[%d]:%s, where %d tells criu which of its own file descriptors
to use for restoring the file identified by %s.
As a debugging aid, if the argument has the format debug[%d]:%s, it tells
criu to write out the string after colon to the file descriptor %d. This
can be used, for example, as an easy way to leave a "restore marker"
in the output stream of the process.
It's important to note that inherit fd support breaks applications
that depend on the state of the file descriptor being inherited. So,
consider inherit fd only for specific use cases that you know for sure
won't break the application.
For examples please visit http://criu.org/Category:HOWTO.
Signed-off-by: Saied Kazemi <saied at google.com>
---
v2: Added a check in send_fd_to_self() to avoid closing an inherit fd.
Also, as an extra measure of caution, added checks in the inherit fd
look up functions to make sure that the inherit fd hasn't been reused.
The patch also includes minor cosmetic changes.
cr-restore.c | 6 ++
crtools.c | 15 +++
files-reg.c | 14 +++
files.c | 257 ++++++++++++++++++++++++++++++++++++++++++++++++++-
include/cr_options.h | 1 +
include/files.h | 7 ++
pipes.c | 33 +++++++
util.c | 5 +
8 files changed, 337 insertions(+), 1 deletion(-)
diff --git a/cr-restore.c b/cr-restore.c
index 93a6ca6..e3ff3a6 100644
--- a/cr-restore.c
+++ b/cr-restore.c
@@ -799,6 +799,9 @@ static int restore_one_alive_task(int pid, CoreEntry *core)
if (collect_helper_pids() < 0)
return -1;
+ if (inherit_fd_fini() < 0)
+ return -1;
+
return sigreturn_restore(pid, core);
}
@@ -861,6 +864,9 @@ static int restore_one_zombie(int pid, CoreEntry *core)
pr_info("Restoring zombie with %d code\n", exit_code);
+ if (inherit_fd_fini() < 0)
+ return -1;
+
sys_prctl(PR_SET_NAME, (long)(void *)core->tc->comm, 0, 0, 0);
if (task_entries != NULL) {
diff --git a/crtools.c b/crtools.c
index 0ac667c..c03b025 100644
--- a/crtools.c
+++ b/crtools.c
@@ -52,6 +52,7 @@ void init_opts(void)
INIT_LIST_HEAD(&opts.veth_pairs);
INIT_LIST_HEAD(&opts.scripts);
INIT_LIST_HEAD(&opts.ext_mounts);
+ INIT_LIST_HEAD(&opts.inherit_fds);
INIT_LIST_HEAD(&opts.new_cgroup_roots);
opts.cpu_cap = CPU_CAP_DEFAULT;
@@ -187,6 +188,7 @@ int main(int argc, char *argv[], char *envp[])
{ "exec-cmd", no_argument, 0, 1059},
{ "manage-cgroups", no_argument, 0, 1060},
{ "cgroup-root", required_argument, 0, 1061},
+ { "inherit-fd", required_argument, 0, 1062},
{ },
};
@@ -392,6 +394,10 @@ int main(int argc, char *argv[], char *envp[])
return -1;
}
break;
+ case 1062:
+ if (inherit_fd_add(optarg) < 0)
+ return 1;
+ break;
case 'M':
{
char *aux;
@@ -469,6 +475,15 @@ int main(int argc, char *argv[], char *envp[])
if (log_init(opts.output))
return 1;
+ if (!list_empty(&opts.inherit_fds)) {
+ if (strcmp(argv[optind], "restore")) {
+ pr_err("--inherit-fd is restore-only option\n");
+ return 1;
+ }
+ /* now that log file is set up, print inherit fd list */
+ inherit_fd_log();
+ }
+
if (opts.img_parent)
pr_info("Will do snapshot from %s\n", opts.img_parent);
diff --git a/files-reg.c b/files-reg.c
index 20b50a0..b1e9f73 100644
--- a/files-reg.c
+++ b/files-reg.c
@@ -921,6 +921,20 @@ int open_path(struct file_desc *d,
rfi = container_of(d, struct reg_file_info, d);
+ tmp = inherit_fd_lookup_id(rfi->path);
+ if (tmp >= 0) {
+ int fd;
+
+ fd = dup(tmp);
+ if (fd == -1) {
+ pr_perror("Can't dup inherit fd %d\n", fd);
+ return -1;
+ }
+ pr_info("File %s will be restored from fd %d duped from inherit fd %d\n",
+ rfi->path, fd, tmp);
+ return fd;
+ }
+
if (rfi->remap) {
mutex_lock(ghost_file_mutex);
if (rfi->remap->is_dir) {
diff --git a/files.c b/files.c
index f009e7d..e2c4aa3 100644
--- a/files.c
+++ b/files.c
@@ -37,6 +37,7 @@
#include "imgset.h"
#include "fs-magic.h"
#include "proc_parse.h"
+#include "cr_options.h"
#include "parasite.h"
#include "parasite-syscall.h"
@@ -786,6 +787,10 @@ static int send_fd_to_self(int fd, struct fdinfo_list_entry *fle, int *sock)
if (fd == dfd)
return 0;
+ /* make sure we won't clash with an inherit fd */
+ if (inherit_fd_resolve_clash(dfd) < 0)
+ return -1;
+
pr_info("\t\t\tGoing to dup %d into %d\n", fd, dfd);
if (move_img_fd(sock, dfd))
return -1;
@@ -945,7 +950,8 @@ int close_old_fds(struct pstree_item *me)
return -1;
}
- if ((!is_any_service_fd(fd)) && (dirfd(dir) != fd))
+ if ((!is_any_service_fd(fd)) && (dirfd(dir) != fd) &&
+ !inherit_fd_lookup_fd(fd, __FUNCTION__))
close_safe(&fd);
}
@@ -1176,3 +1182,252 @@ int shared_fdt_prepare(struct pstree_item *item)
return 0;
}
+
+/*
+ * Inherit fd support.
+ *
+ * There are cases where a process's file descriptor cannot be restored
+ * from the checkpointed image. For example, a pipe file descriptor with
+ * one end in the checkpointed process and the other end in a separate
+ * process (that was not part of the checkpointed process tree) cannot be
+ * restored because after checkpoint the pipe would be broken and removed.
+ *
+ * There are also cases where the user wants to use a new file during
+ * restore instead of the original file in the checkpointed image. For
+ * example, the user wants to change the log file of a process from
+ * /path/to/oldlog to /path/to/newlog.
+ *
+ * In these cases, criu's caller should set up a new file descriptor to be
+ * inherited by the restored process and specify it with the --inherit-fd
+ * command line option. The argument of --inherit-fd has the format
+ * fd[%d]:%s, where %d tells criu which of its own file descriptor to use
+ * for restoring file identified by %s.
+ *
+ * As a debugging aid, if the argument has the format debug[%d]:%s, it tells
+ * criu to write out the string after colon to the file descriptor %d. This
+ * can be used to leave a "restore marker" in the output stream of the process.
+ *
+ * It's important to note that inherit fd support breaks applications
+ * that depend on the state of the file descriptor being inherited. So,
+ * consider inherit fd only for specific use cases that you know for sure
+ * won't break the application.
+ *
+ * For examples please visit http://criu.org/Category:HOWTO.
+ */
+
+struct inherit_fd {
+ struct list_head inh_list;
+ char *inh_id; /* file identifier */
+ int inh_fd; /* criu's descriptor to inherit */
+ dev_t inh_dev;
+ ino_t inh_ino;
+ mode_t inh_mode;
+ dev_t inh_rdev;
+};
+
+/*
+ * Return 1 if inherit fd has been closed or reused, 0 otherwise.
+ */
+static int inherit_fd_reused(struct inherit_fd *inh)
+{
+ struct stat sbuf;
+
+ if (fstat(inh->inh_fd, &sbuf) == -1) {
+ if (errno == EBADF) {
+ pr_debug("Inherit fd %s -> %d has been closed\n",
+ inh->inh_id, inh->inh_fd);
+ return 1;
+ }
+ pr_perror("Can't fstat inherit fd %d", inh->inh_fd);
+ return -1;
+ }
+
+ if (inh->inh_dev != sbuf.st_dev || inh->inh_ino != sbuf.st_ino ||
+ inh->inh_mode != sbuf.st_mode || inh->inh_rdev != sbuf.st_rdev) {
+ pr_info("Inherit fd %s -> %d has been reused\n",
+ inh->inh_id, inh->inh_fd);
+ return 1;
+ }
+ return 0;
+}
+
+/*
+ * We can't print diagnostics messages in this function because the
+ * log file isn't initialized yet.
+ */
+int inherit_fd_add(char *optarg)
+{
+ char *cp = NULL;
+ int n = -1;
+ int fd = -1;
+ int dbg = 0;
+ struct stat sbuf;
+ struct inherit_fd *inh;
+
+ /*
+ * Parse the argument.
+ */
+ if (!strncmp(optarg, "fd", 2))
+ cp = &optarg[2];
+ else if (!strncmp(optarg, "debug", 5)) {
+ cp = &optarg[5];
+ dbg = 1;
+ }
+ if (cp) {
+ n = sscanf(cp, "[%d]:", &fd);
+ cp = strchr(optarg, ':');
+ }
+ if (n != 1 || fd < 0 || !cp || !cp[1]) {
+ pr_err("Invalid inherit fd argument: %s\n", optarg);
+ return -1;
+ }
+
+ /*
+ * If the argument is a debug string, write it to fd.
+ * Otherwise, add it to the inherit fd list.
+ */
+ cp++;
+ if (dbg) {
+ n = strlen(cp);
+ if (write(fd, cp, n) != n) {
+ pr_err("Can't write debug message %s to inherit fd %d\n",
+ cp, fd);
+ return -1;
+ }
+ return 0;
+ }
+
+ if (fstat(fd, &sbuf) == -1) {
+ pr_perror("Can't fstat inherit fd %d", fd);
+ return -1;
+ }
+
+ inh = xmalloc(sizeof *inh);
+ if (inh == NULL)
+ return -1;
+
+ inh->inh_id = cp;
+ inh->inh_fd = fd;
+ inh->inh_dev = sbuf.st_dev;
+ inh->inh_ino = sbuf.st_ino;
+ inh->inh_mode = sbuf.st_mode;
+ inh->inh_rdev = sbuf.st_rdev;
+ list_add_tail(&inh->inh_list, &opts.inherit_fds);
+ return 0;
+}
+
+/*
+ * Log the inherit fd list. Called for diagnostics purposes
+ * after the log file is initialized.
+ */
+void inherit_fd_log(void)
+{
+ struct inherit_fd *inh;
+
+ list_for_each_entry(inh, &opts.inherit_fds, inh_list) {
+ pr_info("File %s will be restored from inherit fd %d\n",
+ inh->inh_id, inh->inh_fd);
+ }
+}
+
+/*
+ * Look up the inherit fd list by a file identifier.
+ */
+int inherit_fd_lookup_id(char *id)
+{
+ int ret;
+ struct inherit_fd *inh;
+
+ ret = -1;
+ list_for_each_entry(inh, &opts.inherit_fds, inh_list) {
+ if (!strcmp(inh->inh_id, id)) {
+ if (!inherit_fd_reused(inh)) {
+ ret = inh->inh_fd;
+ pr_debug("Found id %s (fd %d) in inherit fd list\n",
+ id, ret);
+ }
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * Look up the inherit fd list by a file descriptor.
+ */
+struct inherit_fd *inherit_fd_lookup_fd(int fd, const char *caller)
+{
+ struct inherit_fd *ret;
+ struct inherit_fd *inh;
+
+ ret = NULL;
+ list_for_each_entry(inh, &opts.inherit_fds, inh_list) {
+ if (inh->inh_fd == fd) {
+ if (!inherit_fd_reused(inh)) {
+ ret = inh;
+ pr_debug("Found fd %d (id %s) in inherit fd list (caller %s)\n",
+ fd, inh->inh_id, caller);
+ }
+ break;
+ }
+ }
+ return ret;
+}
+
+/*
+ * If the specified fd clashes with an inherit fd,
+ * move the inherit fd.
+ */
+int inherit_fd_resolve_clash(int fd)
+{
+ int newfd;
+ struct inherit_fd *inh;
+
+ inh = inherit_fd_lookup_fd(fd, __FUNCTION__);
+ if (inh == NULL)
+ return 0;
+
+ newfd = dup(fd);
+ if (newfd == -1) {
+ pr_perror("Can't dup inherit fd %d", fd);
+ return -1;
+ }
+
+ if (close(fd) == -1) {
+ pr_perror("Can't close inherit fd %d", fd);
+ return -1;
+ }
+
+ inh->inh_fd = newfd;
+ pr_debug("Inherit fd %d moved to %d to resolve clash\n", fd, inh->inh_fd);
+ return 0;
+}
+
+/*
+ * Close all inherit fds.
+ */
+int inherit_fd_fini()
+{
+ int reused;
+ struct inherit_fd *inh;
+
+ list_for_each_entry(inh, &opts.inherit_fds, inh_list) {
+ if (inh->inh_fd < 0) {
+ pr_err("File %s in inherit fd list has invalid fd %d\n",
+ inh->inh_id, inh->inh_fd);
+ return -1;
+ }
+
+ reused = inherit_fd_reused(inh);
+ if (reused < 0)
+ return -1;
+
+ if (!reused) {
+ pr_debug("Closing inherit fd %d -> %s\n", inh->inh_fd,
+ inh->inh_id);
+ if (close_safe(&inh->inh_fd) < 0)
+ return -1;
+ }
+ }
+ return 0;
+}
diff --git a/include/cr_options.h b/include/cr_options.h
index a9f9e92..3735b09 100644
--- a/include/cr_options.h
+++ b/include/cr_options.h
@@ -43,6 +43,7 @@ struct cr_options {
struct list_head veth_pairs;
struct list_head scripts;
struct list_head ext_mounts;
+ struct list_head inherit_fds;
char *libdir;
bool use_page_server;
unsigned short ps_port;
diff --git a/include/files.h b/include/files.h
index 4c9300d..e3f79cc 100644
--- a/include/files.h
+++ b/include/files.h
@@ -168,4 +168,11 @@ extern struct collect_image_info ext_file_cinfo;
extern int dump_unsupp_fd(struct fd_parms *p, int lfd,
struct cr_img *, char *more, char *info);
+extern int inherit_fd_add(char *optarg);
+extern void inherit_fd_log(void);
+extern int inherit_fd_lookup_id(char *id);
+extern struct inherit_fd *inherit_fd_lookup_fd(int fd, const char *caller);
+extern int inherit_fd_resolve_clash(int fd);
+extern int inherit_fd_fini(void);
+
#endif /* __CR_FILES_H__ */
diff --git a/pipes.c b/pipes.c
index ddcc5cf..24a83ff 100644
--- a/pipes.c
+++ b/pipes.c
@@ -293,8 +293,20 @@ static int recv_pipe_fd(struct pipe_info *pi)
return fd;
}
+static char *pipe_id_string(int pipe_id)
+{
+ static char idstr[16];
+
+ if (snprintf(idstr, sizeof idstr, "pipe:[%d]", pipe_id) >= sizeof idstr) {
+ pr_err("Not enough room for pipe %d identifier string\n", pipe_id);
+ return NULL;
+ }
+ return idstr;
+}
+
static int open_pipe(struct file_desc *d)
{
+ char *pipe_name;
struct pipe_info *pi, *p;
int ret, tmp;
int pfd[2];
@@ -302,6 +314,27 @@ static int open_pipe(struct file_desc *d)
pi = container_of(d, struct pipe_info, d);
+ /*
+ * If the pipe is in the inherit fd list,
+ * it should be inherited rather than created.
+ */
+ pipe_name = pipe_id_string(pi->pe->pipe_id);
+ if (pipe_name == NULL)
+ return -1;
+ tmp = inherit_fd_lookup_id(pipe_name);
+ if (tmp >= 0) {
+ int fd;
+
+ fd = dup(tmp);
+ if (fd == -1) {
+ pr_perror("Can't dup inherit fd %d\n", fd);
+ return -1;
+ }
+ pr_info("Pipe %s will be restored from fd %d duped from inherit fd %d\n",
+ pipe_name, fd, tmp);
+ return fd;
+ }
+
pr_info("\t\tCreating pipe pipe_id=%#x id=%#x\n", pi->pe->pipe_id, pi->pe->id);
if (!pi->create)
diff --git a/util.c b/util.c
index dd76863..084d98e 100644
--- a/util.c
+++ b/util.c
@@ -41,6 +41,7 @@
#include "cr_options.h"
#include "servicefd.h"
#include "cr-service.h"
+#include "files.h"
#define VMA_OPT_LEN 128
@@ -93,6 +94,7 @@ void pr_vma(unsigned int loglevel, const struct vma_area *vma_area)
int close_safe(int *fd)
{
int ret = 0;
+
if (*fd > -1) {
ret = close(*fd);
if (!ret)
@@ -109,6 +111,9 @@ int reopen_fd_as_safe(char *file, int line, int new_fd, int old_fd, bool allow_r
int tmp;
if (old_fd != new_fd) {
+ /* make sure we won't clash with an inherit fd */
+ if (inherit_fd_resolve_clash(new_fd) < 0)
+ return -1;
if (!allow_reuse_fd) {
if (fcntl(new_fd, F_GETFD) != -1 || errno != EBADF) {
--
2.2.0.rc0.207.ga3a616c
More information about the CRIU
mailing list