[CRIU] [PATCH v6] Add docker phaul driver
Hui Kang
hkang.sunysb at gmail.com
Tue Oct 27 19:38:16 PDT 2015
See the instruction at test/docker/HOWTO
Changes since v5:
- Remove all docker-specfic checks
- Add dump_need_ps() to control if page_server is needed or not
Changes since v4:
- Remove all docker-specfic check in iter.py
- dst node should create docker runtime meta state.json
Changes since v3:
- Use sync_imgs_to_target to sync criu image for docker
- Add can_pre_dump() to check if type supports pre_dump
TODO (suggestions from xemul, avagin, nikita):
(1) Resolve the criu_conn error at the end of start_migration()
for docker
(2) Remove sleep, call wait for docker daemon
(3) Remove get_driver_name; using some fake object, additional
abstraction layers and such stuff
(4) Wait for docker-py to integrate the C/R APIs
Signed-off-by: Hui Kang <hkang.sunysb at gmail.com>
---
p.haul | 2 +-
phaul/fs_haul_subtree.py | 26 +++++---
phaul/htype.py | 1 +
phaul/iters.py | 6 +-
phaul/p_haul_docker.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++
phaul/p_haul_pid.py | 6 ++
phaul/p_haul_vz.py | 5 ++
phaul/service.py | 5 +-
test/docker/HOWTO | 82 +++++++++++++++++++++++++
9 files changed, 274 insertions(+), 16 deletions(-)
create mode 100644 phaul/p_haul_docker.py
create mode 100644 test/docker/HOWTO
diff --git a/p.haul b/p.haul
index 11460b4..478e1c7 100755
--- a/p.haul
+++ b/p.haul
@@ -25,7 +25,7 @@ import phaul.htype
parser = argparse.ArgumentParser("Process HAULer")
parser.add_argument("type", choices=phaul.htype.get_haul_names(),
- help="Type of hat to haul, e.g. vz or lxc")
+ help="Type of hat to haul, e.g. vz, lxc, or docker")
parser.add_argument("id", help="ID of what to haul")
parser.add_argument("--to", help="IP where to haul")
parser.add_argument("--fdrpc", help="File descriptor of rpc socket", type=int, required=True)
diff --git a/phaul/fs_haul_subtree.py b/phaul/fs_haul_subtree.py
index a9bd559..32334cb 100644
--- a/phaul/fs_haul_subtree.py
+++ b/phaul/fs_haul_subtree.py
@@ -11,9 +11,12 @@ import logging
rsync_log_file = "rsync.log"
class p_haul_fs:
- def __init__(self, subtree_path):
- logging.info("Initialized subtree FS hauler (%s)", subtree_path)
- self.__root = subtree_path
+ def __init__(self, subtree_paths):
+ self.__roots = []
+ for path in subtree_paths:
+ logging.info("Initialized subtree FS hauler (%s)", path)
+ self.__roots.append(path)
+
self.__thost = None
def set_options(self, opts):
@@ -24,16 +27,19 @@ class p_haul_fs:
def __run_rsync(self):
logf = open(os.path.join(self.__wdir, rsync_log_file), "w+")
- dst = "%s:%s" % (self.__thost, os.path.dirname(self.__root))
- # First rsync might be very long. Wait for it not
- # to produce big pause between the 1st pre-dump and
- # .stop_migration
+ for dir_name in self.__roots:
+
+ dst = "%s:%s" % (self.__thost, os.path.dirname(dir_name))
+
+ # First rsync might be very long. Wait for it not
+ # to produce big pause between the 1st pre-dump and
+ # .stop_migration
- ret = sp.call(["rsync", "-a", self.__root, dst],
+ ret = sp.call(["rsync", "-a", dir_name, dst],
stdout = logf, stderr = logf)
- if ret != 0:
- raise Exception("Rsync failed")
+ if ret != 0:
+ raise Exception("Rsync failed")
def start_migration(self):
logging.info("Starting FS migration")
diff --git a/phaul/htype.py b/phaul/htype.py
index f3a4774..bee49b4 100644
--- a/phaul/htype.py
+++ b/phaul/htype.py
@@ -11,6 +11,7 @@ __haul_modules = {
"vz": "p_haul_vz",
"pid": "p_haul_pid",
"lxc": "p_haul_lxc",
+ "docker": "p_haul_docker",
}
def __get(id):
diff --git a/phaul/iters.py b/phaul/iters.py
index 9be4e97..a578078 100644
--- a/phaul/iters.py
+++ b/phaul/iters.py
@@ -113,7 +113,7 @@ class phaul_iter_worker:
if self.pre_dump == PRE_DUMP_AUTO_DETECT:
# pre-dump auto-detection
try:
- self.pre_dump = self.pre_dump_check()
+ self.pre_dump = (self.pre_dump_check() and self.htype.can_pre_dump())
logging.info("\t`- Auto %s" % (self.pre_dump and 'enabled' or 'disabled'))
except:
# The available criu seems to not
@@ -132,7 +132,7 @@ class phaul_iter_worker:
while self.pre_dump:
logging.info("* Iteration %d", iter_index)
- self.target_host.start_iter()
+ self.target_host.start_iter(True)
self.img.new_image_dir()
logging.info("\tIssuing pre-dump command to service")
@@ -186,7 +186,7 @@ class phaul_iter_worker:
logging.info("Final dump and restore")
- self.target_host.start_iter()
+ self.target_host.start_iter(self.htype.dump_need_ps())
self.img.new_image_dir()
logging.info("\tIssuing dump command to service")
diff --git a/phaul/p_haul_docker.py b/phaul/p_haul_docker.py
new file mode 100644
index 0000000..045558d
--- /dev/null
+++ b/phaul/p_haul_docker.py
@@ -0,0 +1,157 @@
+#
+# Docker container hauler
+#
+
+import os
+import logging
+import time
+import fs_haul_subtree
+import pycriu.rpc
+import json
+import subprocess as sp
+from subprocess import PIPE
+
+# TODO use docker-py
+# import docker
+
+# Some constants for docker
+docker_bin = "/usr/bin/docker-1.9.0-dev"
+docker_dir = "/var/lib/docker/"
+docker_run_meta_dir = "/var/run/docker/execdriver/native"
+
+class p_haul_type:
+ def __init__(self, ctid):
+
+ # TODO ctid must > 3 digit; with docker-py, we can also resolve
+ # container name
+ if len(ctid) < 3:
+ raise Exception("Docker container ID must be > 3 digits")
+
+ self._ctid = ctid
+ self._ct_rootfs = ""
+
+ def init_src(self):
+ self.full_ctid = self.get_full_ctid()
+ self.__load_ct_config(docker_dir)
+
+ def init_dst(self):
+ pass
+
+ def adjust_criu_req(self, req):
+ """Add module-specific options to criu request"""
+ pass
+
+ def root_task_pid(self):
+ # Do we need this for Docker?
+ return self.full_ctid
+
+ def __load_ct_config(self, path):
+
+ # Each docker container has 3 directories that need to be
+ # migrated: (1) root filesystem, (2) container configuration,
+ # (3) runtime meta state
+ self._ct_rootfs = os.path.join(docker_dir, "aufs/mnt", self.full_ctid)
+ self._ct_config_dir = os.path.join(docker_dir, "containers", self.full_ctid)
+ self._ct_run_meta_dir = os.path.join(docker_run_meta_dir, self.full_ctid)
+ logging.info("Container rootfs: %s", self._ct_rootfs)
+ logging.info("Container config: %s", self._ct_config_dir)
+ logging.info("Container meta: %s", self._ct_run_meta_dir)
+
+ def set_options(self, opts):
+ pass
+
+ # Remove any specific FS setup
+ def umount(self):
+ pass
+
+ def get_fs(self, fs_sk=None):
+ # use rsync for rootfs and configuration directories
+ return fs_haul_subtree.p_haul_fs([self._ct_rootfs, self._ct_config_dir])
+
+ def get_fs_receiver(self, fs_sk=None):
+ return None
+
+ def get_full_ctid(self):
+ dir_name_list = os.listdir(os.path.join(docker_dir, "containers"))
+
+ full_id = ""
+ for name in dir_name_list:
+ name = name.rsplit("/")
+ if (name[0].find(self._ctid) == 0):
+ full_id = name[0]
+ break
+
+ if full_id != "":
+ return full_id
+ else:
+ raise Exception("Can not find container fs")
+
+ def final_dump(self, pid, img, ccon, fs):
+ logging.info("Dump docker container %s", pid)
+
+ # TODO: docker API does not have checkpoint right now
+ # cli.checkpoint() so we have to use the command line
+ # cli = docker.Client(base_url='unix://var/run/docker.sock')
+ # output = cli.info()
+ # call docker API
+
+ logf = open("/tmp/docker_checkpoint.log", "w+")
+ image_path_opt = "--image-dir=" + img.image_dir()
+ ret = sp.call([docker_bin, "checkpoint", image_path_opt, self._ctid],
+ stdout = logf, stderr = logf)
+ if ret != 0:
+ raise Exception("docker checkpoint failed")
+ #
+ # Meta-images for docker -- /var/run/docker
+ #
+ def get_meta_images(self, path):
+ # Send the meta state file with criu images
+ return [(os.path.join(self._ct_run_meta_dir, "state.json"), "state.json"),
+ (os.path.join(path, "descriptors.json"), "descriptors.json")]
+
+ def put_meta_images(self, dir):
+ # Create docker runtime meta dir on dst side
+ with open(os.path.join(dir, "state.json")) as data_file:
+ data = json.load(data_file)
+ self.full_ctid=data["id"]
+
+ self.__load_ct_config(docker_dir)
+ os.makedirs(self._ct_run_meta_dir)
+ pd = sp.Popen(["cp", os.path.join(dir, "state.json"), self._ct_run_meta_dir], stdout = PIPE)
+ status = pd.wait()
+
+ def kill_last_docker_daemon(self):
+ p = sp.Popen(['pgrep', '-l' , docker_bin], stdout=sp.PIPE)
+ out, err = p.communicate()
+
+ for line in out.splitlines():
+ line = bytes.decode(line)
+ pid = int(line.split(None, 1)[0])
+ os.kill(pid, signal.SIGKILL)
+
+ def final_restore(self, img, criu):
+ logf = open("/tmp/docker_restore.log", "w+")
+
+ # Kill any previous docker daemon in order to reload the
+ # status of the migrated container
+ self.kill_last_docker_daemon()
+
+ # start docker daemon in background
+ daemon = sp.Popen([docker_bin, "daemon", "-s", "aufs"],
+ stdout = logf, stderr = logf)
+ # daemon.wait() TODO: docker daemon not return
+ time.sleep(2)
+
+ image_path_opt = "--image-dir=" + img.image_dir()
+ ret = sp.call([docker_bin, "restore", image_path_opt, self._ctid],
+ stdout = logf, stderr = logf)
+ if ret != 0:
+ raise Exception("docker restore failed")
+
+ def can_pre_dump(self):
+ # XXX: Do not do predump for docker right now. Add page-server
+ # to docker C/R API, then we can enable the pre-dump
+ return False
+
+ def dump_need_ps(self):
+ return False
diff --git a/phaul/p_haul_pid.py b/phaul/p_haul_pid.py
index e354dff..c445bd2 100644
--- a/phaul/p_haul_pid.py
+++ b/phaul/p_haul_pid.py
@@ -94,3 +94,9 @@ class p_haul_type:
# Get list of veth pairs if any
def veths(self):
return []
+
+ def can_pre_dump(self):
+ return True
+
+ def dump_need_ps(self):
+ return True
diff --git a/phaul/p_haul_vz.py b/phaul/p_haul_vz.py
index 903006d..7971a65 100644
--- a/phaul/p_haul_vz.py
+++ b/phaul/p_haul_vz.py
@@ -243,6 +243,11 @@ class p_haul_type:
def can_migrate_tcp(self):
return True
+ def can_pre_dump(self):
+ return True
+
+ def dump_need_ps(self):
+ return False
def parse_vz_config(body):
"""Parse shell-like virtuozzo config file"""
diff --git a/phaul/service.py b/phaul/service.py
index a03b43e..305f729 100644
--- a/phaul/service.py
+++ b/phaul/service.py
@@ -69,10 +69,11 @@ class phaul_service:
logging.info("\tPage server started at %d", resp.ps.pid)
- def rpc_start_iter(self):
+ def rpc_start_iter(self, need_ps):
self.dump_iter_index += 1
self.img.new_image_dir()
- self.start_page_server()
+ if need_ps:
+ self.start_page_server()
def rpc_end_iter(self):
pass
diff --git a/test/docker/HOWTO b/test/docker/HOWTO
new file mode 100644
index 0000000..3255360
--- /dev/null
+++ b/test/docker/HOWTO
@@ -0,0 +1,82 @@
+This HOWTO describes how to _non_ live-migrate a docker container from one
+docker host to another.
+
+** This is an experimental implementation of docker migration, which may affect
+your running containers.
+
+0. Install CRIU, p.haul, docker on both nodes
+
+ Besides the packages that are needed to compile and run CRIU and p.haul,
+ the specific docker binary that supports checkpoint/restore should be used.
+
+ Refer to step 0 in test/mtouch/HOWTO about the pacekages for CRIU and p.haul.
+
+ The docker version that supports checkpoint and restore can be obtained by
+
+ # git clone https://github.com/boucher/docker.git
+ # cd docker.git
+ # git checkout cr-combined
+
+ On both nodes, compile and store the the docker binary as
+
+ /usr/bin/docker-1.9.0-dev
+
+ Note that the path above is for now hard-coded in p_haul_docker.py
+
+1. Prepare criu and p.haul on both nodes (adapted from test/mtouch/HOWTO)
+
+ a) CRIU
+
+ * Clone CRIU repository from git://github.com/xemul/criu
+ and compile it with 'make'
+
+ * Make _local_ directory for service ($csdir)
+ * Start CRIU service by root user
+ Note that this step is mandatory. Although runC will start criu in swrk
+ mode, p.haul needs to connect to criu service for validating CPU.
+
+ # criu service -o $csdir/log -v4 --daemon
+
+ b) On destination node start the p.haul service
+
+ [dst]# ./p.haul-wrap service
+ Starting p.haul rpyc service
+
+ It will go daemon and will print logs on the terminals
+
+3. Run the test container on source node
+
+ a) Start the docker daemon
+
+ # /usr/bin/docker-1.9.0-dev daemon -s aufs
+
+ b) Start the container
+
+ # /usr/bin/docker-1.9.0-dev run -d busybox:latest /bin/sh -c 'i=0; while true; do echo $i >> /foo; i=$(expr $i + 1); sleep 1; done'
+
+ This command will return the container's ID, e.g., d78.
+ (borrowed from https://criu.org/Docker)
+
+4. Migrate container from source node
+
+ [src]# ./p.haul-wrap client to docker [container ID, e.g., d78]
+
+ to is the ip address of the dst node
+
+ For example:
+
+ [src]# ./p.haul-wrap client 192.168.11.106 docker d78
+ 192.168.11.106 is the destination node IP and d78 is the container ID
+
+
+ When the command returns, on the destination node run
+
+ [dst]# /usr/bin/docker-1.9.0-dev ps
+ [dst]# /usr/bin/docker-1.9.0-dev exec d78 cat /foo
+
+ to verify the counter is continuously being incremented.
+
+Known limitations.
+
+1. No support from docker python binding
+2. Docker daemon has to be restarted on the destination node
--
1.9.1
More information about the CRIU
mailing list