[CRIU] [PATCH 2/4] tools: add criu2text
Pavel Emelyanov
xemul at parallels.com
Tue Sep 30 01:12:40 PDT 2014
On 09/19/2014 05:53 PM, Ruslan Kuprieiev wrote:
> criu2text.py handles cmdline opts, opens files and reads\writes them
> properly. images.py has a dict with MAGIC:pb_entry_type to specify what
> type of pb entry we should read from image with magic MAGIC.
> magic-gen.py parses include/magic.h to get all MAGIC values except
> RAW and V1, because criu2text can't handle them for now.
OK, I like the code, but let's rework the way tool works a little bit.
1. Right now, AFAIK, the tool turns one .img file into a set of .txt
ones. Let's make tool convert _one_ file to _one_ file, e.g. core.img
into core.txt. And the syntax should look like this:
criu2text -i|--in <filename> -o|--out <filename>
If <filename> == "-", then stdin/stdout is used.
The conversion from .img to .txt would work automatically, the conversion
back would have to know what kind of image we're working on. For this
the tool can write the magic value as the first line (in a form of a
comment if it makes sense).
2. I propose to name it "crit" -- CRiu Image Tool :)
3. Can we have all auxiliary stuff in the scripts/ dir? E.g. the magic.h
parsing script is such
4. I'd appreciate if the tool itself sits in the sources top directory.
What do you think?
> Signed-off-by: Ruslan Kuprieiev <kupruser at gmail.com>
> ---
> tools/criu2text/.gitignore | 4 ++
> tools/criu2text/Makefile | 15 +++++
> tools/criu2text/criu2text | 156 +++++++++++++++++++++++++++++++++++++++++++
> tools/criu2text/images.py | 112 +++++++++++++++++++++++++++++++
> tools/criu2text/magic-gen.py | 44 ++++++++++++
> 5 files changed, 331 insertions(+)
> create mode 100644 tools/criu2text/.gitignore
> create mode 100644 tools/criu2text/Makefile
> create mode 100755 tools/criu2text/criu2text
> create mode 100644 tools/criu2text/images.py
> create mode 100644 tools/criu2text/magic-gen.py
>
> diff --git a/tools/criu2text/.gitignore b/tools/criu2text/.gitignore
> new file mode 100644
> index 0000000..550f4b1
> --- /dev/null
> +++ b/tools/criu2text/.gitignore
> @@ -0,0 +1,4 @@
> +*.pyc
> +*_pb2.py
> +magic.py
> +__pycache__
> diff --git a/tools/criu2text/Makefile b/tools/criu2text/Makefile
> new file mode 100644
> index 0000000..986f426
> --- /dev/null
> +++ b/tools/criu2text/Makefile
> @@ -0,0 +1,15 @@
> +.PHONY: all clean protobuf
> +
> +PROTOBUF_DIR = ../../protobuf
> +MAGIC_C_HEADER = ../../include/magic.h
> +
> +all: magic.py protobuf
> +
> +magic.py: $(MAGIC_C_HEADER) protobuf
> + python magic-gen.py $(MAGIC_C_HEADER)
> +
> +protobuf:
> + protoc -I=$(PROTOBUF_DIR) --python_out=./ $(PROTOBUF_DIR)/*.proto
> +
> +clean:
> + rm -rf *.pyc *_pb2.py __pycache__ magic.py
> diff --git a/tools/criu2text/criu2text b/tools/criu2text/criu2text
> new file mode 100755
> index 0000000..1dfaae9
> --- /dev/null
> +++ b/tools/criu2text/criu2text
> @@ -0,0 +1,156 @@
> +#!/bin/env python
> +import os, sys
> +import struct
> +import google
> +import shutil
> +import argparse
> +
> +import images
> +
> +def img2text(img_name):
> + fin = open(img_name, 'r')
> + # Read magic cookie to identify criu image
> + buf = fin.read(4)
> +
> + # Convert cookie into "0x12345.." string to
> + # use it as a key in criu.magic dict
> + cookie, = struct.unpack('i', buf)
> + cookie = "0x{0:x}".format(cookie)
> +
> + base = os.path.basename(img_name)
> +
> + # Write cookie into file named magic
> + with open(base+'.magic', 'w+') as fout:
> + fout.write(cookie)
> +
> + if cookie in images.pb.keys():
> + pb = images.pb[cookie]()
> + num = 0
> +
> + while True:
> + # Read size of the following pb message
> + buf = fin.read(4)
> + if buf == '':
> + break
> +
> + size, = struct.unpack('i', buf)
> +
> + # Read pb message
> + pb.ParseFromString(fin.read(size))
> +
> + # Write text representation of
> + # pb message into num file
> + with open(base+'.'+str(num), 'w+') as fout:
> + google.protobuf.text_format.PrintMessage(pb, fout)
> +
> + num += 1
> + # pagemap.img is a special one. It starts with
> + # pagemap_head message and is followed by an
> + # array of pagemap_entry's.
> + if cookie == images.PAGEMAP_MAGIC:
> + pb = images.pagemap_entry()
> +
> + fin.close()
> + else:
> + print("not criu image or unknown format")
> + fin.close()
> + exit(1)
> +
> +def text2img(img_name):
> + base = os.path.basename(img_name)
> +
> + fout = open(base, 'w+')
> +
> + # Open magic file to get magic cookie
> + with open(img_name+'.magic', 'r') as fin:
> + cookie = fin.read()
> +
> + if cookie in images.pb.keys():
> + # Write magic cookie into img file
> + fout.write(struct.pack('i', int(cookie,16)))
> +
> + # Corresponding function that we need to call
> + # to init pb message
> + pb_init_function = images.pb[cookie]
> +
> + num = 0
> + while True:
> + if not os.path.isfile(img_name+'.'+str(num)):
> + break
> +
> + pb = pb_init_function()
> +
> + # Read pb message int text format from
> + # file num
> + with open(img_name+'.'+str(num), 'r') as fin:
> + pb_text = fin.read()
> +
> + # Parse pb from text format and write
> + # it to img file
> + google.protobuf.text_format.Merge(pb_text, pb)
> + size = len(pb.SerializeToString())
> + fout.write(struct.pack('i', size))
> + fout.write(pb.SerializeToString())
> +
> + num += 1
> + # pagemap.img is a special one. It starts with
> + # pagemap_head message and is followed by an
> + # array of pagemap_entry's.
> + if cookie == images.PAGEMAP_MAGIC:
> + pb_init_function = images.pagemap_entry
> +
> + fout.close()
> + else:
> + print("unknown format")
> + fout.close()
> + exit(1)
> +
> +def main():
> + desc = "Convert criu images to/from set of files in human-readable format"
> + epi = "Examples:\n"\
> + "1) Convert ./core-1234.img to set of text files (core-1234.img.magic,\n"\
> + " core-1234.img.0) and place them at text_imgs_dir.\n"\
> + "\n"\
> + " $ criu2text to-text core-1234.img -o text_imgs_dir\n"\
> + " $ ls text_imgs_dir\n"\
> + " core-1234.img.magic\n"\
> + " core-1234.img.0\n"\
> + "\n"\
> + "2) Convert text_imgs_dir/core-1234.img.magic, text_imgs_dir/core-1234.img.0\n"\
> + " (note common base text_imgs_dir/core-1234.img, which we're specifying)\n"\
> + " to criu img format and place it at imgs_dir.\n"\
> + "\n"\
> + " $ criu2text to-img text_imgs_dir/core-1234.img -o imgs_dir\n"\
> + " $ ls imgs_dir\n"\
> + " core-1234.img\n"\
> + "\n"
> +
> + parser = argparse.ArgumentParser(description = desc,
> + epilog = epi,
> + formatter_class = argparse.RawTextHelpFormatter)
> + parser.add_argument('command',
> + choices = ['to-text', 'to-img'],
> + help = 'to what type convert input file(s)')
> + parser.add_argument('image',
> + help = 'path to criu image or base path to criu image files in '\
> + 'human-readable format (e.g. ./imgs/core-1234.img)')
> + parser.add_argument('-o',
> + '--out',
> + default = './',
> + help = 'directory where output file(s) should be placed')
> +
> + opts = vars(parser.parse_args())
> +
> + opts['image'] = os.path.abspath(opts['image'])
> + opts['out'] = os.path.abspath(opts['out'])
> +
> + os.chdir(opts['out'])
> +
> + if opts['command'] == 'to-text':
> + img2text(opts['image'])
> +
> + if opts['command'] == 'to-img':
> + text2img(opts['image'])
> +
> +if __name__ == "__main__":
> + main()
> diff --git a/tools/criu2text/images.py b/tools/criu2text/images.py
> new file mode 100644
> index 0000000..99bc088
> --- /dev/null
> +++ b/tools/criu2text/images.py
> @@ -0,0 +1,112 @@
> +from stats_pb2 import *
> +from timer_pb2 import *
> +from ipc_sem_pb2 import *
> +from core_pb2 import *
> +from core_x86_pb2 import *
> +from remap_file_path_pb2 import *
> +from fsnotify_pb2 import *
> +from inventory_pb2 import *
> +from tcp_stream_pb2 import *
> +from mnt_pb2 import *
> +from packet_sock_pb2 import *
> +from pipe_data_pb2 import *
> +from ipc_var_pb2 import *
> +from sa_pb2 import *
> +from tun_pb2 import *
> +from fown_pb2 import *
> +from tty_pb2 import *
> +from rlimit_pb2 import *
> +from ipc_shm_pb2 import *
> +from file_lock_pb2 import *
> +from sk_unix_pb2 import *
> +from sk_packet_pb2 import *
> +from pagemap_pb2 import *
> +from pipe_pb2 import *
> +from pstree_pb2 import *
> +from sk_opts_pb2 import *
> +from siginfo_pb2 import *
> +from cgroup_pb2 import *
> +from fh_pb2 import *
> +from ipc_msg_pb2 import *
> +from mm_pb2 import *
> +from netdev_pb2 import *
> +from timerfd_pb2 import *
> +from ns_pb2 import *
> +from fs_pb2 import *
> +from ext_file_pb2 import *
> +from signalfd_pb2 import *
> +from ipc_desc_pb2 import *
> +from fdinfo_pb2 import *
> +from fifo_pb2 import *
> +from eventpoll_pb2 import *
> +from regfile_pb2 import *
> +from core_arm_pb2 import *
> +from sk_netlink_pb2 import *
> +from sk_inet_pb2 import *
> +from utsns_pb2 import *
> +from core_aarch64_pb2 import *
> +from eventfd_pb2 import *
> +from creds_pb2 import *
> +from vma_pb2 import *
> +from rpc_pb2 import *
> +from ghost_file_pb2 import *
> +
> +from magic import *
> +
> +# This dict is needed to identify what type of pb messages
> +# we need to read from image.
> +pb = {
> + IDS_MAGIC : task_kobj_ids_entry,
> + IRMAP_CACHE_MAGIC : irmap_cache_entry,
> + FS_MAGIC : fs_entry,
> + PAGEMAP_MAGIC : pagemap_head,
> + FIFO_MAGIC : fifo_entry,
> + EVENTFD_FILE_MAGIC : eventfd_file_entry,
> + PIPES_DATA_MAGIC : pipe_data_entry,
> + INETSK_MAGIC : inet_sk_entry,
> + TTY_FILES_MAGIC : tty_file_entry,
> + UTSNS_MAGIC : utsns_entry,
> + FDINFO_MAGIC : fdinfo_entry,
> + NS_FILES_MAGIC : ns_file_entry,
> + INOTIFY_WD_MAGIC : inotify_wd_entry,
> + EVENTPOLL_TFD_MAGIC : eventpoll_tfd_entry,
> + MNTS_MAGIC : mnt_entry,
> + VMAS_MAGIC : vma_entry,
> + IPCNS_SHM_MAGIC : ipc_shm_entry,
> + CORE_MAGIC : core_entry,
> + FILE_LOCKS_MAGIC : file_lock_entry,
> + EVENTPOLL_FILE_MAGIC: eventpoll_file_entry,
> + REMAP_FPATH_MAGIC : remap_file_path_entry,
> + SK_QUEUES_MAGIC : sk_packet_entry,
> + REG_FILES_MAGIC : reg_file_entry,
> + TUNFILE_MAGIC : tunfile_entry,
> + IPC_VAR_MAGIC : ipc_var_entry,
> + TTY_INFO_MAGIC : tty_info_entry,
> + PIPES_MAGIC : pipe_entry,
> + NETDEV_MAGIC : net_device_entry,
> + STATS_MAGIC : stats_entry,
> + RLIMIT_MAGIC : rlimit_entry,
> + POSIX_TIMERS_MAGIC : posix_timer_entry,
> + FANOTIFY_MARK_MAGIC : fanotify_mark_entry,
> + TIMERFD_MAGIC : timerfd_entry,
> + ITIMERS_MAGIC : itimer_entry,
> + CREDS_MAGIC : creds_entry,
> + SIGACT_MAGIC : sa_entry,
> + FIFO_DATA_MAGIC : pipe_data_entry,
> + TCP_STREAM_MAGIC : tcp_stream_entry,
> + PSTREE_MAGIC : pstree_entry,
> + UNIXSK_MAGIC : unix_sk_entry,
> + EXT_FILES_MAGIC : ext_file_entry,
> + SIGNAL_MAGIC : siginfo_entry,
> + INOTIFY_FILE_MAGIC : inotify_file_entry,
> + IPCNS_SEM_MAGIC : ipc_sem_entry,
> + NETLINK_SK_MAGIC : netlink_sk_entry,
> + FANOTIFY_FILE_MAGIC : fanotify_file_entry,
> + SIGNALFD_MAGIC : signalfd_entry,
> + MM_MAGIC : mm_entry,
> + CGROUP_MAGIC : cgroup_entry,
> + PACKETSK_MAGIC : packet_sock_entry,
> + GHOST_FILE_MAGIC : ghost_file_entry,
> + IPCNS_MSG_MAGIC : ipc_msg_entry,
> + INVENTORY_MAGIC : inventory_entry,
> +}
> diff --git a/tools/criu2text/magic-gen.py b/tools/criu2text/magic-gen.py
> new file mode 100644
> index 0000000..0bd9050
> --- /dev/null
> +++ b/tools/criu2text/magic-gen.py
> @@ -0,0 +1,44 @@
> +import os, sys
> +import struct
> +
> +# This program parses criu magic.h file and produces
> +# magic.py with all *_MAGIC constants except RAW and V1.
> +def main(argv):
> + if len(argv) != 2:
> + print("Usage: magic-gen.py path/to/image.h")
> + exit(1)
> +
> + magic_c_header = argv[1]
> + magic_py = "magic.py"
> +
> + out = open(magic_py, 'w+')
> +
> + magic = {}
> +
> + f = open(magic_c_header, 'r')
> + for line in f:
> + split = line.split()
> +
> + if len(split) < 3:
> + continue
> +
> + if not '#define' in split[0]:
> + continue
> +
> + key = split[1]
> + value = split[2]
> +
> + if value in magic:
> + value = magic[value]
> +
> + magic[key] = value
> +
> + if value == '0x0' or value == '1':
> + continue
> +
> + out.write(key+" = "+"\'"+value+"\'\n")
> +
> + out.close()
> +
> +if __name__ == "__main__":
> + main(sys.argv)
>
More information about the CRIU
mailing list