[CRIU] [PATCH 2/4] tools: add criu2text
Ruslan Kuprieiev
kupruser at gmail.com
Tue Sep 30 02:08:56 PDT 2014
On 30.09.2014 11:12, Pavel Emelyanov wrote:
> On 09/19/2014 05:53 PM, Ruslan Kuprieiev wrote:
>> criu2text.py handles cmdline opts, opens files and reads\writes them
>> properly. images.py has a dict with MAGIC:pb_entry_type to specify what
>> type of pb entry we should read from image with magic MAGIC.
>> magic-gen.py parses include/magic.h to get all MAGIC values except
>> RAW and V1, because criu2text can't handle them for now.
> OK, I like the code, but let's rework the way tool works a little bit.
>
> 1. Right now, AFAIK, the tool turns one .img file into a set of .txt
> ones. Let's make tool convert _one_ file to _one_ file, e.g. core.img
> into core.txt. And the syntax should look like this:
>
> criu2text -i|--in <filename> -o|--out <filename>
>
> If <filename> == "-", then stdin/stdout is used.
>
> The conversion from .img to .txt would work automatically, the conversion
> back would have to know what kind of image we're working on. For this
> the tool can write the magic value as the first line (in a form of a
> comment if it makes sense).
In early criu2json it worked just as you described =).
In json it was easy to create a new message, but in pbtext it is a bit less
smooth. Well, we can separate msgs in txt file by special comments, e.g:
#0x12345 magic
#0 №
MSG0 pb entry
#1 №
MSG1 pb entry
#2
MSG2
........
Does it look good to you?
Or maybe we should drop № and just leave # as a separator, e.g.:
#0x12345
MSG0
#
MSG1
#
MSG2
?
>
> 2. I propose to name it "crit" -- CRiu Image Tool :)
I like it.
> 3. Can we have all auxiliary stuff in the scripts/ dir? E.g. the magic.h
> parsing script is such
Sure.
> 4. I'd appreciate if the tool itself sits in the sources top directory.
Ok.
> What do you think?
>
>> Signed-off-by: Ruslan Kuprieiev <kupruser at gmail.com>
>> ---
>> tools/criu2text/.gitignore | 4 ++
>> tools/criu2text/Makefile | 15 +++++
>> tools/criu2text/criu2text | 156 +++++++++++++++++++++++++++++++++++++++++++
>> tools/criu2text/images.py | 112 +++++++++++++++++++++++++++++++
>> tools/criu2text/magic-gen.py | 44 ++++++++++++
>> 5 files changed, 331 insertions(+)
>> create mode 100644 tools/criu2text/.gitignore
>> create mode 100644 tools/criu2text/Makefile
>> create mode 100755 tools/criu2text/criu2text
>> create mode 100644 tools/criu2text/images.py
>> create mode 100644 tools/criu2text/magic-gen.py
>>
>> diff --git a/tools/criu2text/.gitignore b/tools/criu2text/.gitignore
>> new file mode 100644
>> index 0000000..550f4b1
>> --- /dev/null
>> +++ b/tools/criu2text/.gitignore
>> @@ -0,0 +1,4 @@
>> +*.pyc
>> +*_pb2.py
>> +magic.py
>> +__pycache__
>> diff --git a/tools/criu2text/Makefile b/tools/criu2text/Makefile
>> new file mode 100644
>> index 0000000..986f426
>> --- /dev/null
>> +++ b/tools/criu2text/Makefile
>> @@ -0,0 +1,15 @@
>> +.PHONY: all clean protobuf
>> +
>> +PROTOBUF_DIR = ../../protobuf
>> +MAGIC_C_HEADER = ../../include/magic.h
>> +
>> +all: magic.py protobuf
>> +
>> +magic.py: $(MAGIC_C_HEADER) protobuf
>> + python magic-gen.py $(MAGIC_C_HEADER)
>> +
>> +protobuf:
>> + protoc -I=$(PROTOBUF_DIR) --python_out=./ $(PROTOBUF_DIR)/*.proto
>> +
>> +clean:
>> + rm -rf *.pyc *_pb2.py __pycache__ magic.py
>> diff --git a/tools/criu2text/criu2text b/tools/criu2text/criu2text
>> new file mode 100755
>> index 0000000..1dfaae9
>> --- /dev/null
>> +++ b/tools/criu2text/criu2text
>> @@ -0,0 +1,156 @@
>> +#!/bin/env python
>> +import os, sys
>> +import struct
>> +import google
>> +import shutil
>> +import argparse
>> +
>> +import images
>> +
>> +def img2text(img_name):
>> + fin = open(img_name, 'r')
>> + # Read magic cookie to identify criu image
>> + buf = fin.read(4)
>> +
>> + # Convert cookie into "0x12345.." string to
>> + # use it as a key in criu.magic dict
>> + cookie, = struct.unpack('i', buf)
>> + cookie = "0x{0:x}".format(cookie)
>> +
>> + base = os.path.basename(img_name)
>> +
>> + # Write cookie into file named magic
>> + with open(base+'.magic', 'w+') as fout:
>> + fout.write(cookie)
>> +
>> + if cookie in images.pb.keys():
>> + pb = images.pb[cookie]()
>> + num = 0
>> +
>> + while True:
>> + # Read size of the following pb message
>> + buf = fin.read(4)
>> + if buf == '':
>> + break
>> +
>> + size, = struct.unpack('i', buf)
>> +
>> + # Read pb message
>> + pb.ParseFromString(fin.read(size))
>> +
>> + # Write text representation of
>> + # pb message into num file
>> + with open(base+'.'+str(num), 'w+') as fout:
>> + google.protobuf.text_format.PrintMessage(pb, fout)
>> +
>> + num += 1
>> + # pagemap.img is a special one. It starts with
>> + # pagemap_head message and is followed by an
>> + # array of pagemap_entry's.
>> + if cookie == images.PAGEMAP_MAGIC:
>> + pb = images.pagemap_entry()
>> +
>> + fin.close()
>> + else:
>> + print("not criu image or unknown format")
>> + fin.close()
>> + exit(1)
>> +
>> +def text2img(img_name):
>> + base = os.path.basename(img_name)
>> +
>> + fout = open(base, 'w+')
>> +
>> + # Open magic file to get magic cookie
>> + with open(img_name+'.magic', 'r') as fin:
>> + cookie = fin.read()
>> +
>> + if cookie in images.pb.keys():
>> + # Write magic cookie into img file
>> + fout.write(struct.pack('i', int(cookie,16)))
>> +
>> + # Corresponding function that we need to call
>> + # to init pb message
>> + pb_init_function = images.pb[cookie]
>> +
>> + num = 0
>> + while True:
>> + if not os.path.isfile(img_name+'.'+str(num)):
>> + break
>> +
>> + pb = pb_init_function()
>> +
>> + # Read pb message int text format from
>> + # file num
>> + with open(img_name+'.'+str(num), 'r') as fin:
>> + pb_text = fin.read()
>> +
>> + # Parse pb from text format and write
>> + # it to img file
>> + google.protobuf.text_format.Merge(pb_text, pb)
>> + size = len(pb.SerializeToString())
>> + fout.write(struct.pack('i', size))
>> + fout.write(pb.SerializeToString())
>> +
>> + num += 1
>> + # pagemap.img is a special one. It starts with
>> + # pagemap_head message and is followed by an
>> + # array of pagemap_entry's.
>> + if cookie == images.PAGEMAP_MAGIC:
>> + pb_init_function = images.pagemap_entry
>> +
>> + fout.close()
>> + else:
>> + print("unknown format")
>> + fout.close()
>> + exit(1)
>> +
>> +def main():
>> + desc = "Convert criu images to/from set of files in human-readable format"
>> + epi = "Examples:\n"\
>> + "1) Convert ./core-1234.img to set of text files (core-1234.img.magic,\n"\
>> + " core-1234.img.0) and place them at text_imgs_dir.\n"\
>> + "\n"\
>> + " $ criu2text to-text core-1234.img -o text_imgs_dir\n"\
>> + " $ ls text_imgs_dir\n"\
>> + " core-1234.img.magic\n"\
>> + " core-1234.img.0\n"\
>> + "\n"\
>> + "2) Convert text_imgs_dir/core-1234.img.magic, text_imgs_dir/core-1234.img.0\n"\
>> + " (note common base text_imgs_dir/core-1234.img, which we're specifying)\n"\
>> + " to criu img format and place it at imgs_dir.\n"\
>> + "\n"\
>> + " $ criu2text to-img text_imgs_dir/core-1234.img -o imgs_dir\n"\
>> + " $ ls imgs_dir\n"\
>> + " core-1234.img\n"\
>> + "\n"
>> +
>> + parser = argparse.ArgumentParser(description = desc,
>> + epilog = epi,
>> + formatter_class = argparse.RawTextHelpFormatter)
>> + parser.add_argument('command',
>> + choices = ['to-text', 'to-img'],
>> + help = 'to what type convert input file(s)')
>> + parser.add_argument('image',
>> + help = 'path to criu image or base path to criu image files in '\
>> + 'human-readable format (e.g. ./imgs/core-1234.img)')
>> + parser.add_argument('-o',
>> + '--out',
>> + default = './',
>> + help = 'directory where output file(s) should be placed')
>> +
>> + opts = vars(parser.parse_args())
>> +
>> + opts['image'] = os.path.abspath(opts['image'])
>> + opts['out'] = os.path.abspath(opts['out'])
>> +
>> + os.chdir(opts['out'])
>> +
>> + if opts['command'] == 'to-text':
>> + img2text(opts['image'])
>> +
>> + if opts['command'] == 'to-img':
>> + text2img(opts['image'])
>> +
>> +if __name__ == "__main__":
>> + main()
>> diff --git a/tools/criu2text/images.py b/tools/criu2text/images.py
>> new file mode 100644
>> index 0000000..99bc088
>> --- /dev/null
>> +++ b/tools/criu2text/images.py
>> @@ -0,0 +1,112 @@
>> +from stats_pb2 import *
>> +from timer_pb2 import *
>> +from ipc_sem_pb2 import *
>> +from core_pb2 import *
>> +from core_x86_pb2 import *
>> +from remap_file_path_pb2 import *
>> +from fsnotify_pb2 import *
>> +from inventory_pb2 import *
>> +from tcp_stream_pb2 import *
>> +from mnt_pb2 import *
>> +from packet_sock_pb2 import *
>> +from pipe_data_pb2 import *
>> +from ipc_var_pb2 import *
>> +from sa_pb2 import *
>> +from tun_pb2 import *
>> +from fown_pb2 import *
>> +from tty_pb2 import *
>> +from rlimit_pb2 import *
>> +from ipc_shm_pb2 import *
>> +from file_lock_pb2 import *
>> +from sk_unix_pb2 import *
>> +from sk_packet_pb2 import *
>> +from pagemap_pb2 import *
>> +from pipe_pb2 import *
>> +from pstree_pb2 import *
>> +from sk_opts_pb2 import *
>> +from siginfo_pb2 import *
>> +from cgroup_pb2 import *
>> +from fh_pb2 import *
>> +from ipc_msg_pb2 import *
>> +from mm_pb2 import *
>> +from netdev_pb2 import *
>> +from timerfd_pb2 import *
>> +from ns_pb2 import *
>> +from fs_pb2 import *
>> +from ext_file_pb2 import *
>> +from signalfd_pb2 import *
>> +from ipc_desc_pb2 import *
>> +from fdinfo_pb2 import *
>> +from fifo_pb2 import *
>> +from eventpoll_pb2 import *
>> +from regfile_pb2 import *
>> +from core_arm_pb2 import *
>> +from sk_netlink_pb2 import *
>> +from sk_inet_pb2 import *
>> +from utsns_pb2 import *
>> +from core_aarch64_pb2 import *
>> +from eventfd_pb2 import *
>> +from creds_pb2 import *
>> +from vma_pb2 import *
>> +from rpc_pb2 import *
>> +from ghost_file_pb2 import *
>> +
>> +from magic import *
>> +
>> +# This dict is needed to identify what type of pb messages
>> +# we need to read from image.
>> +pb = {
>> + IDS_MAGIC : task_kobj_ids_entry,
>> + IRMAP_CACHE_MAGIC : irmap_cache_entry,
>> + FS_MAGIC : fs_entry,
>> + PAGEMAP_MAGIC : pagemap_head,
>> + FIFO_MAGIC : fifo_entry,
>> + EVENTFD_FILE_MAGIC : eventfd_file_entry,
>> + PIPES_DATA_MAGIC : pipe_data_entry,
>> + INETSK_MAGIC : inet_sk_entry,
>> + TTY_FILES_MAGIC : tty_file_entry,
>> + UTSNS_MAGIC : utsns_entry,
>> + FDINFO_MAGIC : fdinfo_entry,
>> + NS_FILES_MAGIC : ns_file_entry,
>> + INOTIFY_WD_MAGIC : inotify_wd_entry,
>> + EVENTPOLL_TFD_MAGIC : eventpoll_tfd_entry,
>> + MNTS_MAGIC : mnt_entry,
>> + VMAS_MAGIC : vma_entry,
>> + IPCNS_SHM_MAGIC : ipc_shm_entry,
>> + CORE_MAGIC : core_entry,
>> + FILE_LOCKS_MAGIC : file_lock_entry,
>> + EVENTPOLL_FILE_MAGIC: eventpoll_file_entry,
>> + REMAP_FPATH_MAGIC : remap_file_path_entry,
>> + SK_QUEUES_MAGIC : sk_packet_entry,
>> + REG_FILES_MAGIC : reg_file_entry,
>> + TUNFILE_MAGIC : tunfile_entry,
>> + IPC_VAR_MAGIC : ipc_var_entry,
>> + TTY_INFO_MAGIC : tty_info_entry,
>> + PIPES_MAGIC : pipe_entry,
>> + NETDEV_MAGIC : net_device_entry,
>> + STATS_MAGIC : stats_entry,
>> + RLIMIT_MAGIC : rlimit_entry,
>> + POSIX_TIMERS_MAGIC : posix_timer_entry,
>> + FANOTIFY_MARK_MAGIC : fanotify_mark_entry,
>> + TIMERFD_MAGIC : timerfd_entry,
>> + ITIMERS_MAGIC : itimer_entry,
>> + CREDS_MAGIC : creds_entry,
>> + SIGACT_MAGIC : sa_entry,
>> + FIFO_DATA_MAGIC : pipe_data_entry,
>> + TCP_STREAM_MAGIC : tcp_stream_entry,
>> + PSTREE_MAGIC : pstree_entry,
>> + UNIXSK_MAGIC : unix_sk_entry,
>> + EXT_FILES_MAGIC : ext_file_entry,
>> + SIGNAL_MAGIC : siginfo_entry,
>> + INOTIFY_FILE_MAGIC : inotify_file_entry,
>> + IPCNS_SEM_MAGIC : ipc_sem_entry,
>> + NETLINK_SK_MAGIC : netlink_sk_entry,
>> + FANOTIFY_FILE_MAGIC : fanotify_file_entry,
>> + SIGNALFD_MAGIC : signalfd_entry,
>> + MM_MAGIC : mm_entry,
>> + CGROUP_MAGIC : cgroup_entry,
>> + PACKETSK_MAGIC : packet_sock_entry,
>> + GHOST_FILE_MAGIC : ghost_file_entry,
>> + IPCNS_MSG_MAGIC : ipc_msg_entry,
>> + INVENTORY_MAGIC : inventory_entry,
>> +}
>> diff --git a/tools/criu2text/magic-gen.py b/tools/criu2text/magic-gen.py
>> new file mode 100644
>> index 0000000..0bd9050
>> --- /dev/null
>> +++ b/tools/criu2text/magic-gen.py
>> @@ -0,0 +1,44 @@
>> +import os, sys
>> +import struct
>> +
>> +# This program parses criu magic.h file and produces
>> +# magic.py with all *_MAGIC constants except RAW and V1.
>> +def main(argv):
>> + if len(argv) != 2:
>> + print("Usage: magic-gen.py path/to/image.h")
>> + exit(1)
>> +
>> + magic_c_header = argv[1]
>> + magic_py = "magic.py"
>> +
>> + out = open(magic_py, 'w+')
>> +
>> + magic = {}
>> +
>> + f = open(magic_c_header, 'r')
>> + for line in f:
>> + split = line.split()
>> +
>> + if len(split) < 3:
>> + continue
>> +
>> + if not '#define' in split[0]:
>> + continue
>> +
>> + key = split[1]
>> + value = split[2]
>> +
>> + if value in magic:
>> + value = magic[value]
>> +
>> + magic[key] = value
>> +
>> + if value == '0x0' or value == '1':
>> + continue
>> +
>> + out.write(key+" = "+"\'"+value+"\'\n")
>> +
>> + out.close()
>> +
>> +if __name__ == "__main__":
>> + main(sys.argv)
>>
More information about the CRIU
mailing list