[CRIU] [PATCH 2/7] pycriu: add python package, v3

[Pavel Emelyanov]

> diff --git a/pycriu/.gitignore b/pycriu/.gitignore
> new file mode 100644
> index 0000000..8d503da
> --- /dev/null
> +++ b/pycriu/.gitignore
> @@ -0,0 +1,3 @@
> +*_pb2.py
> +*.pyc
> +rpc.py
> diff --git a/pycriu/Makefile b/pycriu/Makefile
> new file mode 100644
> index 0000000..135fe1c
> --- /dev/null
> +++ b/pycriu/Makefile
> @@ -0,0 +1,16 @@
> +all: images rpc.py
> +
> +.PHONY: all images clean
> +
> +images:
> +	$(Q) $(MAKE) -C images all
> +
> +# rpc_pb2.py doesn't depend on any other file, so
> +# it is safe to rename it, dropping ugly _pb2 suffix.
> +rpc.py:
> +	$(Q) protoc -I=$(SRC_DIR)/protobuf/ --python_out=./ $(SRC_DIR)/protobuf/$(@:.py=.proto)
> +	$(Q) mv $(@:.py=_pb2.py) $@

Why do we need rpc.py?

> +clean:
> +	$(Q) $(MAKE) -C images clean
> +	$(Q) $(RM) rpc.py *.pyc
> diff --git a/pycriu/__init__.py b/pycriu/__init__.py
> new file mode 100644
> index 0000000..5205973
> --- /dev/null
> +++ b/pycriu/__init__.py
> @@ -0,0 +1,2 @@
> +import rpc
> +import images
> diff --git a/pycriu/images/.gitignore b/pycriu/images/.gitignore
> new file mode 100644
> index 0000000..234bfe9
> --- /dev/null
> +++ b/pycriu/images/.gitignore
> @@ -0,0 +1,4 @@
> +*.pyc
> +*_pb2.py
> +magic.py
> +pb.py
> diff --git a/pycriu/images/Makefile b/pycriu/images/Makefile
> new file mode 100644
> index 0000000..6aa115e
> --- /dev/null
> +++ b/pycriu/images/Makefile
> @@ -0,0 +1,26 @@
> +all: pb.py protobuf magic.py
> +
> +.PHONY: all protobuf clean pb.py
> +
> +proto := $(filter-out $(SRC_DIR)/protobuf/rpc.proto, $(wildcard $(SRC_DIR)/protobuf/*.proto))
> +proto-py-modules := $(foreach m,$(proto),$(subst -,_,$(notdir $(m:.proto=_pb2))))
> +
> +# We don't need rpc_pb2.py here, as it is not related to the
> +# images.
> +# Unfortunately, we can't drop ugly _pb2 suffixes here, because
> +# some _pb2 files depend on others _pb2 files.
> +protobuf:
> +	$(Q) protoc -I=$(SRC_DIR)/protobuf --python_out=./ $(proto)
> +
> +magic.py: $(SRC_DIR)/scripts/magic-gen.py $(SRC_DIR)/include/magic.h
> +	$(E) "  GEN  " $@
> +	$(Q) python $^ $@
> +
> +pb.py: protobuf
> +	$(Q) echo "# Autogenerated. Do not edit!" > $@
> +	$(Q) for m in $(proto-py-modules); do \
> +		echo "from $$m import *" >> $@ ;\
> +	done
> +
> +clean:
> +	$(Q) $(RM) ./*_pb2.py ./*.pyc magic.py pb.py
> diff --git a/pycriu/images/__init__.py b/pycriu/images/__init__.py
> new file mode 100644
> index 0000000..379943b
> --- /dev/null
> +++ b/pycriu/images/__init__.py
> @@ -0,0 +1,3 @@
> +from magic import *
> +from images import *
> +from pb import *
> diff --git a/pycriu/images/images.py b/pycriu/images/images.py
> new file mode 100644
> index 0000000..afcaa87
> --- /dev/null
> +++ b/pycriu/images/images.py
> @@ -0,0 +1,208 @@
> +#!/bin/env python
> +import io
> +import google
> +import struct
> +import os
> +import sys
> +
> +from magic import *
> +from pb import *
> +
> +class _cr_desc:
> +	def __init__(self, magic, head, body='', single=False):

I'm not sure the single bit is needed. But what's needed is some
sign that image may have raw data inside. For example:

# ../crit convert --in dump/static/pipe00/6596/1/pipes-data.img 
Traceback (most recent call last):
  ...
  File "/usr/lib/python2.7/site-packages/google/protobuf/internal/decoder.py", line 685, in _RaiseInvalidWireType
    raise _DecodeError('Tag had invalid wire type.')
google.protobuf.message.DecodeError: Tag had invalid wire type.

That's because crit tries todecode raw pipe data as message.

> +		self.magic_name = magic
> +		self.magic = eval(magic+"_MAGIC")
> +		self.head_name = head
> +		self.head = eval(head)
> +		if single:
> +			self.body_name = ''
> +			self.body = lambda : None
> +		else:
> +			if body == '':
> +				body = head
> +			self.body_name = body
> +			self.body = eval(body)
> +
> +class criu_image_descs:
...
> +
> +	def get(self, magic):
> +		desc = None
> +		for d in self.descs:
> +			if d.magic == magic or d.magic_name == magic:
> +				desc = d
> +				break

Can we utilize python power to search objects in lists/hashes/sets
and not do it by hands?

> +		if desc == None:
> +			raise Exception("Unknown magic")
> +		else:
> +			return desc
> +
> +class criu_image:
> +	def __init__(self):
> +		self.desc	= None
> +		self.msgs	= []
> +		self.orig_type	= None
> +
> +	def ParseFromStringIMG(self, s):
> +		s = io.BytesIO(s)
> +		magic, = struct.unpack('i', s.read(4))
> +		magic = "0x{0:x}".format(magic)
> +
> +		descs = criu_image_descs()
> +		try:
> +			self.desc = descs.get(magic)
> +		except Exception as e:
> +			raise e
> +
> +		pb = self.desc.head()
> +		while pb != None:
> +			buf = s.read(4)
> +			if buf == '':
> +				break
> +			size, = struct.unpack('i', buf)
> +			pb.ParseFromString(s.read(size))
> +			self.msgs.append(pb)
> +			pb = self.desc.body()
> +
> +	def ParseFromStringTXT(self, s):
> +		s = io.BytesIO(s)
> +		magic = (s.readline())[1:-1]
> +
> +		descs = criu_image_descs()
> +		try:
> +			self.desc = descs.get(magic)
> +		except Exception as e:
> +			raise e
> +
> +		pb = self.desc.head()
> +		while pb != None:
> +			buf = s.readline()
> +			if buf == '':
> +				break
> +			if buf[0] != '#':
> +				raise Exception('# expected')
> +			pb_text = ''
> +			while True:
> +				buf = s.readline()
> +				if buf == '':
> +					break
> +				if buf[0] == '#':
> +					s.seek(-len(buf), 1)
> +					break
> +				elif buf[0] == '':
> +					raise Exception("data expected")
> +				else:
> +					pb_text += buf
> +			google.protobuf.text_format.Merge(pb_text, pb)
> +			self.msgs.append(pb)
> +			pb = self.desc.body()
> +
> +	def ParseFromString(self, s):

Let's make this call be ParseFromFile and pass here not
sting, but an object from which we can read().

> +		if s[0] == '#':
> +			self.orig_type = 'txt'
> +			self.ParseFromStringTXT(s)
> +		else:
> +			self.orig_type = 'img'
> +			self.ParseFromStringIMG(s)
> +
> +	def SerializeToStringIMG(self):
> +		s = io.BytesIO('')
> +		s.write(struct.pack('i', int(self.desc.magic, 16)))
> +
> +		for pb in self.msgs:
> +			pb_str = pb.SerializeToString()
> +			size = len(pb_str)
> +			s.write(struct.pack('i', size))
> +			s.write(pb_str)
> +		s.seek(0)

Why?

> +		return s.read()
> +
> +	def SerializeToStringTXT(self, nice=False):
> +		s = io.BytesIO('')
> +
> +		if nice:
> +			magic = self.desc.magic_name
> +			indent = 4
> +		else:
> +			magic = self.desc.magic
> +			indent = 0
> +
> +		s.write('#'+magic+'\n')
> +
> +		index = 0
> +		for pb in self.msgs:
> +			if nice:
> +				if index == 0:
> +					pb_name = self.desc.head_name
> +				else:
> +					pb_name = self.desc.body_name
> +			else:
> +				pb_name = ''
> +
> +			s.write('#'+str(index)+' '+pb_name+'\n')
> +			google.protobuf.text_format.PrintMessage(pb, s, indent=indent)
> +			index += 1
> +		s.seek(0)
> +		return s.read()
> +
> +	def SerializeToString(self, nice=False):
> +		if self.orig_type == 'img':
> +			return self.SerializeToStringTXT(nice=nice)
> +		else:
> +			return self.SerializeToStringIMG()
>