[CRIU] [PATCH 2/7] pycriu: add python package, v3

Tue Dec 2 06:05:26 PST 2014

On 12/02/2014 03:04 PM, Pavel Emelyanov wrote:
>> diff --git a/pycriu/.gitignore b/pycriu/.gitignore
>> new file mode 100644
>> index 0000000..8d503da
>> --- /dev/null
>> +++ b/pycriu/.gitignore
>> @@ -0,0 +1,3 @@
>> +*_pb2.py
>> +*.pyc
>> +rpc.py
>> diff --git a/pycriu/Makefile b/pycriu/Makefile
>> new file mode 100644
>> index 0000000..135fe1c
>> --- /dev/null
>> +++ b/pycriu/Makefile
>> @@ -0,0 +1,16 @@
>> +all: images rpc.py
>> +
>> +.PHONY: all images clean
>> +
>> +images:
>> +	$(Q) $(MAKE) -C images all
>> +
>> +# rpc_pb2.py doesn't depend on any other file, so
>> +# it is safe to rename it, dropping ugly _pb2 suffix.
>> +rpc.py:
>> +	$(Q) protoc -I=$(SRC_DIR)/protobuf/ --python_out=./ $(SRC_DIR)/protobuf/$(@:.py=.proto)
>> +	$(Q) mv $(@:.py=_pb2.py) $@
>
> Why do we need rpc.py?

Because it is handy to have all pythonic criu stuff in one
package called pycriu.

>
>> +clean:
>> +	$(Q) $(MAKE) -C images clean
>> +	$(Q) $(RM) rpc.py *.pyc
>> diff --git a/pycriu/__init__.py b/pycriu/__init__.py
>> new file mode 100644
>> index 0000000..5205973
>> --- /dev/null
>> +++ b/pycriu/__init__.py
>> @@ -0,0 +1,2 @@
>> +import rpc
>> +import images
>> diff --git a/pycriu/images/.gitignore b/pycriu/images/.gitignore
>> new file mode 100644
>> index 0000000..234bfe9
>> --- /dev/null
>> +++ b/pycriu/images/.gitignore
>> @@ -0,0 +1,4 @@
>> +*.pyc
>> +*_pb2.py
>> +magic.py
>> +pb.py
>> diff --git a/pycriu/images/Makefile b/pycriu/images/Makefile
>> new file mode 100644
>> index 0000000..6aa115e
>> --- /dev/null
>> +++ b/pycriu/images/Makefile
>> @@ -0,0 +1,26 @@
>> +all: pb.py protobuf magic.py
>> +
>> +.PHONY: all protobuf clean pb.py
>> +
>> +proto := $(filter-out $(SRC_DIR)/protobuf/rpc.proto, $(wildcard $(SRC_DIR)/protobuf/*.proto))
>> +proto-py-modules := $(foreach m,$(proto),$(subst -,_,$(notdir $(m:.proto=_pb2))))
>> +
>> +# We don't need rpc_pb2.py here, as it is not related to the
>> +# images.
>> +# Unfortunately, we can't drop ugly _pb2 suffixes here, because
>> +# some _pb2 files depend on others _pb2 files.
>> +protobuf:
>> +	$(Q) protoc -I=$(SRC_DIR)/protobuf --python_out=./ $(proto)
>> +
>> +magic.py: $(SRC_DIR)/scripts/magic-gen.py $(SRC_DIR)/include/magic.h
>> +	$(E) "  GEN  " $@
>> +	$(Q) python $^ $@
>> +
>> +pb.py: protobuf
>> +	$(Q) echo "# Autogenerated. Do not edit!" > $@
>> +	$(Q) for m in $(proto-py-modules); do \
>> +		echo "from $$m import *" >> $@ ;\
>> +	done
>> +
>> +clean:
>> +	$(Q) $(RM) ./*_pb2.py ./*.pyc magic.py pb.py
>> diff --git a/pycriu/images/__init__.py b/pycriu/images/__init__.py
>> new file mode 100644
>> index 0000000..379943b
>> --- /dev/null
>> +++ b/pycriu/images/__init__.py
>> @@ -0,0 +1,3 @@
>> +from magic import *
>> +from images import *
>> +from pb import *
>> diff --git a/pycriu/images/images.py b/pycriu/images/images.py
>> new file mode 100644
>> index 0000000..afcaa87
>> --- /dev/null
>> +++ b/pycriu/images/images.py
>> @@ -0,0 +1,208 @@
>> +#!/bin/env python
>> +import io
>> +import google
>> +import struct
>> +import os
>> +import sys
>> +
>> +from magic import *
>> +from pb import *
>> +
>> +class _cr_desc:
>> +	def __init__(self, magic, head, body='', single=False):
>
> I'm not sure the single bit is needed. But what's needed is some
> sign that image may have raw data inside. For example:
>
> # ../crit convert --in dump/static/pipe00/6596/1/pipes-data.img
> Traceback (most recent call last):
>    ...
>    File "/usr/lib/python2.7/site-packages/google/protobuf/internal/decoder.py", line 685, in _RaiseInvalidWireType
>      raise _DecodeError('Tag had invalid wire type.')
> google.protobuf.message.DecodeError: Tag had invalid wire type.
>
> That's because crit tries todecode raw pipe data as message.
>

Not sure about single bit, but will try to optimize it.
Yes, you are right about raw data. Will fix.

>> +		self.magic_name = magic
>> +		self.magic = eval(magic+"_MAGIC")
>> +		self.head_name = head
>> +		self.head = eval(head)
>> +		if single:
>> +			self.body_name = ''
>> +			self.body = lambda : None
>> +		else:
>> +			if body == '':
>> +				body = head
>> +			self.body_name = body
>> +			self.body = eval(body)
>> +
>> +class criu_image_descs:
> ...
>> +
>> +	def get(self, magic):
>> +		desc = None
>> +		for d in self.descs:
>> +			if d.magic == magic or d.magic_name == magic:
>> +				desc = d
>> +				break
>
> Can we utilize python power to search objects in lists/hashes/sets
> and not do it by hands?
>

I guess we can.

>> +		if desc == None:
>> +			raise Exception("Unknown magic")
>> +		else:
>> +			return desc
>> +
>> +class criu_image:
>> +	def __init__(self):
>> +		self.desc	= None
>> +		self.msgs	= []
>> +		self.orig_type	= None
>> +
>> +	def ParseFromStringIMG(self, s):
>> +		s = io.BytesIO(s)
>> +		magic, = struct.unpack('i', s.read(4))
>> +		magic = "0x{0:x}".format(magic)
>> +
>> +		descs = criu_image_descs()
>> +		try:
>> +			self.desc = descs.get(magic)
>> +		except Exception as e:
>> +			raise e
>> +
>> +		pb = self.desc.head()
>> +		while pb != None:
>> +			buf = s.read(4)
>> +			if buf == '':
>> +				break
>> +			size, = struct.unpack('i', buf)
>> +			pb.ParseFromString(s.read(size))
>> +			self.msgs.append(pb)
>> +			pb = self.desc.body()
>> +
>> +	def ParseFromStringTXT(self, s):
>> +		s = io.BytesIO(s)
>> +		magic = (s.readline())[1:-1]
>> +
>> +		descs = criu_image_descs()
>> +		try:
>> +			self.desc = descs.get(magic)
>> +		except Exception as e:
>> +			raise e
>> +
>> +		pb = self.desc.head()
>> +		while pb != None:
>> +			buf = s.readline()
>> +			if buf == '':
>> +				break
>> +			if buf[0] != '#':
>> +				raise Exception('# expected')
>> +			pb_text = ''
>> +			while True:
>> +				buf = s.readline()
>> +				if buf == '':
>> +					break
>> +				if buf[0] == '#':
>> +					s.seek(-len(buf), 1)
>> +					break
>> +				elif buf[0] == '':
>> +					raise Exception("data expected")
>> +				else:
>> +					pb_text += buf
>> +			google.protobuf.text_format.Merge(pb_text, pb)
>> +			self.msgs.append(pb)
>> +			pb = self.desc.body()
>> +
>> +	def ParseFromString(self, s):
>
> Let's make this call be ParseFromFile and pass here not
> sting, but an object from which we can read().
>

It might be useful to have both ParseFromString and ParseFromFile.
I'll take a look.

>> +		if s[0] == '#':
>> +			self.orig_type = 'txt'
>> +			self.ParseFromStringTXT(s)
>> +		else:
>> +			self.orig_type = 'img'
>> +			self.ParseFromStringIMG(s)
>> +
>> +	def SerializeToStringIMG(self):
>> +		s = io.BytesIO('')
>> +		s.write(struct.pack('i', int(self.desc.magic, 16)))
>> +
>> +		for pb in self.msgs:
>> +			pb_str = pb.SerializeToString()
>> +			size = len(pb_str)
>> +			s.write(struct.pack('i', size))
>> +			s.write(pb_str)
>> +		s.seek(0)
>
> Why?
>

Yeah, it is ugly =). I'll fix it.
Thanks.

>> +		return s.read()
>> +
>> +	def SerializeToStringTXT(self, nice=False):
>> +		s = io.BytesIO('')
>> +
>> +		if nice:
>> +			magic = self.desc.magic_name
>> +			indent = 4
>> +		else:
>> +			magic = self.desc.magic
>> +			indent = 0
>> +
>> +		s.write('#'+magic+'\n')
>> +
>> +		index = 0
>> +		for pb in self.msgs:
>> +			if nice:
>> +				if index == 0:
>> +					pb_name = self.desc.head_name
>> +				else:
>> +					pb_name = self.desc.body_name
>> +			else:
>> +				pb_name = ''
>> +
>> +			s.write('#'+str(index)+' '+pb_name+'\n')
>> +			google.protobuf.text_format.PrintMessage(pb, s, indent=indent)
>> +			index += 1
>> +		s.seek(0)
>> +		return s.read()
>> +
>> +	def SerializeToString(self, nice=False):
>> +		if self.orig_type == 'img':
>> +			return self.SerializeToStringTXT(nice=nice)
>> +		else:
>> +			return self.SerializeToStringIMG()
>>
>