[CRIU] Re: [PATCH 3/5] protobuf: Add protobuf engine

[Stanislav Kinsbursky]

04.07.2012 22:05, Cyrill Gorcunov пишет:
> Since protobuf is not that simple to deal with we provide
> helpers to work with it
>
>   - pbimg_write_obj to write PB objects
>   - pbimg_read_obj_eof to read PB objects
>
> Note the plural case of "objects" here. While in previous commit
> descibes the ability to use pretty flexible structure of objects
> which can be stored on disk, at moment most simple way is used
> to not crash overall crtools code but make a transition to new
> format via small steps.
>
> So at moment the img files with PB objects do have the following
> structure
>
>          | 4 bytes magic number as a header
>          +---
>          | 24 bytes message obj_hdr
>          +---
>          | X bytes protobuf-object
>          +---
>          ...
>
>          and so on
>
> Ie it's not that efficient but enough for this stage.
>
> Because messages obj_hdr and img_hdr and fixed in size
> and immutable we provide a fast way to encode/decode them
> via pbimg_write_img_hdr/pbimg_write_obj_hdr helpers. The
> read speed up achieved via img_hrd_t and obj_hdr_t structures
> which are 1:1 binary representation of PB ABI and will never
> change.
>
> Signed-off-by: Cyrill Gorcunov <gorcunov at openvz.org>
> ---
>   Makefile           |    1 +
>   include/protobuf.h |   83 ++++++++++++++++++++++++++++++++++++++++++++++++++++
>   protobuf.c         |   38 ++++++++++++++++++++++++
>   3 files changed, 122 insertions(+), 0 deletions(-)
>   create mode 100644 include/protobuf.h
>   create mode 100644 protobuf.c
>
> diff --git a/Makefile b/Makefile
> index db2e6f7..9905c3d 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -59,6 +59,7 @@ OBJS		+= eventpoll.o
>   OBJS		+= mount.o
>   OBJS		+= inotify.o
>   OBJS		+= pstree.o
> +OBJS		+= protobuf.o
>   
>   PROTOBUF-LIB	:= protobuf/protobuf-lib.o
>   
> diff --git a/include/protobuf.h b/include/protobuf.h
> new file mode 100644
> index 0000000..74b1bb1
> --- /dev/null
> +++ b/include/protobuf.h
> @@ -0,0 +1,83 @@
> +#ifndef PROTOBUF_H__
> +#define PROTOBUF_H__
> +
> +#include "types.h"
> +#include "compiler.h"
> +
> +#include "syscall.h"
> +
> +typedef struct {
> +	u8	___tag_pad1;
> +	u64	magic;
> +	u8	___tag_pad2;
> +	u64	version;
> +	u8	___tag_pad3;
> +	u64	flags;
> +} __packed img_hrd_t;
> +

Could you, please, explain, what for are these ____tag_pad's are required?

> +typedef struct {
> +	u8	___tag_pad1;
> +	u32	type;
> +	u8	___tag_pad2;
> +	u64	pksize;
> +} __packed obj_hdr_t;
> +
> +#define pbimg_read_img_hdr(fd, h)	read_img(fd, h)
> +#define pbimg_read_obj_hdr(fd, h)	read_img(fd, h)
> +
> +#define pbimg_read_img_hdr_eof(fd, h)	read_img_eof(fd, h)
> +#define pbimg_read_obj_hdr_eof(fd, h)	read_img_eof(fd, h)
> +
> +extern int pbimg_write_img_hdr(int fd, img_hrd_t *h);
> +extern int pbimg_write_obj_hdr(int fd, obj_hdr_t *h);
> +
> +#define pbimg_write_obj(__fd, __object, __type, __get_packed_size, __pack)			\
> +({												\
> +	u8  ___buf[KILO(16)];									\
> +	u32 ___size = __get_packed_size(__object);						\
> +	int ___ret = -1;									\
> +												\
> +	if (___size <= sizeof(___buf)) {							\
> +		obj_hdr_t ___ohdr = { .type = __type, .pksize = ___size };			\
> +		___ret = pbimg_write_obj_hdr(__fd, &___ohdr);					\
> +		if (!___ret) {									\
> +			__pack(__object, ___buf);						\
> +			___ret = write_img(__fd, &___size);					\
> +			if (!___ret)								\
> +				___ret = write_img_buf(__fd, ___buf, ___size);			\
> +		}										\
> +	} else											\
> +		pr_err("Protobuf entry passed is too big (%dK)",				\
> +		       (int)KBYTES(___size));							\
> +	___ret;											\
> +})
> +
> +/* Don't forget to free unpacked data on return */
> +#define pbimg_read_obj_eof(__fd, __object, __type, __unpack)					\
> +({												\
> +	u8  ___buf[KILO(16)];									\
> +	obj_hdr_t ___ohdr;									\
> +	u32 ___size;										\
> +	int ___ret;										\
> +												\
> +	*(__object) = NULL;									\
> +	___ret = pbimg_read_obj_hdr_eof(__fd, &___ohdr);						\
> +	if (___ret > 0) {									\
> +		if (__type == ___ohdr.type) {							\
> +			___ret = read_img_eof(__fd, &___size);					\
> +			if (___ret > 0) {							\
> +				if (___size <= sizeof(___buf))					\
> +					___ret = read_img_buf(__fd, ___buf, ___size);		\
> +					if (___ret == 1)					\
> +						*(__object) = __unpack(NULL, ___size, ___buf);	\
> +			}									\
> +		} else {									\
> +			pr_err("Corrupted image, type mismatch (%x %x)\n",			\
> +				(int)__type, (int)___ohdr.type);				\
> +			___ret = -1;								\
> +		}										\
> +	}											\
> +	___ret;											\
> +})
> +
> +#endif /* PROTOBUF_H__ */
> diff --git a/protobuf.c b/protobuf.c
> new file mode 100644
> index 0000000..17b2052
> --- /dev/null
> +++ b/protobuf.c
> @@ -0,0 +1,38 @@
> +#include <unistd.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
> +#include <stdlib.h>
> +
> +#include "compiler.h"
> +#include "types.h"
> +#include "log.h"
> +#include "util.h"
> +
> +#include "protobuf.h"
> +
> +enum {
> +	CR_PROTOBUF_C_WIRE_TYPE_VARINT,
> +	CR_PROTOBUF_C_WIRE_TYPE_64BIT,
> +	CR_PROTOBUF_C_WIRE_TYPE_LENGTH_PREFIXED,
> +	CR_PROTOBUF_C_WIRE_TYPE_START_GROUP,
> +	CR_PROTOBUF_C_WIRE_TYPE_END_GROUP,
> +	CR_PROTOBUF_C_WIRE_TYPE_32BIT
> +};
> +
> +int pbimg_write_img_hdr(int fd, img_hrd_t *h)
> +{
> +	h->___tag_pad1 = (1 << 3) | CR_PROTOBUF_C_WIRE_TYPE_64BIT;
> +	h->___tag_pad2 = (2 << 3) | CR_PROTOBUF_C_WIRE_TYPE_64BIT;
> +	h->___tag_pad3 = (3 << 3) | CR_PROTOBUF_C_WIRE_TYPE_64BIT;
> +
> +	return write_img(fd, h);
> +}
> +
> +int pbimg_write_obj_hdr(int fd, obj_hdr_t *h)
> +{
> +	h->___tag_pad1 = (1 << 3) | CR_PROTOBUF_C_WIRE_TYPE_32BIT;
> +	h->___tag_pad2 = (2 << 3) | CR_PROTOBUF_C_WIRE_TYPE_64BIT;
> +
> +	return write_img(fd, h);
> +}