[CRIU] [PATCH 2/4] fsnotify: Flush own generated events

[Cyrill Gorcunov]

During restore of fsnotify objects we may generate own
events (in particular when we remove last link to hardlinked
remap entry) but userspace application should not see any
event coming from criu itself.

For this sake at the end of fsnotify object restore read
flush events queue.

I'm not really yet sure that this will solve the problem
in general, eventually it might end up that we need to
restore fsnotify objects after _all_ files being restored,
i.e. at very late stage of restore. But this will require
more code changes so I stick with simplier solution first
(and anyway we will need the helpers implemented in this
 patch).

Signed-off-by: Cyrill Gorcunov <gorcunov at openvz.org>
---
 fsnotify.c | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 114 insertions(+), 4 deletions(-)

diff --git a/fsnotify.c b/fsnotify.c
index 1c348ce5a0d0..35f7fb930fbf 100644
--- a/fsnotify.c
+++ b/fsnotify.c
@@ -461,6 +461,85 @@ err:
 	return path;
 }
 
+static void decode_event_mask(char *buf, size_t size, u32 mask)
+{
+	static const char *names[sizeof(u32) * 8] = {
+		[ 0]	= "IN_ACCESS",
+		[ 1]	= "IN_MODIFY",
+		[ 2]	= "IN_ATTRIB",
+		[ 3]	= "IN_CLOSE_WRITE",
+		[ 4]	= "IN_CLOSE_NOWRITE",
+		[ 5]	= "IN_OPEN",
+		[ 6]	= "IN_MOVED_FROM",
+		[ 7]	= "IN_MOVED_TO",
+		[ 8]	= "IN_CREATE",
+		[ 9]	= "IN_DELETE",
+		[10]	= "IN_DELETE_SELF",
+		[11]	= "IN_MOVE_SELF",
+
+		[13]	= "IN_UNMOUNT",
+		[14]	= "IN_Q_OVERFLOW",
+		[15]	= "IN_IGNORED",
+
+		[24]	= "IN_ONLYDIR",
+		[25]	= "IN_DONT_FOLLOW",
+		[26]	= "IN_EXCL_UNLINK",
+
+		[29]	= "IN_MASK_ADD",
+		[30]	= "IN_ISDIR",
+		[31]	= "IN_ONESHOT",
+	};
+
+	size_t i, j;
+
+	memzero(buf, size);
+	for (i = 0, j = 0; i < (sizeof(u32) * 8) && j < size; i++) {
+		if (!(mask & (1u << i)))
+			continue;
+		if (j)
+			j += snprintf(&buf[j], size - j, " | %s", names[i]);
+		else
+			j += snprintf(&buf[j], size - j, "%s", names[i]);
+	}
+}
+
+static int flush_inotify_events(int inotify_fd, unsigned int id)
+{
+	struct inotify_event *event;
+	char buf[sizeof(*event) + PATH_MAX];
+	int ret, off;
+
+	/*
+	 * NOTE: It's up to a caller to make @inotify_fd being
+	 * created with O_NONBLOCK if needed.
+	 */
+	while (1) {
+		ret = read(inotify_fd, buf, sizeof(buf));
+		if (ret < 0) {
+			if (errno != EAGAIN) {
+				pr_perror("Can't read events queue for 0x%08x", id);
+				return -1;
+			} else
+				return 0;
+		} else if (ret == 0)
+			break;
+
+		if (pr_quelled(LOG_DEBUG))
+			continue;
+
+		for (off = 0; off < ret; off += sizeof(*event) + event->len) {
+			char emask[128];
+
+			event = (void *)(buf + off);
+			decode_event_mask(emask, sizeof(emask), event->mask);
+			pr_debug("\t0x%08x: flushing event %#10x -> %s\n",
+				 id, event->mask, emask);
+		}
+	}
+
+	return ret;
+}
+
 static int restore_one_inotify(int inotify_fd, struct fsnotify_mark_info *info)
 {
 	InotifyWdEntry *iwe = info->iwe;
@@ -580,7 +659,14 @@ static int open_inotify_fd(struct file_desc *d)
 
 	info = container_of(d, struct fsnotify_file_info, d);
 
-	tmp = inotify_init1(info->ife->flags);
+	/*
+	 * Note we're creating nonblock version of inotify,
+	 * this is because we will need to flush events we
+	 * might generate during restore. Same applies to
+	 * fanotify.
+	 */
+
+	tmp = inotify_init1(info->ife->flags | IN_NONBLOCK);
 	if (tmp < 0) {
 		pr_perror("Can't create inotify for 0x%08x", info->ife->id);
 		return -1;
@@ -595,6 +681,17 @@ static int open_inotify_fd(struct file_desc *d)
 	if (restore_fown(tmp, info->ife->fown))
 		goto err;
 
+	if (flush_inotify_events(tmp, info->ife->id))
+		goto err;
+
+	if (!(info->ife->flags & IN_NONBLOCK)) {
+		if (fcntl(tmp, F_SETFD, fcntl(tmp, F_GETFD) | O_NONBLOCK)) {
+			pr_perror("Can't restore inotify flags on 0x%08x",
+				  info->ife->id);
+			return -1;
+		}
+	}
+
 	return tmp;
 err:
 	close_safe(&tmp);
@@ -610,11 +707,13 @@ static int open_fanotify_fd(struct file_desc *d)
 
 	info = container_of(d, struct fsnotify_file_info, d);
 
-	flags = info->ffe->faflags;
+	/*
+	 * Make it nonblock by default, to flush own generated
+	 * events, and then restore it back.
+	 */
+	flags = info->ffe->faflags | FAN_NONBLOCK;
 	if (info->ffe->flags & O_CLOEXEC)
 		flags |= FAN_CLOEXEC;
-	if (info->ffe->flags & O_NONBLOCK)
-		flags |= FAN_NONBLOCK;
 
 	tmp = sys_fanotify_init(flags, info->ffe->evflags);
 	if (tmp < 0) {
@@ -632,6 +731,17 @@ static int open_fanotify_fd(struct file_desc *d)
 	if (restore_fown(tmp, info->ffe->fown))
 		goto err;
 
+	if (flush_inotify_events(tmp, info->ffe->id))
+		goto err;
+
+	if (!(info->ffe->faflags & O_NONBLOCK)) {
+		if (fcntl(tmp, F_SETFD, fcntl(tmp, F_GETFD) | O_NONBLOCK)) {
+			pr_perror("Can't restore fanotify flags on 0x%08x",
+				  info->ffe->id);
+			return -1;
+		}
+	}
+
 	return tmp;
 err:
 	close_safe(&tmp);
-- 
1.9.3