[Devel] [PATCH VZ10 1/3] bpf: report original insns of cgroup device programs

[Pavel Tikhomirov]

Since device cgroup BPF programs were recently allowed in VE, CRIU needs
an interface to checkpoint and restore those.

The verified/JITed instructions are not portable: the verifier rewrites
context accesses and helper calls into offsets and addresses specific to
the running kernel, so if we dump prog->insnsi and try to load it again,
it will break for at least two reasons: a) verifier will refuse the
translated version, b) on another kernel the translated addresses will
be completely irrelevant.

Instead keep a copy of the original, pre-verification instructions the
user loaded. Their encoding (struct bpf_insn) and the CGROUP_DEVICE
context (struct bpf_cgroup_dev_ctx) are stable UAPI, so on restore CRIU
can reload the program and let the destination kernel re-verify and
re-JIT it.

Expose the copy through BPF_OBJ_GET_INFO_BY_FD in the new
bpf_prog_info::orig_prog_insns field.

Implementation notes:

Save orig_insns for CGROUP_DEVICE programs only, and only up to
BPF_MAXINSNS - this covers every program loadable inside a VE (which is
all CRIU needs) and skips large host programs.

Unlike the xlated/jited dumps it needs no bpf_dump_raw_ok() check:
orig_insns is the original, pre-verification program and holds no kernel
address or kallsyms related information.

As this is only useful for checkpoint/restore, gate it on
CONFIG_CHECKPOINT_RESTORE.

Use kvmemdup so the copy does not require a high-order contiguous
allocation (maximum size is 32KB), with __GFP_NOWARN since the size is
user-controlled.

Feature: BPF checkpoint/restore
https://virtuozzo.atlassian.net/browse/VSTOR-121776
Signed-off-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
---
 include/linux/bpf.h            |  2 ++
 include/uapi/linux/bpf.h       |  2 ++
 kernel/bpf/core.c              |  1 +
 kernel/bpf/syscall.c           | 30 ++++++++++++++++++++++++++++++
 tools/include/uapi/linux/bpf.h |  2 ++
 5 files changed, 37 insertions(+)

diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index 1e30bb0867834..87a1a48ca5141 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1654,6 +1654,8 @@ struct bpf_prog_aux {
 #ifdef CONFIG_SECURITY
 	void *security;
 #endif
+	struct bpf_insn *orig_insns;
+	u32 orig_insn_cnt;
 	struct bpf_token *token;
 #ifdef CONFIG_VE
 	/* VE that loaded the program via VE_FEATURE_BPF path and against whose
diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h
index a31fba921e869..378db5313733e 100644
--- a/include/uapi/linux/bpf.h
+++ b/include/uapi/linux/bpf.h
@@ -6638,6 +6638,8 @@ struct bpf_prog_info {
 	__u32 verified_insns;
 	__u32 attach_btf_obj_id;
 	__u32 attach_btf_id;
+	__u32 orig_prog_len;
+	__aligned_u64 orig_prog_insns;
 } __attribute__((aligned(8)));
 
 struct bpf_map_info {
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 5503ec305a846..c9fb3739efeaf 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -285,6 +285,7 @@ void __bpf_prog_free(struct bpf_prog *fp)
 		mutex_destroy(&fp->aux->used_maps_mutex);
 		mutex_destroy(&fp->aux->dst_mutex);
 		kfree(fp->aux->poke_tab);
+		kvfree(fp->aux->orig_insns);
 		kfree(fp->aux);
 	}
 	free_percpu(fp->stats);
diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index ff2a51c59f047..661efe01ca4a0 100644
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -2985,6 +2985,26 @@ static int bpf_prog_load(union bpf_attr *attr, bpfptr_t uattr, u32 uattr_size)
 			     make_bpfptr(attr->insns, uattr.is_kernel),
 			     bpf_prog_insn_size(prog)) != 0)
 		goto free_prog;
+
+	/*
+	 * Save the original insns so CRIU can dump the program and reload it
+	 * after migration, hence only needed with CONFIG_CHECKPOINT_RESTORE.
+	 * Restrict it to small programs (up to BPF_MAXINSNS), which covers all
+	 * programs loaded inside a VE, avoids the extra allocation for large
+	 * host progs, and keeps the copy at most BPF_MAXINSNS *
+	 * sizeof(struct bpf_insn) in size. The size is user-controlled, so use
+	 * __GFP_NOWARN to avoid log spam on failed allocations.
+	 */
+	if (IS_ENABLED(CONFIG_CHECKPOINT_RESTORE) &&
+	    type == BPF_PROG_TYPE_CGROUP_DEVICE && prog->len <= BPF_MAXINSNS) {
+		prog->aux->orig_insns = kvmemdup(prog->insns,
+						 bpf_prog_insn_size(prog),
+						 GFP_KERNEL | __GFP_NOWARN);
+		if (!prog->aux->orig_insns)
+			goto free_prog;
+		prog->aux->orig_insn_cnt = prog->len;
+	}
+
 	/* copy eBPF program license from user space */
 	if (strncpy_from_bpfptr(license,
 				make_bpfptr(attr->license, uattr.is_kernel),
@@ -5056,6 +5076,7 @@ static int bpf_prog_get_info_by_fd(struct file *file,
 		info.nr_func_info = 0;
 		info.nr_line_info = 0;
 		info.nr_jited_line_info = 0;
+		info.orig_prog_len = 0;
 		goto done;
 	}
 
@@ -5266,6 +5287,15 @@ static int bpf_prog_get_info_by_fd(struct file *file,
 		}
 	}
 
+	ulen = info.orig_prog_len;
+	info.orig_prog_len = prog->aux->orig_insn_cnt * sizeof(struct bpf_insn);
+	if (info.orig_prog_len && ulen) {
+		uinsns = u64_to_user_ptr(info.orig_prog_insns);
+		ulen = min_t(u32, info.orig_prog_len, ulen);
+		if (copy_to_user(uinsns, prog->aux->orig_insns, ulen))
+			return -EFAULT;
+	}
+
 done:
 	if (copy_to_user(uinfo, &info, info_len) ||
 	    put_user(info_len, &uattr->info.info_len))
diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h
index 611ba71044cef..ef05a749b8ebb 100644
--- a/tools/include/uapi/linux/bpf.h
+++ b/tools/include/uapi/linux/bpf.h
@@ -6638,6 +6638,8 @@ struct bpf_prog_info {
 	__u32 verified_insns;
 	__u32 attach_btf_obj_id;
 	__u32 attach_btf_id;
+	__u32 orig_prog_len;
+	__aligned_u64 orig_prog_insns;
 } __attribute__((aligned(8)));
 
 struct bpf_map_info {
-- 
2.54.0