[CRIU] [PATCHv2 6/9] compel/x86: Add workaround on ptrace() bug on Skylake

[Dmitry Safonov]

On Skylake processors and kernel older than v4.14
    ptrace(PTRACE_GETREGSET, pid, NT_X86_XSTATE, iov)
may return not full xstate, ommiting FP part (that is XFEATURE_MASK_FP).
There is a patch which describes this bug:
  https://www.mail-archive.com/linux-kernel@vger.kernel.org/msg1318800.html
Anyway, it's fixed in v4.14 kernel by (what we believe with Andrey) this:
  https://patchwork.kernel.org/patch/9567939/

As we still support kernels from v3.10 and newer, we need to have a
workaround for this kernel bug on Skylake CPUs.

Big thanks to Shlomi for the reports, the effort and for providing an
Amazon VM to test this. I wish more bug reporters were like you.

Reported-by: Shlomi Matichin <shlomi at binaris.com>
Provided-test-env: Shlomi Matichin <shlomi at binaris.com>
Investigated-with: Andrei Vagin <avagin at virtuozzo.com>
Signed-off-by: Dmitry Safonov <dima at arista.com>
---
 compel/arch/x86/src/lib/infect.c | 22 ++++++++++++++++------
 compel/include/uapi/infect.h     |  2 ++
 criu/parasite-syscall.c          |  2 ++
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/compel/arch/x86/src/lib/infect.c b/compel/arch/x86/src/lib/infect.c
index d95299b3b692..85c93ad8aeee 100644
--- a/compel/arch/x86/src/lib/infect.c
+++ b/compel/arch/x86/src/lib/infect.c
@@ -251,7 +251,7 @@ static int get_task_fpregs(pid_t pid, user_fpregs_struct_t *xsave)
 }
 
 int get_task_regs(pid_t pid, user_regs_struct_t *regs, save_regs_t save,
-		  void *arg, __maybe_unused unsigned long flags)
+		  void *arg, unsigned long flags)
 {
 	user_fpregs_struct_t xsave = { }, *xs = NULL;
 	int ret = -1;
@@ -286,14 +286,24 @@ int get_task_regs(pid_t pid, user_regs_struct_t *regs, save_regs_t save,
 
 	pr_info("Dumping GP/FPU registers for %d\n", pid);
 
-	if (compel_cpu_has_feature(X86_FEATURE_OSXSAVE)) {
-		if (get_task_xsave(pid, &xsave))
-			goto err;
+	if (!compel_cpu_has_feature(X86_FEATURE_OSXSAVE)) {
+		ret = get_task_fpregs(pid, &xsave);
+	} else if (unlikely(flags & INFECT_X86_PTRACE_MXCSR_BUG)) {
+		/*
+		 * get_task_fpregs() will fill FP state,
+		 * get_task_xsave() will overwrite rightly sse/mmx/etc
+		 */
+		pr_warn("Skylake xsave fpu bug workaround used\n");
+		ret = get_task_fpregs(pid, &xsave);
+		if (!ret)
+			ret = get_task_xsave(pid, &xsave);
 	} else {
-		if (get_task_fpregs(pid, &xsave))
-			goto err;
+		ret = get_task_xsave(pid, &xsave);
 	}
 
+	if (ret)
+		goto err;
+
 	xs = &xsave;
 out:
 	ret = save(arg, regs, xs);
diff --git a/compel/include/uapi/infect.h b/compel/include/uapi/infect.h
index d9bf069b2458..e0ff233e70a9 100644
--- a/compel/include/uapi/infect.h
+++ b/compel/include/uapi/infect.h
@@ -127,6 +127,8 @@ extern struct infect_ctx *compel_infect_ctx(struct parasite_ctl *);
 #define INFECT_FAIL_CONNECT	0x2	/* make parasite connect() fail */
 #define INFECT_NO_BREAKPOINTS	0x4	/* no breakpoints in pie tracking */
 #define INFECT_COMPATIBLE	0x8	/* can run parasite inside compat tasks */
+/* Workaround for ptrace bug on Skylake CPUs with kernels older than v4.14 */
+#define INFECT_X86_PTRACE_MXCSR_BUG	0x10
 
 /*
  * There are several ways to describe a blob to compel
diff --git a/criu/parasite-syscall.c b/criu/parasite-syscall.c
index 2650d7615b32..6a8abc853742 100644
--- a/criu/parasite-syscall.c
+++ b/criu/parasite-syscall.c
@@ -523,6 +523,8 @@ struct parasite_ctl *parasite_infect_seized(pid_t pid, struct pstree_item *item,
 		ictx->flags |= INFECT_NO_BREAKPOINTS;
 	if (kdat.compat_cr)
 		ictx->flags |= INFECT_COMPATIBLE;
+	if (kdat.x86_has_ptrace_fpu_xsave_bug)
+		ictx->flags |= INFECT_X86_PTRACE_MXCSR_BUG;
 
 	ictx->log_fd = log_get_fd();
 
-- 
2.13.6