[Devel] [PATCH RHEL9 COMMIT] cgroup_freezer: Print information about unfreezable process

[Konstantin Khorenko]

The commit is pushed to "branch-rh9-5.14.0-70.22.1.vz9.17.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh9-5.14.0-70.22.1.vz9.17.12
------>
commit ec9280be8a973d5dcde1fba07872bfec05ac99ef
Author: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
Date:   Wed Nov 30 14:22:42 2022 +0300

    cgroup_freezer: Print information about unfreezable process
    
    Add a sysctl kernel.freeze_cgroup_timeout (default value 30 * HZ).
    
    If one writes FROZEN to freezer.state file and after a timeout of
    kernel.freeze_cgroup_timeout one still reads FREEZING from freezer.state
    file (meaning that kernel does not succeed to freeze cgroup processes
    still) - let's print a warning with information about the problem, e.g.:
    
    [ 7196.621368] Freeze of /test took 0 sec, due to unfreezable process 13732:bash, stack:
    [ 7196.621396] [<ffffffffa2df9556>] retint_careful+0x14/0x32
    [ 7196.621431] [<ffffffffffffffff>] 0xffffffffffffffff
    
    The output includes:
     - path to problematic freezer cgroup
     - timeout in seconds
     - unfeezable process pid, comm and stack
    
    https://jira.sw.ru/browse/PSBM-142970
    
    Signed-off-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
    
    Feature: cgroup/freeze: enhance logging
---
 include/linux/sysctl.h         |  2 ++
 kernel/cgroup/legacy_freezer.c | 71 ++++++++++++++++++++++++++++++++++++++++--
 kernel/sysctl.c                | 10 ++++++
 3 files changed, 80 insertions(+), 3 deletions(-)

diff --git a/include/linux/sysctl.h b/include/linux/sysctl.h
index 8fd2d3c217c2..b641dd2bba82 100644
--- a/include/linux/sysctl.h
+++ b/include/linux/sysctl.h
@@ -188,6 +188,8 @@ struct ctl_path {
 };
 
 extern int trusted_exec;
+#define DEFAULT_FREEZE_TIMEOUT (30 * HZ)
+extern int sysctl_freeze_timeout;
 
 extern int ve_allow_module_load;
 
diff --git a/kernel/cgroup/legacy_freezer.c b/kernel/cgroup/legacy_freezer.c
index 08236798d173..eea2f0b924f1 100644
--- a/kernel/cgroup/legacy_freezer.c
+++ b/kernel/cgroup/legacy_freezer.c
@@ -22,6 +22,10 @@
 #include <linux/freezer.h>
 #include <linux/seq_file.h>
 #include <linux/mutex.h>
+#include <linux/jiffies.h>
+#include <linux/ratelimit.h>
+#include <linux/stacktrace.h>
+#include <linux/sysctl.h>
 
 /*
  * A cgroup is freezing if any FREEZING flags are set.  FREEZING_SELF is
@@ -43,6 +47,7 @@ enum freezer_state_flags {
 struct freezer {
 	struct cgroup_subsys_state	css;
 	unsigned int			state;
+	unsigned long			freeze_jiffies;
 };
 
 static DEFINE_MUTEX(freezer_mutex);
@@ -225,6 +230,60 @@ static void freezer_fork(struct task_struct *task)
 	mutex_unlock(&freezer_mutex);
 }
 
+#define MAX_STACK_TRACE_DEPTH   64
+
+static void check_freezer_timeout(struct cgroup_subsys_state *css,
+				  struct task_struct *task)
+
+{
+	static DEFINE_RATELIMIT_STATE(freeze_timeout_rs,
+				      DEFAULT_FREEZE_TIMEOUT, 1);
+	int __freeze_timeout = READ_ONCE(sysctl_freeze_timeout);
+	struct freezer *freezer = css_freezer(css);
+	unsigned long nr_entries;
+	unsigned long *entries;
+	char *freezer_cg_name;
+	pid_t tgid;
+	int i;
+
+	if (!freezer->freeze_jiffies ||
+	    freezer->freeze_jiffies + __freeze_timeout > get_jiffies_64())
+		return;
+
+	if (!__ratelimit(&freeze_timeout_rs))
+		return;
+
+	freezer_cg_name = kmalloc(PATH_MAX, GFP_KERNEL);
+	if (!freezer_cg_name)
+		return;
+
+	if (cgroup_path(css->cgroup, freezer_cg_name, PATH_MAX) < 0)
+		goto free_cg_name;
+
+	tgid = task_pid_nr_ns(task, &init_pid_ns);
+
+	printk(KERN_WARNING "Freeze of %s took %d sec, "
+	       "due to unfreezable process %d:%s, stack:\n",
+	       freezer_cg_name, __freeze_timeout/HZ, tgid, task->comm);
+
+	entries = kmalloc(MAX_STACK_TRACE_DEPTH * sizeof(*entries),
+			  GFP_KERNEL);
+	if (!entries)
+		goto free_cg_name;
+
+	nr_entries = stack_trace_save_tsk(task, entries,
+					  MAX_STACK_TRACE_DEPTH, 0);
+
+	for (i = 0; i < nr_entries; i++) {
+		printk(KERN_WARNING "[<%pK>] %pB\n",
+		       (void *)entries[i], (void *)entries[i]);
+	}
+
+	kfree(entries);
+free_cg_name:
+	kfree(freezer_cg_name);
+}
+
 /**
  * update_if_frozen - update whether a cgroup finished freezing
  * @css: css of interest
@@ -278,8 +337,10 @@ static void update_if_frozen(struct cgroup_subsys_state *css)
 			 * completion.  Consider it frozen in addition to
 			 * the usual frozen condition.
 			 */
-			if (!frozen(task) && !freezer_should_skip(task))
+			if (!frozen(task) && !freezer_should_skip(task)) {
+				check_freezer_timeout(css, task);
 				goto out_iter_end;
+			}
 		}
 	}
 
@@ -356,8 +417,10 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze,
 		return;
 
 	if (freeze) {
-		if (!(freezer->state & CGROUP_FREEZING))
+		if (!(freezer->state & CGROUP_FREEZING)) {
 			atomic_inc(&system_freezing_cnt);
+			freezer->freeze_jiffies = get_jiffies_64();
+		}
 		freezer->state |= state;
 		freeze_cgroup(freezer);
 	} else {
@@ -366,8 +429,10 @@ static void freezer_apply_state(struct freezer *freezer, bool freeze,
 		freezer->state &= ~state;
 
 		if (!(freezer->state & CGROUP_FREEZING)) {
-			if (was_freezing)
+			if (was_freezing) {
+				freezer->freeze_jiffies = 0;
 				atomic_dec(&system_freezing_cnt);
+			}
 			freezer->state &= ~CGROUP_FROZEN;
 			unfreeze_cgroup(freezer);
 		}
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index 16b6bc8db93b..6e369ec106b4 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -120,6 +120,8 @@ __setup("trusted_exec", set_trusted_exec);
 int ve_allow_module_load = 1;
 EXPORT_SYMBOL(ve_allow_module_load);
 
+int sysctl_freeze_timeout = DEFAULT_FREEZE_TIMEOUT;
+
 /* Constants used for minimum and  maximum */
 #ifdef CONFIG_LOCKUP_DETECTOR
 static int sixty = 60;
@@ -2847,6 +2849,14 @@ static struct ctl_table kern_table[] = {
 		.extra2		= SYSCTL_ONE,
 	},
 #endif
+	{
+		.procname	= "freeze_cgroup_timeout",
+		.data		= &sysctl_freeze_timeout,
+		.maxlen		= sizeof(int),
+		.mode		= 0644,
+		.proc_handler	= proc_dointvec_minmax,
+		.extra1		= &zero,
+	},
 	{ }
 };