[Devel] [PATCH RHEL9 COMMIT] ve/cgroup: Implement per-ve workqueue

[Konstantin Khorenko]

The commit is pushed to "branch-rh9-5.14.vz9.1.x-ovz" and will appear at https://src.openvz.org/scm/ovz/vzkernel.git
after rh9-5.14.0-4.vz9.10.12
------>
commit e63f876108f2a7e9f2878e1eda5af556ad3e7ccb
Author: Valeriy Vdovin <valeriy.vdovin at virtuozzo.com>
Date:   Wed Oct 20 11:43:55 2021 +0300

    ve/cgroup: Implement per-ve workqueue
    
    Per-ve workqueue is started at ve cgroup start before ve->is_running is
    set to true, it is stopped at ve cgroup stop after ve->is_running is set
    to false and is guarded via synchronize_rcu, stop also implies waiting
    for all queued and yet unfinished works to finish.
    
    So the user of ve->wq should take rcu_read_lock, under the lock check
    ve->is_running, if ve->is_running is true user can safely queue_work on
    the workqueue and release the lock.
    
    We need per-ve workqueue for per-ve release-agent implementation.
    Release-agent code (in next patches) would use call_usermodehelper_ve
    from per-ve workqueue worker to run the release-agent binary in actual
    ve-context, so that workqueue is started after umh started and stopped
    before umh kthread stopped.
    
    Short history of the patch:
    
    https://jira.sw.ru/browse/PSBM-83887
    Signed-off-by: Valeriy Vdovin <valeriy.vdovin at virtuozzo.com>
    
    Reviewed-by: Kirill Tkhai <ktkhai at virtuozzo.com>
    (cherry picked from vz7 commit 0293870666c4f96bd56f612d94f560626c76e2fd)
    https://jira.sw.ru/browse/PSBM-108270
    (cherry picked from vz8 commit 58b7b3bb335b2bafca9caccfee8c435242baa664)
    +++
    cgroup/ve: Do not run release_agent on non-running ve
    +++
    cgroup/ve: Move ve workqueue stop to ve_stop_ns()
    (cherry picked from vz8 commit 1481d130afa32eecffe28b34fd00b57e11d2666f)
    
    vz9 changes:
    - merge per-ve workqueue fixes from 1481d130afa3
    - add comment to wq field about syncronization using rcu and is_running
    
    https://jira.sw.ru/browse/PSBM-134002
    Signed-off-by: Pavel Tikhomirov <ptikhomirov at virtuozzo.com>
    
    Reviewed-by: Alexander Mikhalitsyn <alexander.mikhalitsyn at virtuozzo.com>
    Reviewed-by: Kirill Tkhai <ktkhai at virtuozzo.com>
---
 include/linux/ve.h |  3 +++
 kernel/ve/ve.c     | 32 ++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/include/linux/ve.h b/include/linux/ve.h
index 60947c200e8d..eb223c9db6e5 100644
--- a/include/linux/ve.h
+++ b/include/linux/ve.h
@@ -106,6 +106,9 @@ struct ve_struct {
 	unsigned long		aio_nr;
 	unsigned long		aio_max_nr;
 #endif
+	/* Should take rcu_read_lock and check ve->is_running before queue */
+	struct workqueue_struct	*wq;
+
 	struct vfsmount		*devtmpfs_mnt;
 };
 
diff --git a/kernel/ve/ve.c b/kernel/ve/ve.c
index 41f9e385bd28..d8fd86568533 100644
--- a/kernel/ve/ve.c
+++ b/kernel/ve/ve.c
@@ -462,6 +462,22 @@ static int ve_start_kthreadd(struct ve_struct *ve)
 	return err;
 }
 
+static int ve_workqueue_start(struct ve_struct *ve)
+{
+	ve->wq = alloc_workqueue("ve_wq_%s", WQ_SYSFS|WQ_FREEZABLE|WQ_UNBOUND,
+				 8, ve->ve_name);
+
+	if (!ve->wq)
+		return -ENOMEM;
+	return 0;
+}
+
+static void ve_workqueue_stop(struct ve_struct *ve)
+{
+	destroy_workqueue(ve->wq);
+	ve->wq = NULL;
+}
+
 /* under ve->op_sem write-lock */
 static int ve_start_container(struct ve_struct *ve)
 {
@@ -506,6 +522,10 @@ static int ve_start_container(struct ve_struct *ve)
 	if (err)
 		goto err_umh;
 
+	err = ve_workqueue_start(ve);
+	if (err)
+		goto err_workqueue;
+
 	err = ve_hook_iterate_init(VE_SS_CHAIN, ve);
 	if (err < 0)
 		goto err_iterate;
@@ -525,6 +545,8 @@ static int ve_start_container(struct ve_struct *ve)
 err_mark_ve:
 	ve_hook_iterate_fini(VE_SS_CHAIN, ve);
 err_iterate:
+	ve_workqueue_stop(ve);
+err_workqueue:
 	ve_stop_umh(ve);
 err_umh:
 	ve_stop_kthreadd(ve);
@@ -554,6 +576,14 @@ void ve_stop_ns(struct pid_namespace *pid_ns)
 	 * ve_mutex works as barrier for ve_can_attach().
 	 */
 	ve->is_running = 0;
+	synchronize_rcu();
+
+	/*
+	 * release_agent works on top of umh_worker, so we must make sure, that
+	 * ve workqueue is stopped first.
+	 */
+	ve_workqueue_stop(ve);
+
 	/*
 	 * Neither it can be in pseudosuper state
 	 * anymore, setup it again if needed.
@@ -1585,6 +1615,8 @@ static int __init ve_subsys_init(void)
 {
 	ve_cachep = KMEM_CACHE_USERCOPY(ve_struct, SLAB_PANIC, core_pattern);
 	list_add(&ve0.ve_list, &ve_list_head);
+	ve0.wq = alloc_workqueue("ve0_wq", WQ_FREEZABLE|WQ_UNBOUND, 8);
+	BUG_ON(!ve0.wq);
 	return 0;
 }
 late_initcall(ve_subsys_init);