[Libct] Regarding libct and cgroup subgroups w/ or w/o mount namespaces.
Andrew Vagin
avagin at parallels.com
Wed Dec 24 07:54:09 PST 2014
On Wed, Dec 24, 2014 at 01:39:40AM +0000, Monali Porob wrote:
> Hi Andrew ,
>
>
>
> For container with a mount namespaces and bind mounted controllers into its
> file tree,. I was able to check spawn a callback to check creation of subgroups
> or spawn a program.
>
>
>
> However using the libct_container_enter_execv() to enter a program in container
> is returning failure (-1). Attached is the program with failing enter_exec()
> but working spawn code ..
I found a mistake. Could you try out the attached patch. Thanks.
>
> Wanted to check if I am wrongly programming the parameters for
> libct_container_enter_execv. Tried libct_container_enter_cb() and that fails
> too with -1.
>
> Hence thought of getting pointers to debug the issue further
>
>
>
> Qn 2 :
>
> how do we spawn a container process in one of the subgroups created under the
> container file tree which has all the cgroup controllers bind mounted.
>
>
>
> Qn3:
>
> Also ,
>
> For a container with cgroups and without any namespaces (not even mount) , I
> am looking to check if the following functionalities would be supported by
> libct .
>
> -Ability to create container with a default subgroup , configure default
> subgroup and start a container process in the default subgroup.
>
> - Ability to create and configure additional subgroups and then move process
> from default subgroup to other already configured subgroups.
>
>
>
> Regards,
>
> Monali
>
>
>
> /*
> * Test creation of container with cgroup controller with subgroups and mount namespaces
> */
> #define _XOPEN_SOURCE
> #include <libct.h>
> #include <stdio.h>
> #include <sys/mman.h>
> #include <fcntl.h>
> #include <string.h>
> #include <stdlib.h>
> #include <unistd.h>
> #include <libct.h>
> #include <stdio.h>
> #include <unistd.h>
> #include <sys/mman.h>
> #include <string.h>
> #include <fcntl.h>
> #include <unistd.h>
> #include <sys/stat.h>
> #include <sys/types.h>
> #include "test.h"
>
> #define CPUS "0"
> #define STR_HELPER(x) #x
> #define STR(x) STR_HELPER(x)
>
> #define MEMLIMIT 134217728 //128 MB
> #define MEMLIMIT_STR STR(MEMLIMIT)
>
> #define LOWER_THAN_LIMIT 104857600 //100 MB
> #define LOWER_THAN_LIMIT_STR STR(LOWER_THAN_LIMIT)
>
> #define FS_ROOT "libct_test_root_ns"
> #define FS_PRIVATE "libct_test_private_ns"
> #define FS_CG "cg"
>
> // External directories that will be bind mounted in the container
> #define FS_EXT_BIN "/usr/bin"
> #define FS_EXT_LIB "/lib"
> #define FS_EXT_LIB64 "/lib64"
>
> #define FS_CT_BIN "bin"
> #define FS_CT_LIB "lib"
> #define FS_CT_LIB64 "lib64"
>
>
> #ifndef CLONE_NEWNS
> #define CLONE_NEWNS 0x00020000
> #endif
>
> // For testing Define STRESS_MEM_STR to either LOWER_THAN_LIMIT_STR or MEMLIMIT_STR
> #define STRESS_MEM_STR LOWER_THAN_LIMIT_STR
> //#define STRESS_MEM_STR MEMLIMIT_STR
>
> int is_memory_limit_correct(const char *expected_limit)
> {
> int ret = 0;
> FILE *f = NULL;
> char buf[1024] = {'\0'};
>
> f = fopen("/sys/fs/cgroup/memory/test1/memory.limit_in_bytes", "r");
> if (!f) {
> fprintf(stderr, "Unable to open memory limit file !\n");
> goto err;
> }
>
> if (fscanf(f, "%s", buf) != 1) {
> fprintf(stderr, "fscanf failed!\n");
> goto err;
> }
> printf(" /sys/fs/cgroup/memory/test1/memory.limit_in_bytes now contains %s \n",buf);
> ret = !strcmp(buf, expected_limit);
> err:
> fclose(f);
> return ret;
> }
>
> static int check_cgroup(void *a)
> {
> FILE *f = NULL;
> char buf[1024] = {'\0'};
> mkdir("/"FS_CG"/memory/x", 0600);
> f = fopen("/"FS_CG"/memory/x/memory.limit_in_bytes", "r");
> if (!f) {
> fprintf(stderr, "Unable to open memory limit file !\n");
> goto err;
> }
>
> if (fscanf(f, "%s", buf) != 1) {
> fprintf(stderr, "fscanf failed!\n");
> goto err;
> }
> printf(" /"FS_CG"/memory/x/memory.limit_in_bytes now contains %s \n",buf);
> if (access("/"FS_CT_BIN"/stress", F_OK) == 0)
> printf (" Stress program is accessible in the CT \n");
> else
> printf (" Stress program is NOT available !! \n");
>
> // rmdir("/"FS_CG"/memory/x");
>
> err:
> fclose(f);
> return 0;
> }
>
> int main(int argc, char **argv)
> {
> libct_session_t s;
> ct_handler_t ct;
> ct_process_desc_t p;
> ct_process_t pr;
> int ret = 0;
> int state = 0;
> int fs_err = 0;
>
> mkdir(FS_ROOT, 0600);
> mkdir(FS_ROOT "/" FS_CT_BIN, 0600);
> mkdir(FS_ROOT "/" FS_CT_LIB, 0600);
> mkdir(FS_ROOT "/" FS_CT_LIB64, 0600);
>
> mkdir(FS_PRIVATE, 0600);
> mkdir(FS_PRIVATE "/" FS_CG, 0600);
>
> s = libct_session_open_local();
> ct = libct_container_create(s, "test1");
> p = libct_process_desc_create(s);
>
> libct_container_set_nsmask(ct, CLONE_NEWNS);
>
> libct_controller_add(ct, CTL_MEMORY);
> ret = libct_controller_configure(ct, CTL_MEMORY, "memory.limit_in_bytes", MEMLIMIT_STR);
> printf("\n libct_controller_configure returned %d \n", ret);
>
> ret = libct_controller_configure(ct, CTL_MEMORY, "memory.memsw.limit_in_bytes", MEMLIMIT_STR);
> printf("\n libct_controller_configure returned %d \n", ret);
>
>
> libct_fs_set_root(ct, FS_ROOT);
> libct_fs_add_bind_mount(ct, FS_EXT_BIN, FS_CT_BIN, 0);
> libct_fs_add_bind_mount(ct, FS_EXT_LIB, FS_CT_LIB, 0);
> libct_fs_add_bind_mount(ct, FS_EXT_LIB64, FS_CT_LIB64, 0);
>
> libct_fs_set_private(ct, CT_FS_SUBDIR, FS_PRIVATE);
> libct_container_set_option(ct, LIBCT_OPT_CGROUP_SUBMOUNT, FS_CG);
> #if 0
> pr = libct_container_spawn_cb(ct, p, check_cgroup, NULL);
> if (libct_handle_is_err(pr))
> {
> printf("\n lbct_container_spawn_cb failed with value = %ld \n",libct_handle_to_err(pr));
> return fail("Unable to spawn stress program into CT");
> }
>
> #endif
> state = libct_container_state(ct);
> printf(" Before : state of the container is %d \n",state);
>
> char *stress_a[8];
> stress_a[0] = "/"FS_CT_BIN"/stress";
> stress_a[1] = "--vm";
> stress_a[2] = "1";
> stress_a[3] = "--vm-bytes";
> stress_a[4] = STRESS_MEM_STR;
> stress_a[5] = "--vm-hang";
> stress_a[6] = "0";
> stress_a[7]= NULL;
>
> printf ("Stress path is %s \n ",stress_a[0]);
> pr = libct_container_spawn_execv(ct,p, "/"FS_CT_BIN"/stress",stress_a);
> if (libct_handle_is_err(pr))
> {
> printf("\n libct_container_spawn_execv failed with value = %ld \n",libct_handle_to_err(pr));
> return fail("Unable to spawn stress program into CT");
> }
>
> // This is not working .
> pr = libct_container_enter_cb(ct, p, check_cgroup, NULL);
> if (libct_handle_is_err(pr))
> {
> printf("\n lbct_container_enter_cb failed with value = %ld \n",libct_handle_to_err(pr));
> return fail("Unable to enter callback into CT");
> }
>
> #if 0
> // This is not working either .
> //pr = libct_container_enter_execv(ct,p, "/"FS_CT_BIN"/stress",stress_a);
> stress_a[0] = "/usr/bin/stress";
> printf ("Stress path is %s \n ",stress_a[0]);
> pr = libct_container_enter_execv(ct,p, "/usr/bin/stress",stress_a);
> if (libct_handle_is_err(pr))
> {
> printf("\n libct_container_enter_execv failed with value = %ld \n",libct_handle_to_err(pr));
> return fail("Unable to enter stress program into CT");
> }
>
> #endif
> state = libct_container_state(ct);
> printf("\n After spawn_exec state of the container is %d \n",state);
>
> if (!is_memory_limit_correct(MEMLIMIT_STR))
> return fail("Memory settings dont match to the expected value");
>
> if (libct_container_wait(ct) < 0)
> goto err;
> libct_container_destroy(ct);
> libct_session_close(s);
>
> if (rmdir(FS_PRIVATE "/" FS_CG) < 0)
> fs_err |= 1;
> if (rmdir(FS_ROOT "/" FS_CT_BIN) < 0)
> fs_err |= 2;
> if (rmdir(FS_ROOT "/" FS_CT_LIB) < 0)
> fs_err |= 4;
> if (rmdir(FS_ROOT "/" FS_CT_LIB64) < 0)
> fs_err |= 8;
> if (rmdir(FS_ROOT) < 0)
> fs_err |= 3;
> if (rmdir(FS_PRIVATE) < 0)
> fs_err |= 16;
>
> if (fs_err) {
> printf("FS remove failed %x %d \n", fs_err,fs_err);
> return fail("FS broken");
>
> }
>
> return pass("Memory limited CT is OK");
> err:
> return fail("Something wrong");
> }
-------------- next part --------------
>From 494e43e7c5672de31383de34f8fdc904ea54a8ef Mon Sep 17 00:00:00 2001
From: Andrey Vagin <avagin at openvz.org>
Date: Wed, 24 Dec 2014 16:35:04 +0300
Subject: [PATCH] cgroups: move a process into required cgroups
Currently we fork a process and it moves itself into cgroups.
* A process can have not enought permisions to do this.
* Currently cgroups_attach() is called after setting namespaces,
but cgroups can be not mounted in CT.
Reported-by: Monali Porob <Monali.Porob at huawei.com>
Signed-off-by: Andrey Vagin <avagin at openvz.org>
Signed-off-by: Andrew Vagin <avagin at openvz.org>
---
src/cgroups.c | 8 ++++----
src/ct.c | 40 +++++++++++++++++++++++++---------------
src/include/cgroups.h | 2 +-
3 files changed, 30 insertions(+), 20 deletions(-)
diff --git a/src/cgroups.c b/src/cgroups.c
index f22eb68..7a9f444 100644
--- a/src/cgroups.c
+++ b/src/cgroups.c
@@ -264,15 +264,15 @@ static int cgroup_attach_one(struct container *ct, struct controller *ctl, char
return config_controller(ct, ctl->ctype, "tasks", pid) ? -LCTERR_CGATTACH : 0;
}
-int cgroups_attach(struct container *ct)
+int cgroups_attach(struct container *ct, pid_t pid)
{
- char pid[12];
+ char spid[12];
struct controller *ctl;
int ret = 0;
- snprintf(pid, sizeof(pid), "%d", getpid());
+ snprintf(spid, sizeof(spid), "%d", pid);
list_for_each_entry(ctl, &ct->cgroups, ct_l) {
- ret = cgroup_attach_one(ct, ctl, pid);
+ ret = cgroup_attach_one(ct, ctl, spid);
if (ret)
break;
}
diff --git a/src/ct.c b/src/ct.c
index 3263356..7f3225f 100644
--- a/src/ct.c
+++ b/src/ct.c
@@ -237,10 +237,6 @@ static int ct_clone(void *arg)
if (try_mount_cg(ct))
goto err;
- ret = cgroups_attach(ct);
- if (ret < 0)
- goto err_um;
-
if (ct->root_path) {
/*
* Mount external in child, since it may live
@@ -384,6 +380,10 @@ static ct_process_t __local_spawn_cb(ct_handler_t h, ct_process_desc_t ph, int (
goto err_net;
}
+ ret = cgroups_attach(ct, pid);
+ if (ret < 0)
+ goto err_net;
+
if (net_start(ct))
goto err_net;
@@ -463,8 +463,8 @@ static ct_process_t __local_enter_cb(ct_handler_t h, ct_process_desc_t ph, int (
struct container *ct = cth2ct(h);
struct process_desc *p = prh2pr(ph);
struct process *pr;
- int aux = -1, pid;
- int wait_pipe[2];
+ int aux = -1, pid = -1;
+ int wait_pipe[2] = {-1, -1}, child_wait_pipe[2] = {-1, -1};
if (ct->state != CT_RUNNING)
return ERR_PTR(-LCTERR_BADCTSTATE);
@@ -480,16 +480,20 @@ static ct_process_t __local_enter_cb(ct_handler_t h, ct_process_desc_t ph, int (
local_process_init(pr);
- if (pipe(wait_pipe)) {
- xfree(pr);
- return ERR_PTR(-1);
- }
+ if (pipe(wait_pipe))
+ goto err;
+ if (pipe(child_wait_pipe))
+ goto err;
pid = fork();
if (pid == 0) {
struct ns_desc *ns;
close(wait_pipe[0]);
+ close(child_wait_pipe[1]);
+
+ if (spawn_wait_and_close(child_wait_pipe)) /* wait cgroups */
+ exit(1);
if (p->fds) {
p->fds[p->fdn] = wait_pipe[1];
@@ -512,9 +516,6 @@ static ct_process_t __local_enter_cb(ct_handler_t h, ct_process_desc_t ph, int (
exit(-1);
}
- if (cgroups_attach(ct))
- exit(-1);
-
if (ct->root_path && !(ct->nsmask & CLONE_NEWNS)) {
char nroot[128];
@@ -541,12 +542,17 @@ static ct_process_t __local_enter_cb(ct_handler_t h, ct_process_desc_t ph, int (
spawn_wake_and_close(wait_pipe, -1);
exit(aux);
}
-
close(wait_pipe[1]);
+ close(child_wait_pipe[0]);
if (aux >= 0)
restore_ns(aux, &pid_ns);
+ if (cgroups_attach(ct, pid))
+ goto err;
+
+ spawn_wake_and_close(child_wait_pipe, 0);
+
if (spawn_wait(wait_pipe))
goto err;
@@ -559,7 +565,11 @@ static ct_process_t __local_enter_cb(ct_handler_t h, ct_process_desc_t ph, int (
err:
xfree(pr);
close(wait_pipe[0]);
- waitpid(pid, NULL, 0);
+ close(wait_pipe[1]);
+ close(child_wait_pipe[0]);
+ close(child_wait_pipe[1]);
+ if (pid > 0)
+ waitpid(pid, NULL, 0);
return ERR_PTR(-1);
}
diff --git a/src/include/cgroups.h b/src/include/cgroups.h
index 8963ffa..cbf130f 100644
--- a/src/include/cgroups.h
+++ b/src/include/cgroups.h
@@ -36,7 +36,7 @@ extern struct cg_desc cg_descs[];
extern int cgroup_create_one(struct container *ct, struct controller *ctl);
extern int cgroups_create(struct container *ct);
-extern int cgroups_attach(struct container *ct);
+extern int cgroups_attach(struct container *ct, pid_t pid);
extern void cgroups_destroy(struct container *ct);
extern void cgroups_free(struct container *ct);
--
1.9.1
More information about the Libct
mailing list