[Libct] [PATCH v2 6/6] VZ containers: implement some resources constraints
Alexander Burluka
aburluka at parallels.com
Wed Nov 12 04:52:20 PST 2014
Memory, cpu and blkio limits. Implementation + tests
---
src/cgroups.c | 6 +-
src/include/cgroups.h | 2 +
src/vz.c | 251 ++++++++++++++++++++++++++++++++++++++++-------
test/Makefile | 4 +-
test/vz_cgroup_blkio.c | 112 +++++++++++++++++++++
test/vz_cgroup_cpu.c | 78 +++++++++++++++
test/vz_cgroup_memory.c | 89 +++++++++++++++++
7 files changed, 501 insertions(+), 41 deletions(-)
create mode 100644 test/vz_cgroup_blkio.c
create mode 100644 test/vz_cgroup_cpu.c
create mode 100644 test/vz_cgroup_memory.c
diff --git a/src/cgroups.c b/src/cgroups.c
index d2d8d5b..4d96249 100644
--- a/src/cgroups.c
+++ b/src/cgroups.c
@@ -170,14 +170,14 @@ static struct cg_config *cg_config_alloc(enum ct_controller ctype, char *param,
return cg;
}
-static int config_controller(struct container *ct, enum ct_controller ctype,
+int config_controller(struct container *ct, enum ct_controller ctype,
char *param, char *value)
{
char path[PATH_MAX], *t;
int fd, ret;
t = cgroup_get_path(ctype, path, sizeof(path));
- snprintf(t, sizeof(path) - (t - path), "/%s/%s", ct->name, param);
+ snprintf(t, sizeof(path) - (t - path), "/%s/%s.%s", ct->name, cg_descs[ctype].name, param);
ret = fd = open(path, O_WRONLY);
if (fd >= 0) {
@@ -228,7 +228,7 @@ int local_config_controller(ct_handler_t h, enum ct_controller ctype,
return config_controller(ct, ctype, param, value) ? -LCTERR_CGCONFIG : 0;
}
-static int cgroup_create_one(struct container *ct, struct controller *ctl)
+int cgroup_create_one(struct container *ct, struct controller *ctl)
{
char path[PATH_MAX], *t;
diff --git a/src/include/cgroups.h b/src/include/cgroups.h
index 43639df..8963ffa 100644
--- a/src/include/cgroups.h
+++ b/src/include/cgroups.h
@@ -34,6 +34,7 @@ struct cg_config {
extern struct cg_desc cg_descs[];
+extern int cgroup_create_one(struct container *ct, struct controller *ctl);
extern int cgroups_create(struct container *ct);
extern int cgroups_attach(struct container *ct);
extern void cgroups_destroy(struct container *ct);
@@ -41,6 +42,7 @@ extern void cgroups_free(struct container *ct);
extern int local_add_controller(ct_handler_t h, enum ct_controller ctype);
extern int local_config_controller(ct_handler_t h, enum ct_controller ctype, char *param, char *value);
+extern int config_controller(struct container *ct, enum ct_controller ctype, char *param, char *value);
extern int try_mount_cg(struct container *ct);
diff --git a/src/vz.c b/src/vz.c
index 39f788c..2611bc7 100644
--- a/src/vz.c
+++ b/src/vz.c
@@ -23,6 +23,7 @@
#include <grp.h>
#include <limits.h>
#include <sched.h>
+#include <bc/beancounter.h>
#include "linux-kernel.h"
#include "vz.h"
@@ -43,6 +44,9 @@
#define LINUX_REBOOT_MAGIC1 0xfee1dead
#define LINUX_REBOOT_MAGIC2 672274793
#define LINUX_REBOOT_CMD_POWER_OFF 0x4321FEDC
+#define IOPRIO_CLASS_BE 2
+#define IOPRIO_CLASS_SHIFT 13
+#define IOPRIO_WHO_UBC 1000
typedef enum {
M_HALT,
@@ -156,6 +160,7 @@ static void vz_ct_destroy(ct_handler_t h)
{
struct container *ct = cth2ct(h);
+ cgroups_free(ct);
fs_free(ct);
net_release(ct);
@@ -648,6 +653,206 @@ static int vz_spawn_cb(ct_handler_t h, ct_process_desc_t p, int (*cb)(void *), v
return -1;
}
+static int ublimit_mem_syscall(unsigned int veid, int type, unsigned long value)
+{
+ unsigned long param[2] = {value, value};
+
+ return syscall(__NR_setublimit, veid, type, param);
+}
+
+static int vz_set_memory_param(struct container *ct, char *param, char *value)
+{
+ unsigned long ram = 0;
+ unsigned long swap = 0;
+ float overcommit = LONG_MAX;
+ float memory;
+ unsigned int veid = 0;;
+
+ if (parse_uint(ct->name, &veid) < 0) {
+ pr_err("Unable to parse container's ID");
+ return -1;
+ }
+
+ if (strcmp(param, "limit_in_bytes") == 0) {
+ if (parse_uint(value, (unsigned int *)&ram) < 0) {
+ pr_err("Unable to parse container's RAM");
+ return -1;
+ }
+ ram /= getpagesize();
+
+ if (ublimit_mem_syscall(veid, UB_PHYSPAGES, ram)) {
+ pr_perror("Unable to set UB_LOCKEDPAGES");
+ return -1;
+ }
+
+ if (ublimit_mem_syscall(veid, UB_SWAPPAGES, 0)) {
+ pr_perror("Unable to set UB_LOCKEDPAGES");
+ return -1;
+ }
+
+ if (ublimit_mem_syscall(veid, UB_LOCKEDPAGES, ram)) {
+ pr_perror("Unable to set UB_LOCKEDPAGES");
+ return -1;
+ }
+
+ if (ublimit_mem_syscall(veid, UB_OOMGUARPAGES, ram)) {
+ pr_perror("Unable to set UB_OOMGUARPAGES");
+ return -1;
+ }
+
+ if (ublimit_mem_syscall(veid, UB_VMGUARPAGES, ram + swap)) {
+ pr_perror("Unable to set UB_VMGUARPAGES");
+ return -1;
+ }
+
+ memory = (ram + swap) * overcommit;
+ if (memory > LONG_MAX)
+ memory = UINT_MAX;
+
+ if (ublimit_mem_syscall(veid, UB_PRIVVMPAGES, memory)) {
+ pr_perror("Unable to set UB_PRIVVMPAGES");
+ return -1;
+ }
+ return 0;
+ }
+
+ pr_err("Unsupported param for CTL_MEMORY: %s", param);
+ return -1;
+}
+
+static int vzctl2_set_ioprio(unsigned veid, int prio)
+{
+ int ret;
+
+ if (prio < 0)
+ return -LCTERR_BADARG;
+
+ pr_info("Set up ioprio: %d", prio);
+ ret = syscall(__NR_ioprio_set, IOPRIO_WHO_UBC, veid,
+ prio | IOPRIO_CLASS_BE << IOPRIO_CLASS_SHIFT);
+ if (ret) {
+ if (errno == ESRCH) {
+ pr_err("Container is not running");
+ return -LCTERR_BADCTSTATE;
+ }
+ else if (errno == EINVAL) {
+ pr_warn("ioprio feature is not supported"
+ " by the kernel: ioprio configuration is skippe");
+ return -LCTERR_OPNOTSUPP;
+ }
+ pr_perror("Unable to set ioprio");
+ return -1;
+ }
+ return 0;
+}
+
+static int vzctl2_set_iopslimit(unsigned veid, int limit)
+{
+ int ret;
+ struct iolimit_state io;
+
+ if (limit < 0)
+ return -LCTERR_BADARG;
+ io.id = veid;
+ io.speed = limit;
+ io.burst = limit * 3;
+ io.latency = 10*1000;
+ pr_info("Set up iopslimit: %d", limit);
+ ret = ioctl(get_vzctlfd(), VZCTL_SET_IOPSLIMIT, &io);
+ if (ret) {
+ if (errno == ESRCH) {
+ pr_err("Container is not running");
+ return -LCTERR_BADCTSTATE;
+ }
+ else if (errno == ENOTTY) {
+ pr_warn("iopslimit feature is not supported"
+ " by the kernel; iopslimit configuration is skipped");
+ return -LCTERR_OPNOTSUPP;
+ }
+ pr_perror("Unable to set iopslimit");
+ return -1;
+ }
+ return 0;
+}
+
+
+static int vz_set_io_param(struct container *ct, char *param, char *value)
+{
+ unsigned int veid = 0;
+
+ if (parse_uint(ct->name, &veid) < 0) {
+ pr_err("Unable to parse container's ID");
+ return -1;
+ }
+
+ if (strcmp(param, "weight") == 0) {
+ int prio = -1;
+ if (parse_int(value, &prio)) {
+ pr_err("Unable to parse priority from '%s'", value);
+ return -1;
+ }
+ return vzctl2_set_ioprio(veid, prio);
+ } else if (strcmp(param, "throttle.write_iops_device") == 0 ||
+ strcmp(param, "throttle.read_iops_device") == 0) {
+ int limit = -1;
+ if (parse_int(value, &limit)) {
+ pr_err("Unable to parse limit from '%s'", value);
+ return -1;
+ }
+ return vzctl2_set_iopslimit(veid, limit);
+ }
+
+ pr_err("Unsupported param for CTL_BLKIO: %s", param);
+ return -1;
+}
+
+static int vz_resources_create(struct container *ct)
+{
+ struct controller *ctl;
+ struct cg_config *cfg;
+ int ret = 0;
+
+ list_for_each_entry(ctl, &ct->cgroups, ct_l) {
+ switch (ctl->ctype) {
+ case CTL_MEMORY:
+ break;
+ case CTL_BLKIO:
+ break;
+ case CTL_CPUSET:
+ break;
+ default:
+ ret = cgroup_create_one(ct, ctl);
+ if (ret)
+ return -LCTERR_CGCREATE;
+ break;
+ }
+ }
+
+ list_for_each_entry(cfg, &ct->cg_configs, l) {
+ switch (cfg->ctype) {
+ case CTL_MEMORY:
+ ret = vz_set_memory_param(ct, cfg->param, cfg->value);
+ if (ret)
+ return -LCTERR_CGCONFIG;
+ break;
+ case CTL_BLKIO:
+ ret = vz_set_io_param(ct, cfg->param, cfg->value);
+ if (ret)
+ return -LCTERR_CGCONFIG;
+ break;
+ default:
+ ret = config_controller(ct, cfg->ctype, cfg->param, cfg->value);
+ if (ret) {
+ pr_err("local_config_controller failed %d", ret);
+ return -LCTERR_CGCONFIG;
+ }
+ break;
+ }
+ }
+
+ return 0;
+}
+
static int vz_spawn_execve(ct_handler_t h, ct_process_desc_t p, char *path, char **argv, char **env, int *fds)
{
int ret = -1;
@@ -727,6 +932,12 @@ static int vz_spawn_execve(ct_handler_t h, ct_process_desc_t p, char *path, char
goto err_net;
}
+ ret = vz_resources_create(ct);
+ if (ret) {
+ pr_err("vz_resourse_create");
+ goto err_res;
+ }
+
ret = net_start(ct);
if (ret) {
pr_err("Unable to start network");
@@ -748,6 +959,7 @@ static int vz_spawn_execve(ct_handler_t h, ct_process_desc_t p, char *path, char
err_wait:
net_stop(ct);
err_net:
+err_res:
proc_wake_close(child_wait, -1);
err_fork:
fs_umount(ct);
@@ -823,35 +1035,6 @@ static int wait_env_state(unsigned int veid, int state, unsigned int timeout)
return -1;
}
-static int vzctl2_set_iopslimit(unsigned veid, int limit)
-{
- int ret;
- struct iolimit_state io;
-
- if (limit < 0)
- return -LCTERR_BADARG;
- io.id = veid;
- io.speed = limit;
- io.burst = limit * 3;
- io.latency = 10*1000;
- pr_info("Set up iopslimit: %d", limit);
- ret = ioctl(get_vzctlfd(), VZCTL_SET_IOPSLIMIT, &io);
- if (ret) {
- if (errno == ESRCH) {
- pr_err("Container is not running");
- return -LCTERR_BADCTSTATE;
- }
- else if (errno == ENOTTY) {
- pr_warn("iopslimit feature is not supported"
- " by the kernel; iopslimit configuration is skipped");
- return -LCTERR_OPNOTSUPP;
- }
- pr_perror("Unable to set iopslimit");
- return -1;
- }
- return 0;
-}
-
static int real_env_stop(int stop_mode)
{
int fd;
@@ -1042,14 +1225,8 @@ static int vz_set_nsmask(ct_handler_t h, unsigned long nsmask)
return 0;
}
-static int vz_config_controller(ct_handler_t h, enum ct_controller ctype,
- char *param, char *value)
-{
- pr_perror("Controller configuration are not supported");
- return -LCTERR_CGCONFIG;
-}
-
static int vz_enter_execve(ct_handler_t h, ct_process_desc_t p, char *path, char **argv, char **env, int *fds)
+{
struct container *ct = NULL;
unsigned int veid = -1;
int pid, child_pid, ret = 0;
@@ -1140,7 +1317,7 @@ static const struct container_ops vz_ct_ops = {
.detach = vz_ct_destroy,
.set_nsmask = vz_set_nsmask,
.add_controller = local_add_controller,
- .config_controller = vz_config_controller,
+ .config_controller = local_config_controller,
.fs_set_root = local_fs_set_root,
.fs_set_private = local_fs_set_private,
.fs_add_mount = local_add_mount,
diff --git a/test/Makefile b/test/Makefile
index 4d07094..c9f10c3 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -4,13 +4,15 @@ TESTS = ct_create ct_enter ct_proc ct_root ct_root_enter \
ct_ext_mount ct_private_subdir_ns \
ct_cgroup_sub ct_service ct_kill_nons ct_pid_enter \
ct_userns ct_caps \
- vz_create_exec vz_net_veth vz_enter
+ vz_create_exec vz_net_veth vz_enter \
+ vz_cgroup_memory vz_cgroup_cpu vz_cgroup_blkio
PIGS = file_piggy
CC = gcc
CFLAGS = -I../src/include/uapi/ -g -Wall -Werror
ifneq ("$(wildcard /proc/vz)","")
+ CFLAGS += -I/usr/src/kernels/$(shell uname -r)/include/
CFLAGS += -DVZ
endif
LIBNLDIR = ../.shipped/libnl/lib/.libs
diff --git a/test/vz_cgroup_blkio.c b/test/vz_cgroup_blkio.c
new file mode 100644
index 0000000..cde1dd3
--- /dev/null
+++ b/test/vz_cgroup_blkio.c
@@ -0,0 +1,112 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <libct.h>
+#include <unistd.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <linux/vziolimit.h>
+#include <sys/ioctl.h>
+
+#include "test.h"
+
+#define FS_ROOT "/"
+#define VZCTLDEV "/dev/vzctl"
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+#define CT_ID 1339
+#define CT_NAME STR(CT_ID)
+#define IOPSLIMIT 1
+#define IOPSLIMIT_STR STR(IOPSLIMIT)
+#define IOPRIOLIMIT 7
+#define IOPRIOLIMIT_STR STR(IOPRIOLIMIT)
+
+int is_iopslimit_correct(unsigned int expected_limit)
+{
+ int fd = -1;
+ int ret = 0;
+ struct iolimit_state io;
+
+ fd = open(VZCTLDEV, O_RDWR);
+ if (fd == -1) {
+ fprintf(stderr, "Unable to open %s!\n", VZCTLDEV);
+ goto err;
+ }
+
+ io.id = CT_ID;
+ if (ioctl(fd, VZCTL_GET_IOPSLIMIT, &io)) {
+ perror("ioctl");
+ goto err;
+ }
+ ret = (io.speed == expected_limit);
+
+err:
+ close(fd);
+ return ret;
+}
+
+int is_iopriolimit_correct(unsigned int expected_limit)
+{
+ int ret = 0;
+ FILE *f = NULL;
+ int limit;
+ char buf[1024];
+
+ f = fopen("/proc/bc/" CT_NAME "/ioprio", "r");
+ if (!f) {
+ fprintf(stderr, "Unable to open /proc/bc/" CT_NAME "/ioprio!\n");
+ goto err;
+ }
+
+ if (fscanf(f, "%s %d", buf, &limit) != 2) {
+ fprintf(stderr, "fscanf failed!\n");
+ goto err;
+ }
+ ret = (limit == expected_limit);
+err:
+ fclose(f);
+ return ret;
+}
+
+int main(int argc, char *argv[])
+{
+ libct_session_t s;
+ ct_handler_t ct;
+ ct_process_desc_t p;
+ char *sleep_a[] = { "sleep", "2", NULL};
+ int fds[] = {STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO};
+ int pid;
+
+ s = libct_session_open_local();
+ ct = libct_container_create(s, CT_NAME);
+ p = libct_process_desc_create(s);
+ libct_fs_set_root(ct, FS_ROOT);
+
+ libct_container_set_nsmask(ct,
+ CLONE_NEWNS |
+ CLONE_NEWUTS |
+ CLONE_NEWIPC |
+ CLONE_NEWNET |
+ CLONE_NEWPID);
+
+ libct_controller_add(ct, CTL_BLKIO);
+ libct_controller_configure(ct, CTL_BLKIO, "throttle.write_iops_device", IOPSLIMIT_STR);
+ libct_controller_configure(ct, CTL_BLKIO, "weight", IOPRIOLIMIT_STR);
+ if ((pid = libct_container_spawn_execvfds(ct, p, "/bin/sleep", sleep_a, fds)) <= 0)
+ goto err;
+
+ if (!is_iopslimit_correct(IOPSLIMIT))
+ goto err;
+ if (!is_iopriolimit_correct(IOPRIOLIMIT))
+ goto err;
+
+ libct_container_wait(ct);
+ libct_container_destroy(ct);
+
+ libct_session_close(s);
+
+ return pass("All is ok");;
+err:
+ return fail("Something wrong");
+}
diff --git a/test/vz_cgroup_cpu.c b/test/vz_cgroup_cpu.c
new file mode 100644
index 0000000..a322155
--- /dev/null
+++ b/test/vz_cgroup_cpu.c
@@ -0,0 +1,78 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <libct.h>
+#include <unistd.h>
+#include <string.h>
+#include <sched.h>
+
+#include "test.h"
+
+#define FS_ROOT "/"
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+#define CT_ID 1339
+#define CT_NAME STR(CT_ID)
+#define CPUS "0-1,3"
+
+int is_cpu_count_correct(const char *expected_cpus)
+{
+ int ret = 0;
+ FILE *f = NULL;
+ char buf[1024] = {'\0'};
+
+ f = fopen("/proc/vz/fairsched/" CT_NAME "/cpuset.cpus", "r");
+ if (!f) {
+ fprintf(stderr, "Unable to open /proc/vz/fairsched/" CT_NAME "/cpuset.cpus!\n");
+ goto err;
+ }
+
+ if (fscanf(f, "%s", buf) != 1) {
+ fprintf(stderr, "fscanf failed!\n");
+ goto err;
+ }
+ ret = !strcmp(buf, expected_cpus);
+err:
+ fclose(f);
+ return ret;
+}
+
+
+int main(int argc, char *argv[])
+{
+ libct_session_t s;
+ ct_handler_t ct;
+ ct_process_desc_t p;
+ char *sleep_a[] = { "sleep", "2", NULL};
+ int fds[] = {STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO};
+
+ s = libct_session_open_local();
+ ct = libct_container_create(s, CT_NAME);
+ p = libct_process_desc_create(s);
+ libct_fs_set_root(ct, FS_ROOT);
+
+ libct_container_set_nsmask(ct,
+ CLONE_NEWNS |
+ CLONE_NEWUTS |
+ CLONE_NEWIPC |
+ CLONE_NEWNET |
+ CLONE_NEWPID);
+
+ libct_controller_add(ct, CTL_CPUSET);
+ libct_controller_configure(ct, CTL_CPUSET, "cpus", CPUS);
+ if (libct_container_spawn_execvfds(ct, p, "/bin/sleep", sleep_a, fds) <= 0)
+ goto err;
+
+ if (!is_cpu_count_correct(CPUS))
+ goto err;
+
+ libct_container_wait(ct);
+ libct_container_destroy(ct);
+
+ libct_session_close(s);
+
+ return pass("All is ok");;
+err:
+ return fail("Something wrong");
+}
diff --git a/test/vz_cgroup_memory.c b/test/vz_cgroup_memory.c
new file mode 100644
index 0000000..190bdbc
--- /dev/null
+++ b/test/vz_cgroup_memory.c
@@ -0,0 +1,89 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <libct.h>
+#include <unistd.h>
+#include <sched.h>
+
+#include "test.h"
+
+#define FS_ROOT "/"
+
+#define STR_HELPER(x) #x
+#define STR(x) STR_HELPER(x)
+
+#define MEMLIMIT 134217728
+#define MEMLIMIT_STR STR(MEMLIMIT)
+
+#define CT_ID 1339
+#define CT_NAME STR(CT_ID)
+
+int is_memory_correct(unsigned long expected_limit)
+{
+ int ret = 0;
+ FILE *f = NULL;
+ char buf[1024] = {'\0'};
+ unsigned long flds[5] = {0};
+ unsigned long limit = 0;
+
+ f = fopen("/proc/bc/" CT_NAME "/resources", "r");
+ if (!f) {
+ fprintf(stderr, "Unable to open /proc/bc/" CT_NAME "/resources!\n");
+ goto err;
+ }
+
+ while (fscanf(f, "%s %lu %lu %lu %lu %lu", buf, &flds[0], &flds[1], &flds[2], &flds[3], &flds[4]) == 6) {
+ if (strcmp(buf, "physpages") == 0 && flds[2] == flds[3]) {
+ limit = flds[2] * getpagesize();
+ break;
+ }
+ }
+ if (!limit) {
+ fprintf(stderr, "unable to read \n");
+ goto err;
+ }
+ ret = (limit == expected_limit);
+err:
+ fclose(f);
+ return ret;
+}
+
+
+int main(int argc, char *argv[])
+{
+ libct_session_t s;
+ ct_handler_t ct;
+ ct_process_desc_t p;
+ char *run_a[3] = { "sleep", "2", NULL};
+ int fds[] = {STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO};
+
+ s = libct_session_open_local();
+ ct = libct_container_create(s, CT_NAME);
+ p = libct_process_desc_create(s);
+ libct_fs_set_root(ct, FS_ROOT);
+
+ libct_container_set_nsmask(ct,
+ CLONE_NEWNS |
+ CLONE_NEWUTS |
+ CLONE_NEWIPC |
+ CLONE_NEWNET |
+ CLONE_NEWPID);
+
+ libct_controller_add(ct, CTL_MEMORY);
+ libct_controller_configure(ct, CTL_MEMORY, "limit_in_bytes", MEMLIMIT_STR);
+
+ if (libct_container_spawn_execvfds(ct, p, "/bin/sleep", run_a, fds) <= 0)
+ goto err;
+
+ if (!is_memory_correct(MEMLIMIT))
+ goto err;
+
+ libct_container_wait(ct);
+ libct_container_destroy(ct);
+
+ libct_session_close(s);
+
+ return pass("All is ok");;
+err:
+ return fail("Something wrong");
+}
--
1.7.1
More information about the Libct
mailing list