[CRIU] [RFC PATCH 17/21] zdtm: add the multi-test launcher
Andrew Vagin
avagin at virtuozzo.com
Thu Feb 25 13:49:49 PST 2016
On Fri, Feb 19, 2016 at 05:50:40PM +0300, Ivan Shapovalov wrote:
> This program works as a "host" to multiple test binaries, which are
> loaded into the same address space and then executed all at once.
>
> The launcher tries to avoid using threads (to reduce probability of
> affecting the test results) and we do not alter the tests themselves.
> Instead, we hook various functions in libzdtmtst and use <ucontext.h>
> to redirect execution flow between the launcher and tests.
>
> That is, first we run each test from start of their main() up to entry
> to test_init(), where each test is interrupted and contexts are saved.
> After all tests have been run to that point, real test_init() is
> performed "once and for all". The same method is used repeatedly for all
> phases of tests' execution.
>
> The only case when we use threads is when some of tests want to run a
> busy loop during C/R. In this case we spawn a thread (using clone(), not
> pthreads) for each such test. After C/R we join all threads and do
> verification in the usual sequential mode.
[avagin at laptop criu]$ git grep '\<wait\>(' test/zdtm | wc -l
66
Many tests calls wait() which waits one of children. How do you handle
this in the launcher? Actually a test wants to wait its children, it
doesn't expect to meet other processes.
>
> Signed-off-by: Ivan Shapovalov <intelfx at intelfx.name>
> ---
> criu/include/compiler.h | 1 +
> test/zdtm/.gitignore | 1 +
> test/zdtm/Makefile | 14 +-
> test/zdtm/launcher/Makefile | 32 ++
> test/zdtm/launcher/launcher.c | 831 +++++++++++++++++++++++++++++++++++++++
> test/zdtm/lib/test.c | 57 ++-
> test/zdtm/lib/zdtmtst.h | 24 +-
> test/zdtm/lib/zdtmtst_internal.h | 28 ++
> 8 files changed, 973 insertions(+), 15 deletions(-)
> create mode 100644 test/zdtm/launcher/Makefile
> create mode 100644 test/zdtm/launcher/launcher.c
>
> diff --git a/criu/include/compiler.h b/criu/include/compiler.h
> index 37a6880..56fd87c 100644
> --- a/criu/include/compiler.h
> +++ b/criu/include/compiler.h
> @@ -18,6 +18,7 @@
> #define __used __attribute__((__used__))
> #define __maybe_unused __attribute__((unused))
> #define __always_unused __attribute__((unused))
> +#define __weak __attribute__((weak))
>
> #define __section(S) __attribute__ ((__section__(#S)))
>
> diff --git a/test/zdtm/.gitignore b/test/zdtm/.gitignore
> index ce4e302..471496a 100644
> --- a/test/zdtm/.gitignore
> +++ b/test/zdtm/.gitignore
> @@ -1,3 +1,4 @@
> +/launcher/launcher
> /lib/libzdtmtst.so
> /live/static/apparmor
> /live/static/arm-neon00
> diff --git a/test/zdtm/Makefile b/test/zdtm/Makefile
> index 4d66395..a437b86 100644
> --- a/test/zdtm/Makefile
> +++ b/test/zdtm/Makefile
> @@ -1,15 +1,17 @@
> -SUBDIRS = lib live
> +SUBDIRS = lib live launcher
>
> default: all
> .PHONY: default lib live
>
> -lib:
> - $(MAKE) -C lib all
> +$(SUBDIRS):
> + $(MAKE) -C $@ all
>
> -live: lib
> - $(MAKE) -C live all
> +live: lib launcher
>
> -all: lib live
> +launcher: lib
> +
> +all: $(SUBDIRS)
> @true
> +
> %:
> set -e; for d in $(SUBDIRS); do $(MAKE) -C $$d $@; done
> diff --git a/test/zdtm/launcher/Makefile b/test/zdtm/launcher/Makefile
> new file mode 100644
> index 0000000..7b00dd3
> --- /dev/null
> +++ b/test/zdtm/launcher/Makefile
> @@ -0,0 +1,32 @@
> +include ../Makefile.inc
> +
> +LIBDIR = ../lib
> +LDLIBS = -lzdtmtst
> +override LDFLAGS += -L$(LIBDIR) -Wl,-rpath,\$$ORIGIN/$(LIBDIR)
> +
> +PROGS = launcher
> +SRC = $(PROGS:%=%.c)
> +OBJ = $(SRC:%.c=%.o)
> +DEP = $(SRC:%.c=%.d)
> +
> +DEPEND.c = $(COMPILE.c) -MM -MP
> +%.d: %.c
> + $(DEPEND.c) $(OUTPUT_OPTION) $<
> +
> +all: $(PROGS)
> +
> +launcher: override LDLIBS += -ldl
> +
> +clean:
> + $(RM) -f $(OBJ) $(PROGS) *~
> +
> +cleandep: clean
> + $(RM) -f $(DEP)
> +
> +realclean: cleandep
> +
> +.PHONY: clean cleandep realclean
> +
> +ifeq ($(filter-out no-deps-targets, $(MAKECMDGOALS)),)
> +-include $(DEP)
> +endif
> diff --git a/test/zdtm/launcher/launcher.c b/test/zdtm/launcher/launcher.c
> new file mode 100644
> index 0000000..82d7b01
> --- /dev/null
> +++ b/test/zdtm/launcher/launcher.c
> @@ -0,0 +1,831 @@
> +#define _GNU_SOURCE
> +#include <dlfcn.h>
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <signal.h>
> +#include <ucontext.h>
> +#include <linux/unistd.h>
> +#include <sys/wait.h>
> +#include <sys/syscall.h>
> +#include <sched.h>
> +
> +#include "zdtmtst.h"
> +#include "zdtmtst_internal.h"
> +#include "lock.h"
> +
> +#define memzero(arg) memset(&arg, 0, sizeof(arg))
> +
> +/* options defined in libzdtmtst itself */
> +static struct long_opt *global_opts_head;
> +
> +/* global variables for the context switching mechanism */
> +static ucontext_t return_ctx;
> +static struct test *current_test, *all_tests;
> +
> +/* real exit() function */
> +static NORETURN void(*real_exit)(int);
> +
> +/* main thread id of the launcher */
> +static pid_t master_tid;
> +
> +/* used to wait until a thread's temporary stack becomes unused */
> +static futex_t test_thread_futex;
> +
> +/* used to enable hooks while we are in test context */
> +static int multitest_enabled;
> +
> +/* used to disable hooks for the main thread while busy-loop threads are
> + * active */
> +static int threading_enabled;
> +
> +/* used to tell whether the last test returned with some signals pending
> + * (which means that we must not run any more tests) */
> +static int have_pending_signals;
> +
> +static long sys_gettid(void)
> +{
> + return syscall(__NR_gettid);
> +}
> +
> +static NORETURN void sys_exit(int status)
> +{
> + syscall(__NR_exit, status);
> + __builtin_unreachable();
> +}
> +
> +static const char *test_phase_str[] = {
> + [PHASE_START] = "<main not started>",
> + [PHASE_TEST_INIT] = "test_init",
> + [PHASE_TEST_EXT_INIT] = "test_ext_init",
> + [PHASE_TEST_DAEMON] = "test_daemon",
> + [PHASE_TEST_WAITSIG] = "test_waitsig/test_go",
> + [PHASE_RETURNED] = "<main finished>",
> +};
> +
> +/* describes a single test executable which is dlopen()'ed */
> +struct test
> +{
> + struct test *next;
> +
> + void *handle;
> + const char *name;
> + struct long_opt *opts_head;
> + int (*main) (int, char **);
> +
> + int argc;
> + char **argv;
> +
> + /* for the freer */
> + int is_last;
> +
> + /* whether this test needs a separate thread for its busy-loop */
> + int is_threaded;
> +
> + /* for threading efficiency evaluation */
> + int busy_loop_iters;
> +
> + /* whether the test is currently executing (used with getcontext() to return from threads) */
> + int in_progress;
> +
> + /* whether this specific test returned with some signals pending */
> +
> + /* whether the test did actually return from main(), as opposed to doing a swapcontext() */
> + enum test_phase phase;
> + int retcode;
> + enum test_result result;
> +
> + /* the context where the test execution left off and returned to the launcher */
> + ucontext_t ctx;
> +
> + /* the tid of the test's busy loop thread (if any) */
> + union {
> + int tid;
> + futex_t tid_futex;
> + };
> +
> + /* the alternate stack for the test execution context */
> + char ctx_stack[8*1<<20];
> +};
> +
> +/* deallocates a heap-allocated list of struct test */
> +static void tests_free(struct test **arg)
> +{
> + if (*arg != NULL) {
> + for (struct test *test = *arg, *next; test != NULL; test = next) {
> + next = test->next;
> + dlclose(test->handle);
> + free(test);
> + }
> + }
> +}
> +
> +static void test_check_ctx(struct test *test)
> +{
> + if (test->phase >= PHASE_RETURNED) {
> + pr_err("Cannot jump to context of test \"%s\", context invalid: %s\n", test->name, test_phase_str[test->phase]);
> + abort();
> + }
> +}
> +
> +static struct test *test_current(void)
> +{
> + assert(getpid() == master_pid);
> +
> + pid_t tid = sys_gettid();
> + if (tid == master_tid) {
> + assert(current_test != NULL);
> + return current_test;
> + } else {
> + assert(current_test == NULL);
> + assert(all_tests != NULL);
> +
> + for (struct test *test = all_tests; test != NULL; test = test->next) {
> + /* test->tid may change behind us due to
> + * CLONE_CHILD_CLEARTID, but _our_ test->tid
> + * won't change while we are alive. */
> + if (test->tid == tid) {
> + return test;
> + }
> + }
> +
> + pr_err("Could not find a test descriptor for tid %d (master tid is %d)\n", tid, master_tid);
> + abort();
> + }
> +}
> +
> +/* a wrapper for the test's main(), executed in an own context */
> +static void test_main(void)
> +{
> + struct test *test = test_current();
> +
> + futex_set_and_wake(&test_thread_futex, 1);
> +
> + int r = test->main(test->argc, test->argv);
> +
> + /* if this is a test's ancillary child process, exit right away */
> + if (getpid() != master_pid) {
> + real_exit(r);
> + }
> +
> + test->phase = PHASE_RETURNED;
> + test->retcode = r;
> + memset(&test->ctx, 0, sizeof(test->ctx));
> +
> + if (sys_gettid() == master_tid) {
> + /* no-op, we return to uc_link of the test context */
> + } else {
> + sys_exit(0);
> + }
> +}
> +
> +/* loads a test binary and fills out the struct test instance */
> +static int test_make(struct test *test, const ucontext_t *ctx)
> +{
> + assert(opt_head == NULL);
> +
> + /* dlopen() the test binary; the constructors will execute at this point and register the test's options */
> + test->handle = dlopen(test->name, RTLD_NOW);
> + if (test->handle == NULL) {
> + pr_err("Failed to dlopen(\"%s\"): %s\n", test->name, dlerror());
> + return -1;
> + }
> +
> + /* save the per-test options */
> + test->opts_head = opt_head;
> + opt_head = NULL;
> +
> + /* get the entry point */
> + test->main = dlsym(test->handle, "main");
> + if (test->main == NULL) {
> + pr_err("Failed to dlsym(\"main\") in \"%s\": %s\n", test->name, dlerror());
> + return -1;
> + }
> +
> + /* prepare the test's execution context */
> + test->ctx = *ctx;
> + test->ctx.uc_stack.ss_sp = test->ctx_stack;
> + test->ctx.uc_stack.ss_size = sizeof(test->ctx_stack);
> + test->ctx.uc_link = &return_ctx;
> +
> + makecontext(&test->ctx, &test_main, 0);
> +
> + return 0;
> +}
> +
> +/* sets all communication global variables and switches to the test context */
> +static void test_run(struct test *test)
> +{
> + int r;
> +
> + test_check_ctx(test);
> +
> + if (have_pending_signals) {
> + pr_err("Trying to switch to test \"%s\" with signals pending from previous tests -- such tests must be last in sequence\n", test->name);
> + abort();
> + }
> +
> + opt_head = test->opts_head;
> + current_test = test;
> + multitest_enabled = 1;
> +
> + test->in_progress = 1;
> + r = swapcontext(&return_ctx, &test->ctx);
> +
> + if (r < 0) {
> + pr_perror("Failed to swapcontext() to switch to the test context for \"%s\"", test->name);
> + abort();
> + }
> +
> + opt_head = NULL;
> + current_test = NULL;
> + multitest_enabled = 0;
> +}
> +
> +static int test_thread_main(void *arg)
> +{
> + struct test *test = arg;
> + assert(test != NULL);
> +
> + int r = setcontext(&test->ctx);
> +
> + pr_perror("Failed to setcontext() in busy loop thread to switch to the test context");
> + abort();
> +}
> +
> +/* creates an additional thread and switches to the test context there */
> +static void test_run_thread(struct test *test)
> +{
> + int r;
> +
> + /*
> + * Note that we do not set any global variables and expect
> + * multitest_enabled to be set by the caller. This is because we
> + * do not use TLS for these variables to avoid increasing the process
> + * environment complexity too much.
> + */
> +
> + assert(multitest_enabled == 1);
> + assert(threading_enabled == 1);
> + assert(current_test == NULL);
> +
> + test_check_ctx(test);
> +
> + if (have_pending_signals) {
> + pr_err("Trying to create a thread for test \"%s\" with signals pending from previous tests -- such tests must be last in sequence\n", test->name);
> + abort();
> + }
> +
> + /*
> + * Let's create a temporary stack for the new thread.
> + * We don't know the direction in which it grows, and clone(2) API
> + * doesn't abstract this for us, so we allocate twice what we need and
> + * point new thread to the middle of the temporary stack space.
> + */
> +
> + futex_init(&test_thread_futex);
> +
> + char test_thread_stack[2*1<<10];
> +
> + test->in_progress = 1;
> + r = clone(&test_thread_main, test_thread_stack + sizeof(test_thread_stack) / 2,
> + 0 /* no signal */ |
> + CLONE_FILES |
> + CLONE_FS |
> + CLONE_IO |
> + CLONE_SIGHAND |
> + CLONE_SYSVSEM |
> + CLONE_THREAD |
> + CLONE_VM |
> + CLONE_CHILD_SETTID |
> + CLONE_CHILD_CLEARTID,
> + test /* arg */,
> + NULL /* ptid */,
> + NULL /* tls */,
> + &test->tid /* ctid */);
> +
> + if (r < 0) {
> + pr_perror("Failed to clone() to create a busy-loop thread for test \"%s\"", test->name);
> + abort();
> + }
> +
> + /*
> + * Wait until the control returns to the test to be able to destroy the
> + * temporary stack.
> + */
> +
> + futex_wait_while(&test_thread_futex, 0);
> +}
> +
> +/* says whether the call needs to be hooked */
> +int test_multi_hooked(void)
> +{
> + if (getpid() != master_pid) {
> + return 0;
> + }
> +
> + if (!multitest_enabled) {
> + return 0;
> + }
> +
> + if (!threading_enabled == (sys_gettid() != master_tid)) {
> + return 0;
> + }
> +
> + return 1;
> +}
> +
> +/* returns from the test context */
> +void test_multi_return(enum test_phase phase)
> +{
> + int r;
> + struct test *test = test_current();
> +
> + sigset_t pending_signals;
> + sigemptyset(&pending_signals);
> + r = sigpending(&pending_signals);
> + if (r < 0) {
> + pr_perror("Failed to sigpending() to check for stray signals at return to launcher");
> + abort();
> + }
> +
> + if (!sigisemptyset(&pending_signals)) {
> + pr_err("Pending signals at exit from test on phase %s\n", test_phase_str[phase]);
> + for (int sig = 0; sig < SIGRTMAX; ++sig) {
> + if (sigismember(&pending_signals, sig)) {
> + pr_err("Pending signal: %d (%s)\n", sig, sys_siglist[sig] ?: "<unknown>");
> + }
> + }
> +
> + /*
> + * We shouldn't have let ourselves even launch any tests after
> + * returning from one of them with pending signals.
> + */
> + assert(!have_pending_signals);
> + have_pending_signals = 1;
> + }
> +
> + test->in_progress = 0;
> + test->phase = phase;
> +
> + if (!test->tid) {
> + r = swapcontext(&test->ctx, &return_ctx);
> + if (r < 0) {
> + pr_perror("Failed to swapcontext() to jump to launcher");
> + abort();
> + }
> + } else {
> + r = getcontext(&test->ctx);
> + if (r < 0) {
> + pr_perror("Failed to getcontext() to save the busy-loop thread context in tid %d", test->tid);
> + abort();
> + }
> +
> + if (!test->in_progress) {
> + sys_exit(0);
> + }
> + }
> +
> + futex_set_and_wake(&test_thread_futex, 1);
> +}
> +
> +/* ultimately returns from the test context, simulating exit() */
> +NORETURN void test_multi_exit(int status)
> +{
> + struct test *test = test_current();
> +
> + test->phase = PHASE_RETURNED;
> + test->retcode = status;
> +
> + if (!test->tid) {
> + setcontext(&return_ctx);
> +
> + pr_perror("Failed to setcontext() to exit from test");
> + abort();
> + } else {
> + sys_exit(0);
> + }
> +}
> +
> +void test_multi_report(enum test_result result)
> +{
> + assert(current_test != NULL);
> +
> + current_test->result = result;
> +}
> +
> +void test_multi_go(int cr_done)
> +{
> + struct test *test = test_current();
> +
> + /*
> + * If test_go() is called between test_daemon() and test_waitsig(),
> + * this is an indication of that the test wants to spin in a busy loop
> + * during C/R.
> + * Otherwise, this is uninteresting for us.
> + */
> +
> + if (test->phase < PHASE_TEST_DAEMON ||
> + test->phase >= PHASE_TEST_WAITSIG) {
> + return;
> + }
> +
> + if (test->tid) {
> + ++test->busy_loop_iters;
> +
> + if (cr_done) {
> + test_multi_return(PHASE_TEST_WAITSIG);
> + }
> + } else {
> + assert(cr_done != 0);
> + pr_err("test_go() called from non-threaded test before test_waitsig(): \"%s\"\n", current_test->name);
> + abort();
> + }
> +}
> +
> +void test_multi_waitsig(void)
> +{
> + struct test *test = test_current();
> +
> + if (test->phase >= PHASE_TEST_WAITSIG) {
> + return;
> + }
> +
> + if (test->tid) {
> + test_multi_return(PHASE_TEST_WAITSIG);
> + } else {
> + /* even if we don't return to the launcher here, we still need
> + * to remember that we're past test_waitsig() */
> + test->phase = PHASE_TEST_WAITSIG;
> + }
> +}
> +
> +static void usage(const char *argv0)
> +{
> + fprintf(stderr,
> + "Usage: %s [<global options...>] --test <path to test .so> [<test .so options...>] ...\n"
> + "Global options:\n",
> + argv0);
> +
> + for (struct long_opt *opt = global_opts_head; opt != NULL; opt = opt->next) {
> + print_opt(opt);
> + }
> +}
> +
> +NORETURN void exit(int status)
> +{
> + assert(real_exit != NULL);
> +
> + /* if we are outside of any test, or if this is a test's ancillary
> + * child process, exit right away */
> + if (!test_multi_hooked()) {
> + real_exit(status);
> + }
> +
> + /* otherwise treat this as a return from test's main() */
> + test_multi_exit(status);
> +}
> +
> +int main(int argc, char **argv)
> +{
> + if (argc < 2) {
> + pr_err("This program expects at least one argument.\n");
> + usage(argv[0]);
> + return 1;
> + }
> +
> + int r;
> +
> + /*
> + * Find the real exit() function.
> + */
> +
> + real_exit = dlsym(RTLD_NEXT, "exit");
> + assert(real_exit != NULL);
> +
> + /*
> + * Capture a base execution context for all tests before we modify the
> + * execution environment.
> + *
> + * Note that even at this point we may be running with signals blocked
> + * due to fork+exec in test_init() not resetting signal masks, so
> + * force-clear the signal mask in the base test execution context.
> + *
> + * I hope that no tests rely on masking some signals before test_init()
> + * and expecting that mask to survive fork+exec to the final child...
> + */
> +
> + ucontext_t test_context;
> + r = getcontext(&test_context);
> + if (r < 0) {
> + pr_perror("Failed to getcontext()");
> + return 1;
> + }
> + sigemptyset(&test_context.uc_sigmask);
> +
> + /*
> + * Process arguments.
> + * We split the global argv into several sub-arrays per each loaded test
> + * and use arguments before the first "--lib*" as globals.
> + */
> +
> + pr_debug("Reading arguments and initializing");
> +
> + real_argc = argc;
> + real_argv = argv;
> +
> + global_opts_head = opt_head;
> + opt_head = NULL;
> +
> + __cleanup(tests_free) struct test *test_head = NULL;
> + struct test *test_tail = NULL;
> +
> + /* allocate and load tests */
> + for (ssize_t a = real_argc - 1; a >= 0; --a) {
> + int is_threaded;
> +
> + if (!strcmp(argv[a], "--lib")) {
> + is_threaded = 0;
> + } else if (!strcmp(argv[a], "--lib-threaded")) {
> + is_threaded = 1;
> + } else {
> + continue;
> + }
> +
> + if (a >= real_argc - 1) {
> + pr_err("Test binary path expected after last '--lib'\n");
> + return 1;
> + }
> +
> + struct test *test = calloc(1, sizeof(struct test));
> + test->name = argv[a+1];
> + test->argv = &argv[a+1];
> + test->is_threaded = is_threaded;
> +
> + test->next = test_head;
> + test_head = test;
> + if (test_tail == NULL) {
> + test_tail = test;
> + }
> + }
> +
> + if (test_head == NULL) {
> + pr_err("No tests specified.\n");
> + usage(argv[0]);
> + return 1;
> + }
> +
> + all_tests = test_head;
> +
> + /* compute argument array boundaries */
> + argc = test_head->argv - argv - 1;
> + for (struct test *test = test_head; test->next != NULL; test = test->next) {
> + test->argc = test->next->argv - test->argv - 1;
> + }
> + test_tail->argc = argv + real_argc - test_tail->argv;
> +
> + /*
> + * Then load tests binaries one by one, saving option descriptor lists
> + * pertaining to each of them, and prepare execution contexts.
> + */
> +
> + pr_debug("Loading tests");
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + r = test_make(test, &test_context);
> + if (r < 0) {
> + return 1;
> + }
> + }
> +
> + /*
> + * Then finally run tests. Each of them will return control (using
> + * swapcontext()) after reaching key points in their execution --
> + * at test_init(), at test_daemon() and after return from main().
> + * After letting all tests go up to the same point, we actually
> + * perform the corresponding action and then continue all tests
> + * up to the next stage.
> + *
> + * Now we run the tests from start of main() to test_init().
> + * At entry to test_init() the tests take an alternative path which
> + * saves the context and passes the control back to the launcher.
> + */
> +
> + pr_debug("Running tests init phase");
> +
> + /* HACK: run parseargs() on global arguments before everything to
> + * propagate global arguments to children which are created before
> + * test_init() and run test_ext_init() themselves.
> + * (see zdtm/live/static/socket-ext.c for why this matters) */
> + opt_head = global_opts_head;
> + parseargs(argc, argv);
> + opt_head = NULL;
> +
> + /* Fetch pid and tid for the threading detection machinery to work */
> + master_pid = getpid();
> + master_tid = sys_gettid();
> +
> + enum test_phase test_init_kind;
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + pr_debug("Initializing test \"%s\"", test->name);
> + test_run(test);
> +
> + if (test->phase != PHASE_TEST_INIT &&
> + test->phase != PHASE_TEST_EXT_INIT) {
> + pr_err("Test \"%s\" returned with unexpected phase: %s\n", test->name, test_phase_str[test->phase]);
> + return 1;
> + }
> +
> + if (test == test_head) {
> + test_init_kind = test->phase;
> + } else if (test->phase != test_init_kind) {
> + pr_err("Test \"%s\" wants %s while previous tests want %s -- mixing different kinds not supported\n", test->name, test_phase_str[test->phase], test_phase_str[test_init_kind]);
> + return 1;
> + }
> +
> + if (have_pending_signals) {
> + pr_err("Pending signals in tests in init phase are not supported\n");
> + return 1;
> + }
> + }
> +
> + /*
> + * Then run the real test_init(), parsing "global" arguments (which
> + * appear on the command line before any test specifications) against
> + * "global" option descriptors (registered by libzdtmtst.so's ctors).
> + */
> +
> + switch(test_init_kind) {
> + case PHASE_TEST_INIT:
> + test_init(1, argv);
> + break;
> +
> + case PHASE_TEST_EXT_INIT:
> + test_ext_init(1, argv);
> + break;
> +
> + default:
> + pr_err("Switch error\n");
> + return 1;
> + }
> +
> + /* update tid because we fork in test_init() (pid is updated there) */
> + assert(master_pid == getpid());
> + master_tid = sys_gettid();
> +
> + /*
> + * After doing global init, finally block all signals except SIGTERM
> + * to avoid spoiling tests which call into test_daemon() with some signals pending.
> + */
> +
> + sigset_t all_but_sigterm;
> + sigfillset(&all_but_sigterm);
> + sigdelset(&all_but_sigterm, SIGTERM);
> + r = sigprocmask(SIG_BLOCK, &all_but_sigterm, NULL);
> + if (r < 0) {
> + pr_perror("Failed to sigprocmask(SIG_BLOCK) all signals but SIGTERM");
> + return 1;
> + }
> +
> + /*
> + * Run the tests from test_init() to test_daemon(). At this point,
> + * after restoring context inside test_init(), we parse test-specific
> + * arguments against test-specific option descriptors.
> + *
> + * Again, inside test_daemon() tests take a fake code path which passes
> + * the control back and immediately returns after regaining control,
> + * because the real daemonization happens here "once and for all".
> + */
> +
> + pr_debug("Running tests prepare phase");
> +
> + /*
> + * Right now we support only one test which returns with pending signals
> + * in prepare phase.
> + */
> + struct test *pending_signals_test = NULL;
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + pr_debug("Preparing test \"%s\"", test->name);
> + test_run(test);
> +
> + if (test->phase != PHASE_TEST_DAEMON) {
> + pr_err("Test \"%s\" returned with unexpected phase: %s\n", test->name, test_phase_str[test->phase]);
> + return 1;
> + }
> +
> + /* If we got some pending signals, remember this test as the
> + * "creator" of those. Note that we are not ready for more
> + * than one such test (and also for it being non-last). */
> + if (have_pending_signals) {
> + assert(pending_signals_test == NULL);
> + pending_signals_test = test;
> + }
> + }
> +
> + /*
> + * And here goes the most interesting part. Some tests want to run busy
> + * loops while the C/R is in progress. Obviously, if we want to run
> + * multiple such tests in one process and preserve their value, we will need
> + * to do some threading.
> + */
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + if (test->is_threaded) {
> + pr_debug("Running busy loop of test \"%s\"", test->name);
> +
> + multitest_enabled = 1;
> + threading_enabled = 1;
> + test_run_thread(test);
> + }
> + }
> +
> + /*
> + * Now really daemonize and wait for SIGTERM from the parent.
> + */
> +
> + pr_debug("Tests prepared, signaling parent for C/R");
> +
> + test_daemon();
> + test_waitsig();
> +
> + /*
> + * Wait for all threads to exit.
> + */
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + pr_debug("Waiting for the busy loop thread of test \"%s\"", test->name);
> +
> + if (test->is_threaded) {
> + futex_wait_until(&test->tid_futex, 0);
> +
> + if (test->phase != PHASE_TEST_WAITSIG) {
> + pr_err("Test \"%s\" exited busy-loop thread with unexpected phase: %s\n", test->name, test_phase_str[test->phase]);
> + }
> +
> + pr_debug("Busy loop thread of test \"%s\" made %d iterations", test->name, test->busy_loop_iters);
> + if (test->busy_loop_iters < 2) {
> + pr_err("Busy loop thread of test \"%s\" made less than one full iteration\n", test->name);
> + abort();
> + }
> + }
> + }
> +
> + multitest_enabled = 0;
> + threading_enabled = 0;
> +
> + /*
> + * Run the tests from test_daemon() to the end. Note that test_waitsig()
> + * is idempotent, so it will not wait for yet another SIGTERM.
> + */
> +
> + pr_debug("C/R done, continuing tests to verify phase");
> +
> + /* first, reverse the test list in-place */
> + for (struct test *test_tail_initial = test_tail; test_head != test_tail_initial; ) {
> + test_tail->next = test_head;
> + test_head = test_head->next;
> + test_tail->next->next = NULL;
> + }
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + pr_debug("Verifying test \"%s\"", test->name);
> +
> + /* If we (still) have pending signals, check that this is
> + * _the_ test which created those pending signals. */
> + if (have_pending_signals) {
> + assert(test == pending_signals_test);
> + have_pending_signals = 0;
> + }
> +
> + test_run(test);
> +
> + if (test->phase != PHASE_RETURNED) {
> + pr_err("Test \"%s\" returned with unexpected phase: %s\n", test->name, test_phase_str[test->phase]);
> + return 1;
> + }
> + }
> +
> + /*
> + * Done. Check results and do final fail() or pass().
> + */
> +
> + pr_debug("Wow, we survived!");
> +
> + for (struct test *test = test_head; test != NULL; test = test->next) {
> + if (test->retcode != 0) {
> + fail("Test \"%s\" returned %d", test->name, test->retcode);
> + return 0;
> + }
> +
> + if (test->result != RESULT_PASS) {
> + fail("Test \"%s\" reported %s",
> + test->name,
> + test->result == RESULT_FAIL ? "failure" : "neither success nor failure");
> + return 0;
> + }
> + }
> +
> + pass();
> + return 0;
> +}
> diff --git a/test/zdtm/lib/test.c b/test/zdtm/lib/test.c
> index fcff71c..06fedce 100644
> --- a/test/zdtm/lib/test.c
> +++ b/test/zdtm/lib/test.c
> @@ -17,6 +17,7 @@
> #include <grp.h>
>
> #include "zdtmtst.h"
> +#include "zdtmtst_internal.h"
> #include "lock.h"
> #include "ns.h"
>
> @@ -34,11 +35,19 @@ TEST_OPTION(pidfile, string, "file to store pid", 1);
>
> pid_t master_pid = 0;
>
> +int real_argc = 0;
> +char **real_argv = NULL;
> +
> int test_fork_id(int id)
> {
> return fork();
> }
>
> +__weak int test_multi_hooked(void)
> +{
> + return 0;
> +}
> +
> #define INPROGRESS ".inprogress"
> static void test_fini(void)
> {
> @@ -65,6 +74,7 @@ static void setup_outfile()
> fprintf(stderr, "Can't register exit function\n");
> exit(1);
> }
> +
> if (test_log_init(outfile, INPROGRESS))
> exit(1);
> }
> @@ -86,6 +96,15 @@ static void redir_stdfds()
>
> void test_ext_init(int argc, char **argv)
> {
> + if (test_multi_hooked()) {
> + test_multi_return(PHASE_TEST_EXT_INIT);
> +
> + /* we continue from here after letting all tests go up to this
> + * point and running test_ext_init() once and for all */
> + parseargs(argc, argv);
> + return;
> + }
> +
> parseargs(argc, argv);
> if (test_log_init(outfile, ".external"))
> exit(1);
> @@ -93,6 +112,15 @@ void test_ext_init(int argc, char **argv)
>
> void test_init(int argc, char **argv)
> {
> + if (test_multi_hooked()) {
> + test_multi_return(PHASE_TEST_INIT);
> +
> + /* we continue from here after letting all tests go up to this
> + * point and running test_init() once and for all */
> + parseargs(argc, argv);
> + return;
> + }
> +
> pid_t pid;
> static FILE *pidf;
> char *val;
> @@ -107,14 +135,15 @@ void test_init(int argc, char **argv)
> val = getenv("ZDTM_NEWNS");
> if (val) {
> if (!strcmp(val, "1")) {
> - ns_create(argc, argv);
> + ns_create(real_argc ?: argc, real_argv ?: argv);
> exit(1);
> }
>
> if (!strcmp(val, "2")) {
> test_log_init(outfile, "ns");
> redir_stdfds();
> - ns_init(argc, argv);
> + ns_init(real_argc ?: argc, real_argv ?: argv);
> +
> }
> }
>
> @@ -224,6 +253,13 @@ void test_init(int argc, char **argv)
>
> void test_daemon()
> {
> + if (test_multi_hooked()) {
> + test_multi_return(PHASE_TEST_DAEMON);
> +
> + /* we continue from here after C/R */
> + return;
> + }
> +
> pid_t ppid;
>
> ppid = getppid();
> @@ -243,14 +279,29 @@ out:
>
> int test_go(void)
> {
> - return !futex_get(&sig_received);
> + int cr_done = futex_get(&sig_received);
> + if (test_multi_hooked()) {
> + test_multi_go(cr_done);
> + }
> + return !cr_done;
> }
>
> void test_waitsig(void)
> {
> + /* in multi-test mode, tests invoke test_waitsig() after real C/R
> + * and waiting for signal in the launcher, but test_waitsig() is
> + * idempotent (i. e. once a signal has been received, it will never
> + * wait anymore), so it will work as is (which is very good due to
> + * the fact that test_waitsig() is (ab)used by tests' own children). */
> + if (test_multi_hooked()) {
> + test_multi_waitsig();
> + }
> futex_wait_while(&sig_received, 0);
> }
>
> void test_report(int ok)
> {
> + if (test_multi_hooked()) {
> + test_multi_report(ok ? RESULT_PASS : RESULT_FAIL);
> + }
> }
> diff --git a/test/zdtm/lib/zdtmtst.h b/test/zdtm/lib/zdtmtst.h
> index 1ccc2f2..6837c45 100644
> --- a/test/zdtm/lib/zdtmtst.h
> +++ b/test/zdtm/lib/zdtmtst.h
> @@ -39,7 +39,7 @@ extern int test_go(void);
> /* sleep until SIGTERM is delivered */
> extern void test_waitsig(void);
> /* report the test result (apart from logging PASS or FAIL) */
> -extern void test_report(int result);
> +extern void test_report(int ok);
>
> #include <stdint.h>
>
> @@ -119,12 +119,24 @@ extern int zdtm_seccomp;
> test_msg("ERR: %s:%d: " format " (errno = %d (%s))\n", \
> __FILE__, __LINE__, ## arg, errno, strerror(errno))
> #define fail(format, arg...) \
> - test_msg("FAIL: %s:%d: " format " (errno = %d (%s))\n", \
> - __FILE__, __LINE__, ## arg, errno, strerror(errno))
> + do { \
> + test_report(0); \
> + test_msg("FAIL: %s:%d: " format " (errno = %d (%s))\n", \
> + __FILE__, __LINE__, ## arg, errno, strerror(errno)); \
> + } while (0)
> #define skip(format, arg...) \
> - test_msg("SKIP: %s:%d: " format "\n", \
> - __FILE__, __LINE__, ## arg)
> -#define pass() test_msg("PASS\n")
> + do { \
> + test_report(1); \
> + test_msg("SKIP: %s:%d: " format "\n", \
> + __FILE__, __LINE__, ## arg); \
> + } while (0)
> +#define pass() \
> + do { \
> + test_report(1); \
> + test_msg("PASS\n"); \
> + } while (0)
> +
> +#define log(fmt, args...) fprintf(stderr, fmt "\n", ## args)
>
> #ifndef NDEBUG
> #define assert(expr) \
> diff --git a/test/zdtm/lib/zdtmtst_internal.h b/test/zdtm/lib/zdtmtst_internal.h
> index c0c90fc..355fca0 100644
> --- a/test/zdtm/lib/zdtmtst_internal.h
> +++ b/test/zdtm/lib/zdtmtst_internal.h
> @@ -9,4 +9,32 @@ extern struct long_opt *opt_head;
> /* our main pid, i. e. right after forking in test_init() */
> extern pid_t master_pid;
>
> +/* "real" (initial) argc and argv -- may be set by main() if it wants to
> + * pass partial parameters to test_init().
> + * "real" parameters will be used if test_init() decides to reexec. */
> +extern int real_argc;
> +extern char **real_argv;
> +
> +enum test_result {
> + RESULT_INDETERMINATE = 0,
> + RESULT_PASS,
> + RESULT_FAIL
> +};
> +
> +enum test_phase {
> + PHASE_START = 0,
> + PHASE_TEST_INIT,
> + PHASE_TEST_EXT_INIT,
> + PHASE_TEST_DAEMON,
> + PHASE_TEST_WAITSIG,
> + PHASE_RETURNED,
> +};
> +
> +__weak extern int test_multi_hooked(void);
> +__weak extern void test_multi_return(enum test_phase phase);
> +__weak extern void test_multi_report(enum test_result result);
> +__weak NORETURN extern void test_multi_exit(int status);
> +__weak extern void test_multi_go(int cr_done);
> +__weak extern void test_multi_waitsig(void);
> +
> #endif /* _ZDTMTST_INTERNAL_H_ */
> --
> 2.7.1
>
> _______________________________________________
> CRIU mailing list
> CRIU at openvz.org
> https://lists.openvz.org/mailman/listinfo/criu
More information about the CRIU
mailing list