[Devel] [PATCH vz10 2/2] selftests: ve: introduce vtty pair teardown stress test

[Vasileios Almpanis]

> A vtty pair is created by the first master open and torn down by the
> last close. The close racing the peer's close used to let both sides
> decide they were the final one and call release_tty() twice, and a
> master opening again while a teardown was in flight used to hand out a
> reference to a tty_struct about to be freed.
> 
> Hammer both ends. The master comes from the vzctl ioctl, which does
> not look the VE up. The slave is only reachable from inside the VE, as
> TTY_MAJOR minor 1..MAX_NR_VTTY_CONSOLES, which vtty_driver() maps to
> vttys index minor - 1, so the test builds a VE with cgroup2, sets
> ve.veid to the cgroup it made, and runs the slave side there.
> 
> One of the symtoms of a buggy kernel is dmesg warning, not just an
> eventual crash, so the test checks that. A crashed node should also
> report test failure in CI, but that's less verbose, hence the check.
> 
> https://virtuozzo.atlassian.net/browse/VSTOR-144771
> 
> Feature: ve selftest
> Signed-off-by: Eva Kurchatova <eva.kurchatova at virtuozzo.com>
>
> diff --git a/tools/testing/selftests/ve/Makefile b/tools/testing/selftests/ve/Makefile
> index ec40cbc7b3a1..6cd5823c4d93 100644
> --- a/tools/testing/selftests/ve/Makefile
> +++ b/tools/testing/selftests/ve/Makefile
> @@ -4,5 +4,6 @@ CFLAGS += -g -Wall -O2
>  
>  TEST_GEN_PROGS += ve_ns_owner_test
>  TEST_GEN_PROGS += ve_perms_test
> +TEST_GEN_PROGS += ve_vtty_stress

Please add to .gitignore too

>  
>  include ../lib.mk
> diff --git a/tools/testing/selftests/ve/settings b/tools/testing/selftests/ve/settings
> new file mode 100644
> index 000000000000..e2206265f67c
> --- /dev/null
> +++ b/tools/testing/selftests/ve/settings
> @@ -0,0 +1 @@
> +timeout=900
> diff --git a/tools/testing/selftests/ve/ve_vtty_stress.c b/tools/testing/selftests/ve/ve_vtty_stress.c
> new file mode 100644
> index 000000000000..9037f731c58d
> --- /dev/null
> +++ b/tools/testing/selftests/ve/ve_vtty_stress.c
> @@ -0,0 +1,244 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * ve_vtty_stress - close a vtty pair from both ends at once.
> + *
> + * A vtty pair is created by the first master open and torn down by the
> + * last close.  A close racing the peer's close used to let both sides
> + * decide they were the final one and call release_tty() twice, and a
> + * master opened again while a teardown was still in flight used to hand
> + * out a reference to a tty_struct that was about to be freed.  Either
> + * way the next close walked freed memory.
> + *
> + * The master comes from the vzctl ioctl, which does not look the VE up.
> + * The slave is only reachable from inside the VE, as TTY_MAJOR minor
> + * 1..MAX_NR_VTTY_CONSOLES, which vtty_driver() maps to vttys index
> + * minor - 1.  So the test makes a VE with cgroup2, hammers the master
> + * from the host and the slave from a task inside it, and fails if the
> + * kernel logs a complaint about the teardown.
> + *
> + * Surviving is not a pass on its own: the first symptom is a warning,
> + * which stops nothing by itself.
> + *
> + * Usage: ve_vtty_stress [seconds]   (default 600, or $VE_VTTY_SECS)
> + */
> +#define _GNU_SOURCE
> +#include <sched.h>
> +#include <stdbool.h>
> +#include <sys/ioctl.h>
> +#include <linux/major.h>
> +#include <sys/sysmacros.h>
> +#include <sys/wait.h>
> +#include <time.h>
> +
> +#include "ve_selftest.h"
> +#include "../kselftest.h"
> +
> +/* The ioctl argument; the trailing data[] the uapi struct has is unused. */
> +struct vzctl_ve_configure {
> +	unsigned int veid, key, val, size;
> +};
> +
> +#define VZCTL_VE_CONFIGURE	_IOW('.', 15, struct vzctl_ve_configure)
> +#define VE_CONFIGURE_OPEN_TTY	5
> +
> +#define VTTY_IDX	0		/* the pair both ends work on */
> +#define SLAVE_MINOR	(VTTY_IDX + 1)	/* vtty_driver(): index = minor - 1 */
> +#define DEFAULT_SECS	600
> +
> +static int open_master(int ctl_fd, unsigned int veid)
> +{
> +	struct vzctl_ve_configure c = {
> +		.veid = veid, .key = VE_CONFIGURE_OPEN_TTY, .val = VTTY_IDX,
> +	};
> +
> +	return ioctl(ctl_fd, VZCTL_VE_CONFIGURE, &c);
> +}
> +
> +/*
> + * The slave end, inside the VE.  Every error here is one the race is
> + * expected to produce: the pair is gone once the master has closed it.
> + */
> +static int hammer_slave(time_t deadline, unsigned long *opened)
> +{
> +	char node[] = "/tmp/ve_vtty_slave";
> +
> +	unlink(node);
> +	if (mknod(node, S_IFCHR | 0600, makedev(TTY_MAJOR, SLAVE_MINOR)) < 0)
> +		return -1;
> +
> +	while (time(NULL) < deadline) {
> +		int fd = open(node, O_RDWR | O_NOCTTY | O_NONBLOCK);
> +
> +		if (fd >= 0) {
> +			(*opened)++;
> +			close(fd);
> +		}
> +	}
> +	unlink(node);
> +	return 0;
> +}
> +
> +/* The kernel log, since a warning fails nothing by itself. */
> +static int kmsg_open(void)
> +{
> +	int fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK);
> +
> +	if (fd >= 0)
> +		lseek(fd, 0, SEEK_END);
> +	return fd;
> +}
> +
> +static int kmsg_check(int fd)
> +{
> +	static const char * const ours[] = {
> +		"tty_release", "release_tty", "check_tty_count",
> +		"vtty_open_master", "vtty_shutdown",
> +	};
> +	char buf[8192];
> +	int found = 0;
> +	ssize_t r;
> +	size_t i;
> +
> +	while (fd >= 0 && (r = read(fd, buf, sizeof(buf) - 1)) > 0) {
> +		buf[r] = '\0';
> +		if (!strstr(buf, "WARNING") && !strstr(buf, "BUG") &&
> +		    !strstr(buf, "Oops") && !strstr(buf, "general protection"))
> +			continue;
> +		for (i = 0; i < ARRAY_SIZE(ours); i++)
> +			if (strstr(buf, ours[i])) {
> +				ksft_print_msg("kernel log: %s", buf);
> +				found++;
> +				break;
> +			}
> +	}
> +	return found;
> +}
> +
> +int main(int argc, char **argv)
> +{
> +	unsigned long master_opened = 0, slave_opened = 0;
> +	int cg, ctid, ctl, kmsg, ready[2], count[2], status;
> +	const char *env = getenv("VE_VTTY_SECS");
> +	char path[PATH_MAX], val[16], c;
> +	time_t deadline;
> +	long secs;
> +	pid_t pid;
> +
> +	ksft_print_header();
> +	secs = argc > 1 ? atol(argv[1]) : (env ? atol(env) : DEFAULT_SECS);
> +	if (secs <= 0)
> +		secs = DEFAULT_SECS;
> +
> +	ctl = open("/dev/vzctl", O_RDWR);
> +	if (ctl < 0)
> +		ksft_exit_skip("cannot open /dev/vzctl: %s, the vz modules "
> +			       "(vzdev, vzmon) are needed\n", strerror(errno));
> +
> +	cg = mount_cg2_fd();
> +	if (cg < 0)
> +		ksft_exit_skip("cannot mount cgroup2: %s\n", strerror(errno));
> +	write_file_at(cg, "cgroup.subtree_control", VE_CONTROLLERS);
> +
> +	ctid = make_ve(cg, CTID_MIN);
> +	if (ctid < 0)
> +		ksft_exit_skip("cannot make a VE cgroup\n");
> +
> +	/*
> +	 * The pair is filed under the veid of the VE, so the two ends only
> +	 * meet if the cgroup carries the one the master is opened for.
> +	 */
> +	snprintf(path, sizeof(path), "%d/ve.veid", ctid);
> +	snprintf(val, sizeof(val), "%d", ctid);
> +	if (write_file_at(cg, path, val) < 0) {
> +		destroy_ve(cg, ctid);
> +		ksft_exit_skip("cannot set ve.veid: %s\n", strerror(errno));
> +	}
> +
> +	/* Do not report a failure where the vtty pair cannot be had at all. */
> +	status = open_master(ctl, ctid);
> +	if (status < 0 && errno != EBUSY) {
> +		destroy_ve(cg, ctid);
> +		ksft_exit_skip("no vtty master: %s\n", strerror(errno));
> +	}
> +	if (status >= 0)
> +		close(status);
> +
> +	ksft_set_plan(1);
> +	kmsg = kmsg_open();
> +	if (kmsg < 0)
> +		ksft_print_msg("cannot read /dev/kmsg: %s, only a crash will "
> +			       "be noticed\n", strerror(errno));
> +
> +	if (pipe(ready) || pipe(count))
> +		ksft_exit_fail_msg("pipe: %s\n", strerror(errno));
> +
> +	deadline = time(NULL) + secs;
> +	ksft_print_msg("VE %d, pair %d, %ld seconds\n", ctid, VTTY_IDX, secs);
> +
> +	pid = fork();

Fork could also return error. Please handle it.

-- 
Vasileios Almpanis <vasileios.almpanis at virtuozzo.com>