[CRIU] [RFC PATCH 02/21] zdtm: zdtm.py: add --keep-going and --log-failures options

[Andrew Vagin]

On Fri, Feb 19, 2016 at 05:50:25PM +0300, Ivan Shapovalov wrote:
> --keep-going is similar to `make -k`: it makes zdtm.py continue executing tests
> in spite of errors (as compared to exit(1) on first failure).
> 
> --log-failures accepts a filename prefix and makes zdtm.py log each failed
> test name to the file "<prefix>.<flavors>" (where <prefix> is passed as
> argument value, and <flavors> is the list of failed flavors or "all").

I think you need to split this patch on two and introduce these options
in separate patches

> 
> If --keep-going is not passed, but --log-failures is, then only the first
> failed flavor is logged.

What is about --report? Will an report contain all errors?
> 
> Signed-off-by: Ivan Shapovalov <intelfx at intelfx.name>
> ---
>  test/zdtm.py | 31 +++++++++++++++++++++++++++----
>  1 file changed, 27 insertions(+), 4 deletions(-)
> 
> diff --git a/test/zdtm.py b/test/zdtm.py
> index 79b7a95..3caf227 100755
> --- a/test/zdtm.py
> +++ b/test/zdtm.py
> @@ -968,6 +968,8 @@ def do_run_test(tname, tdesc, flavs, opts):
>  
>  	fcg = get_freezer(opts['freezecg'])
>  
> +	failed_flavs = []
> +
>  	for f in flavs:
>  		print
>  		print_sep("Run %s in %s" % (tname, f))
> @@ -995,14 +997,32 @@ def do_run_test(tname, tdesc, flavs, opts):
>  			add_to_report(cr_api.logs(), "cr_logs")
>  			if opts['keep_img'] == 'never':
>  				cr_api.cleanup()
> -			# This exit does two things -- exits from subprocess and
> -			# aborts the main script execution on the 1st error met
> -			sys.exit(1)
> +			failed_flavs.append(f)
> +			if not opts['keep_going']:
> +				break
>  		else:
>  			if opts['keep_img'] != 'always':
>  				cr_api.cleanup()
>  			print_sep("Test %s PASS" % tname)
>  
> +	if failed_flavs:
> +		if cmp(failed_flavs, flavs):
> +			log_failures_suffix = "all"
> +		else:
> +			log_failures_suffix = ",".join(failed_flavs)
> +
> +		try:
> +			failures = open("%s.%s" % (opts["log_failures"], log_failures_suffix), "a")
> +			fcntl.flock(failures, fcntl.LOCK_EX)
> +			print >>failures, tname
> +		except e:
> +			print "Cannot write to failed tests log: %s" % str(e)

zdtm.sh must exit in this case.

> +
> +		# This exit does two things -- exits from subprocess and
> +		# aborts the main script execution on the 1st error met
> +		if not opts['keep_going']:
> +			sys.exit(1)
> +
>  class launcher:
>  	def __init__(self, opts, nr_tests):
>  		self.__opts = opts
> @@ -1039,7 +1059,7 @@ class launcher:
>  
>  		nd = ('nocr', 'norst', 'pre', 'iters', 'page_server', 'sibling', \
>  				'fault', 'keep_img', 'report', 'snaps', 'sat', \
> -				'dedup', 'sbs', 'freezecg', 'user')
> +				'dedup', 'sbs', 'freezecg', 'user', "keep_going", "log_failures")
>  		arg = repr((name, desc, flavor, { d: self.__opts[d] for d in nd }))
>  
>  		if self.__use_log:
> @@ -1418,6 +1438,9 @@ rp.add_argument("--sbs", help = "Do step-by-step execution, asking user for keyp
>  rp.add_argument("--freezecg", help = "Use freeze cgroup (path:state)")
>  rp.add_argument("--user", help = "Run CRIU as regular user", action = 'store_true')
>  
> +rp.add_argument("--keep-going", help = "Keep running tests in spite of failures", action = 'store_true')
> +rp.add_argument("--log-failures", help = "Log each failed test to file \"<argument>.<flavors>\"")
> +
>  rp.add_argument("--page-server", help = "Use page server dump", action = 'store_true')
>  rp.add_argument("-p", "--parallel", help = "Run test in parallel")
>  
> -- 
> 2.7.1
> 
> _______________________________________________
> CRIU mailing list
> CRIU at openvz.org
> https://lists.openvz.org/mailman/listinfo/criu