[CRIU] [RFC PATCH 02/21] zdtm: zdtm.py: add --keep-going and --log-failures options

[Andrew Vagin]

On Fri, Feb 26, 2016 at 01:59:09AM +0300, Ivan Shapovalov wrote:
> On 2016-02-25 at 12:09 -0800, Andrew Vagin wrote:
> > On Fri, Feb 19, 2016 at 05:50:25PM +0300, Ivan Shapovalov wrote:
> > > 
> > > --keep-going is similar to `make -k`: it makes zdtm.py continue
> > > executing tests
> > > in spite of errors (as compared to exit(1) on first failure).
> > > 
> > > --log-failures accepts a filename prefix and makes zdtm.py log each
> > > failed
> > > test name to the file "<prefix>.<flavors>" (where <prefix> is
> > > passed as
> > > argument value, and <flavors> is the list of failed flavors or
> > > "all").
> > I think you need to split this patch on two and introduce these
> > options
> > in separate patches
> 
> OK.
> 
> > 
> > > 
> > > 
> > > If --keep-going is not passed, but --log-failures is, then only the
> > > first
> > > failed flavor is logged.
> > What is about --report? Will an report contain all errors?
> 
> If --keep-going is not passed, current behavior is preserved -- that
> is, if I'm reading the code correctly, it will only have first error
> (then the testing will abort)...

If --keep-going is set, an report should contain information about all
errors.

And an output of zdtm.py should contain a list of failed tests at the end.
> 
> --
> Ivan Shapovalov / intelfx /
> 
> > > 
> > > 
> > > Signed-off-by: Ivan Shapovalov <intelfx at intelfx.name>
> > > ---
> > >  test/zdtm.py | 31 +++++++++++++++++++++++++++----
> > >  1 file changed, 27 insertions(+), 4 deletions(-)
> > > 
> > > diff --git a/test/zdtm.py b/test/zdtm.py
> > > index 79b7a95..3caf227 100755
> > > --- a/test/zdtm.py
> > > +++ b/test/zdtm.py
> > > @@ -968,6 +968,8 @@ def do_run_test(tname, tdesc, flavs, opts):
> > >  
> > >  	fcg = get_freezer(opts['freezecg'])
> > >  
> > > +	failed_flavs = []
> > > +
> > >  	for f in flavs:
> > >  		print
> > >  		print_sep("Run %s in %s" % (tname, f))
> > > @@ -995,14 +997,32 @@ def do_run_test(tname, tdesc, flavs, opts):
> > >  			add_to_report(cr_api.logs(), "cr_logs")
> > >  			if opts['keep_img'] == 'never':
> > >  				cr_api.cleanup()
> > > -			# This exit does two things -- exits from
> > > subprocess and
> > > -			# aborts the main script execution on the
> > > 1st error met
> > > -			sys.exit(1)
> > > +			failed_flavs.append(f)
> > > +			if not opts['keep_going']:
> > > +				break
> > >  		else:
> > >  			if opts['keep_img'] != 'always':
> > >  				cr_api.cleanup()
> > >  			print_sep("Test %s PASS" % tname)
> > >  
> > > +	if failed_flavs:
> > > +		if cmp(failed_flavs, flavs):
> > > +			log_failures_suffix = "all"
> > > +		else:
> > > +			log_failures_suffix =
> > > ",".join(failed_flavs)
> > > +
> > > +		try:
> > > +			failures = open("%s.%s" %
> > > (opts["log_failures"], log_failures_suffix), "a")
> > > +			fcntl.flock(failures, fcntl.LOCK_EX)
> > > +			print >>failures, tname
> > > +		except e:
> > > +			print "Cannot write to failed tests log:
> > > %s" % str(e)
> > zdtm.sh must exit in this case.
> > 
> > > 
> > > +
> > > +		# This exit does two things -- exits from
> > > subprocess and
> > > +		# aborts the main script execution on the 1st
> > > error met
> > > +		if not opts['keep_going']:
> > > +			sys.exit(1)
> > > +
> > >  class launcher:
> > >  	def __init__(self, opts, nr_tests):
> > >  		self.__opts = opts
> > > @@ -1039,7 +1059,7 @@ class launcher:
> > >  
> > >  		nd = ('nocr', 'norst', 'pre', 'iters',
> > > 'page_server', 'sibling', \
> > >  				'fault', 'keep_img', 'report',
> > > 'snaps', 'sat', \
> > > -				'dedup', 'sbs', 'freezecg',
> > > 'user')
> > > +				'dedup', 'sbs', 'freezecg',
> > > 'user', "keep_going", "log_failures")
> > >  		arg = repr((name, desc, flavor, { d:
> > > self.__opts[d] for d in nd }))
> > >  
> > >  		if self.__use_log:
> > > @@ -1418,6 +1438,9 @@ rp.add_argument("--sbs", help = "Do step-by-
> > > step execution, asking user for keyp
> > >  rp.add_argument("--freezecg", help = "Use freeze cgroup
> > > (path:state)")
> > >  rp.add_argument("--user", help = "Run CRIU as regular user",
> > > action = 'store_true')
> > >  
> > > +rp.add_argument("--keep-going", help = "Keep running tests in
> > > spite of failures", action = 'store_true')
> > > +rp.add_argument("--log-failures", help = "Log each failed test to
> > > file \"<argument>.<flavors>\"")
> > > +
> > >  rp.add_argument("--page-server", help = "Use page server dump",
> > > action = 'store_true')
> > >  rp.add_argument("-p", "--parallel", help = "Run test in parallel")
> > >