[CRIU] [PATCH 1/2][v3] zdtm.py: add option --keep-going

[Pavel Emelyanov]

On 03/15/2016 11:39 AM, Sergey Bronnikov wrote:
> Introduce an option --keep-going to make ability to run all planned tests and
> ignore failed tests. Also with this option zdtm.py will keep image files and
> output log for each failed test:
> 
> [vagrant at fedora test]$ find sergeyb-xxx -maxdepth 3
> sergeyb-xxx
> sergeyb-xxx/zdtm_static_bridge_ns
> sergeyb-xxx/zdtm_static_bridge_ns/images
> sergeyb-xxx/zdtm_static_bridge_ns/images/1
> sergeyb-xxx/zdtm_static_bridge_ns/output
> sergeyb-xxx/zdtm_static_bridge_ns/images.0
> sergeyb-xxx/zdtm_static_bridge_ns/images.0/1
> sergeyb-xxx/zdtm_static_bridge_ns/output.0
> sergeyb-xxx/zdtm_static_deleted_unix_sock_h
> sergeyb-xxx/zdtm_static_deleted_unix_sock_h/output
> sergeyb-xxx/zdtm_static_deleted_unix_sock_h/output.0
> sergeyb-xxx/zdtm_static_deleted_unix_sock_h/output.1
> 
> Signed-off-by: Sergey Bronnikov <sergeyb at openvz.org>
> ---
>  test/zdtm.py | 49 ++++++++++++++++++++++++++++++++++++-------------
>  1 file changed, 36 insertions(+), 13 deletions(-)
> 
> diff --git a/test/zdtm.py b/test/zdtm.py
> index b523c62..8067e29 100755
> --- a/test/zdtm.py
> +++ b/test/zdtm.py
> @@ -76,6 +76,8 @@ def add_to_report(path, tgt_name):
>  		if os.path.isdir(path):
>  			shutil.copytree(path, tgt_path)
>  		else:
> +			if not os.path.exists(os.path.dirname(tgt_path)):
> +				os.mkdir(os.path.dirname(tgt_path))
>  			shutil.copy2(path, tgt_path)
>  
>  
> @@ -237,6 +239,21 @@ flavors = { 'h': host_flavor, 'ns': ns_flavor, 'uns': userns_flavor }
>  # Helpers
>  #
>  
> +def encode_flav(f):
> +	index = 0
> +	for key in flavors:
> +		if key == f:
> +			return (index + 128)
> +		index=+1

Can be 
	return flavors.keys().index(f)

> +
> +def decode_flav(i):
> +	index = 0
> +	for key in flavors:
> +		if index == (i - 128):
> +			return key
> +		index=+1
> +	return key

Can be
	return flavors.keys()[index - 128]

> +
>  def tail(path):
>  	p = subprocess.Popen(['tail', '-n1', path],
>  			stdout = subprocess.PIPE)
> @@ -973,12 +990,14 @@ def do_run_test(tname, tdesc, flavs, opts):
>  			print_sep("Test %s FAIL at %s" % (tname, e.step), '#')
>  			t.print_output()
>  			t.kill()
> -			add_to_report(cr_api.logs(), "cr_logs")
> +			if cr_api.logs():
> +				add_to_report(cr_api.logs(), tname.replace('/', '_') + "_" + f + "/images")
>  			if opts['keep_img'] == 'never':
>  				cr_api.cleanup()
> -			# This exit does two things -- exits from subprocess and
> -			# aborts the main script execution on the 1st error met
> -			sys.exit(1)
> +			# When option --ignore-fails not specified this exit

s/--ignore-fails/--keep-going/

> +			# does two things: exits from subprocess and aborts the
> +			# main script execution on the 1st error met
> +			sys.exit(encode_flav(f))
>  		else:
>  			if opts['keep_img'] != 'always':
>  				cr_api.cleanup()
> @@ -1020,7 +1039,7 @@ class launcher:
>  
>  		nd = ('nocr', 'norst', 'pre', 'iters', 'page_server', 'sibling', \
>  				'fault', 'keep_img', 'report', 'snaps', 'sat', \
> -				'dedup', 'sbs', 'freezecg', 'user', 'dry_run')
> +				'dedup', 'sbs', 'freezecg', 'user', 'keep_going', 'dry_run')

Opts here are only those needed in do_run_test which seem not to be the case for the keep-going one.

>  		arg = repr((name, desc, flavor, { d: self.__opts[d] for d in nd }))
>  
>  		if self.__use_log:
> @@ -1033,7 +1052,7 @@ class launcher:
>  		sub = subprocess.Popen(["./zdtm_ct", "zdtm.py"], \
>  				env = dict(os.environ, CR_CT_TEST_INFO = arg ), \
>  				stdout = log, stderr = subprocess.STDOUT)
> -		self.__subs[sub.pid] = { 'sub': sub, 'log': logf }
> +		self.__subs[sub.pid] = { 'sub': sub, 'log': logf, 'name': name }
>  
>  		if test_flag(desc, 'excl'):
>  			self.wait()
> @@ -1045,7 +1064,8 @@ class launcher:
>  			if status != 0:
>  				self.__fail = True

Here the self.__fail = True should only be set if opts['keep_going'] is False.
This would allow to ... (see below)

>  				if sub['log']:
> -					add_to_report(sub['log'], "output")
> +					failed_flavor = decode_flav(os.WEXITSTATUS(status))
> +					add_to_report(sub['log'], sub['name'].replace('/', '_') + "_" + failed_flavor + "/output")
>  
>  			if sub['log']:
>  				print open(sub['log']).read()
> @@ -1064,13 +1084,15 @@ class launcher:
>  		while self.__subs:
>  			if not self.__wait_one(os.WNOHANG):
>  				break
> -		if self.__fail:
> -			raise test_fail_exc('')
> +		if not opts['keep_going']:
> +			if self.__fail:
> +				raise test_fail_exc('')

... avoid this and ...

>  
>  	def wait_all(self):
>  		self.__wait_all()
> -		if self.__fail:
> -			raise test_fail_exc('')
> +		if not opts['keep_going']:
> +			if self.__fail:
> +				raise test_fail_exc('')

... this.

>  
>  	def finish(self):
>  		self.__wait_all()
> @@ -1387,8 +1409,8 @@ if os.environ.has_key('CR_CT_TEST_INFO'):
>  		while True:
>  			wpid, status = os.wait()
>  			if wpid == pid:
> -				if not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
> -					status = 1
> +				if os.WIFEXITED(status):
> +					status = os.WEXITSTATUS(status)

else:
	status = <some value meaning that test just failed as a whole>

>  				break;
>  
>  	sys.exit(status)
> @@ -1427,6 +1449,7 @@ rp.add_argument("--dry-run", help="Don't run tests, just pretend to", action='st
>  rp.add_argument("-k", "--keep-img", help = "Whether or not to keep images after test",
>  		choices = [ 'always', 'never', 'failed' ], default = 'failed')
>  rp.add_argument("--report", help = "Generate summary report in directory")
> +rp.add_argument("--keep-going", help = "Keep running tests in spite of failures", action = 'store_true')
>  
>  lp = sp.add_parser("list", help = "List tests")
>  lp.set_defaults(action = list_tests)
> 

-- Pavel