[CRIU] [PATCH 2/2] zdtm.py: add support of test reports in TAP format

[Pavel Emelyanov]

On 03/03/2016 06:19 PM, Sergey Bronnikov wrote:
> Usually we run CRIU tests automatically using Jenkins CI and it reports status
> as PASS/FAIL for overall testsuite on used environment. You should dig into log
> files to figure out how many tests were failed and skipped. This patch brings
> support of cute reports in TAP (Test Anything Protocol) format.
> 
> The sample of report:
> 
> TAP version 13
> 1..242
> ok 1 - conntracks # SKIP manual run only
> ok 2 - busyloop00
> ok 3 - sleeping00
> ok 4 - pid00
> ok 5 - caps00
> ok 6 - wait00
> 
> Note: report will be generated only with option --ignore-fails, otherwise
> we will get incomplete report.
> 
> Signed-off-by: Sergey Bronnikov <sergeyb at openvz.org>
> ---
>  test/zdtm.py | 29 ++++++++++++++++++++++++++++-
>  1 file changed, 28 insertions(+), 1 deletion(-)
> 
> diff --git a/test/zdtm.py b/test/zdtm.py
> index 75e4514..495b57f 100755
> --- a/test/zdtm.py
> +++ b/test/zdtm.py
> @@ -18,6 +18,7 @@ import imp
>  import socket
>  import fcntl
>  import errno
> +import datetime
>  
>  os.chdir(os.path.dirname(os.path.abspath(__file__)))
>  
> @@ -961,6 +962,7 @@ class launcher:
>  		self.__opts = opts
>  		self.__total = nr_tests
>  		self.__nr = 0
> +		self.__runtest = 0
>  		self.__max = int(opts['parallel'] or 1)
>  		self.__subs = {}
>  		self.__fail = False
> @@ -971,6 +973,21 @@ class launcher:
>  		else:
>  			self.__use_log = False
>  
> +		if opts['ignore_fails']:

Don't hide TAP report under --ignore-fails please. I'd suggest either introduce
separate option for that or generate tap when --report AND --ignore-fails are set.

> +			now = datetime.datetime.now()
> +			if report_dir:
> +				reportname = report_dir + "/" + "criu-testreport-" + now.strftime("%Y-%m-%d-%H-%M-%S") + ".tap"
> +			else:
> +				reportname = "criu-testreport-" + now.strftime("%Y-%m-%d-%H-%M-%S") + ".tap"
> +
> +			global f
> +			f = open(reportname, 'a')
> +			print >> f, "# Hardware architecture: " + arch
> +			print >> f, "# Timestamp: " + now.strftime("%Y-%m-%d %H:%M") + " (GMT+1)"
> +			print >> f, "# "
> +			print >> f, "TAP version 13"
> +			print >> f, "1.." + str(nr_tests)
> +
>  	def __show_progress(self):
>  		perc = self.__nr * 16 / self.__total
>  		print "=== Run %d/%d %s" % (self.__nr, self.__total, '=' * perc + '-' * (16 - perc))
> @@ -978,6 +995,9 @@ class launcher:
>  	def skip(self, name, reason):
>  		print "Skipping %s (%s)" % (name, reason)
>  		self.__nr += 1
> +		self.__runtest += 1
> +		if opts['ignore_fails']:
> +			print >> f, "ok %d - %s # SKIP %s" % (self.__runtest, name.split('/')[-1:][0], reason)
>  
>  	def run_test(self, name, desc, flavor):
>  
> @@ -1005,19 +1025,25 @@ class launcher:
>  		sub = subprocess.Popen(["./zdtm_ct", "zdtm.py"], \
>  				env = dict(os.environ, CR_CT_TEST_INFO = arg ), \
>  				stdout = log, stderr = subprocess.STDOUT)
> -		self.__subs[sub.pid] = { 'sub': sub, 'log': logf }
> +		self.__subs[sub.pid] = { 'sub': sub, 'log': logf, 'name': name }
>  
>  		if test_flag(desc, 'excl'):
>  			self.wait()
>  
>  	def __wait_one(self, flags):
>  		pid, status = os.waitpid(0, flags)
> +		self.__runtest += 1
>  		if pid != 0:
>  			sub = self.__subs.pop(pid)
>  			if status != 0:
> +				if opts['ignore_fails']:
> +					print >> f, "not ok %d - %s" % (self.__runtest, sub['name'].split('/')[-1:][0])
>  				self.__fail = True
>  				if sub['log']:
>  					add_to_report(sub['log'], "output")
> +			else:
> +				if opts['ignore_fails']:
> +					print >> f, "ok %d - %s" % (self.__runtest, sub['name'].split('/')[-1:][0])
>  
>  			if sub['log']:
>  				print open(sub['log']).read()
> @@ -1046,6 +1072,7 @@ class launcher:
>  
>  	def finish(self):
>  		self.__wait_all()
> +		f.close()
>  		if self.__fail:
>  			print_sep("FAIL", "#")
>  			sys.exit(1)
>