[CRIU] [RFC PATCH 02/21] zdtm: zdtm.py: add --keep-going and --log-failures options

[Ivan Shapovalov]

--keep-going is similar to `make -k`: it makes zdtm.py continue executing tests
in spite of errors (as compared to exit(1) on first failure).

--log-failures accepts a filename prefix and makes zdtm.py log each failed
test name to the file "<prefix>.<flavors>" (where <prefix> is passed as
argument value, and <flavors> is the list of failed flavors or "all").

If --keep-going is not passed, but --log-failures is, then only the first
failed flavor is logged.

Signed-off-by: Ivan Shapovalov <intelfx at intelfx.name>
---
 test/zdtm.py | 31 +++++++++++++++++++++++++++----
 1 file changed, 27 insertions(+), 4 deletions(-)

diff --git a/test/zdtm.py b/test/zdtm.py
index 79b7a95..3caf227 100755
--- a/test/zdtm.py
+++ b/test/zdtm.py
@@ -968,6 +968,8 @@ def do_run_test(tname, tdesc, flavs, opts):
 
 	fcg = get_freezer(opts['freezecg'])
 
+	failed_flavs = []
+
 	for f in flavs:
 		print
 		print_sep("Run %s in %s" % (tname, f))
@@ -995,14 +997,32 @@ def do_run_test(tname, tdesc, flavs, opts):
 			add_to_report(cr_api.logs(), "cr_logs")
 			if opts['keep_img'] == 'never':
 				cr_api.cleanup()
-			# This exit does two things -- exits from subprocess and
-			# aborts the main script execution on the 1st error met
-			sys.exit(1)
+			failed_flavs.append(f)
+			if not opts['keep_going']:
+				break
 		else:
 			if opts['keep_img'] != 'always':
 				cr_api.cleanup()
 			print_sep("Test %s PASS" % tname)
 
+	if failed_flavs:
+		if cmp(failed_flavs, flavs):
+			log_failures_suffix = "all"
+		else:
+			log_failures_suffix = ",".join(failed_flavs)
+
+		try:
+			failures = open("%s.%s" % (opts["log_failures"], log_failures_suffix), "a")
+			fcntl.flock(failures, fcntl.LOCK_EX)
+			print >>failures, tname
+		except e:
+			print "Cannot write to failed tests log: %s" % str(e)
+
+		# This exit does two things -- exits from subprocess and
+		# aborts the main script execution on the 1st error met
+		if not opts['keep_going']:
+			sys.exit(1)
+
 class launcher:
 	def __init__(self, opts, nr_tests):
 		self.__opts = opts
@@ -1039,7 +1059,7 @@ class launcher:
 
 		nd = ('nocr', 'norst', 'pre', 'iters', 'page_server', 'sibling', \
 				'fault', 'keep_img', 'report', 'snaps', 'sat', \
-				'dedup', 'sbs', 'freezecg', 'user')
+				'dedup', 'sbs', 'freezecg', 'user', "keep_going", "log_failures")
 		arg = repr((name, desc, flavor, { d: self.__opts[d] for d in nd }))
 
 		if self.__use_log:
@@ -1418,6 +1438,9 @@ rp.add_argument("--sbs", help = "Do step-by-step execution, asking user for keyp
 rp.add_argument("--freezecg", help = "Use freeze cgroup (path:state)")
 rp.add_argument("--user", help = "Run CRIU as regular user", action = 'store_true')
 
+rp.add_argument("--keep-going", help = "Keep running tests in spite of failures", action = 'store_true')
+rp.add_argument("--log-failures", help = "Log each failed test to file \"<argument>.<flavors>\"")
+
 rp.add_argument("--page-server", help = "Use page server dump", action = 'store_true')
 rp.add_argument("-p", "--parallel", help = "Run test in parallel")
 
-- 
2.7.1