re-implement compare-results.sh as compare-results.py

The compare-results.sh is annoyingly slow. Since our ttcn3 tests containers
support Python 2, re-implement in Python for much quicker evaluation.

Change-Id: I0747c9d66ffc7e4121497a2416fca78d7b56c8e6

3 files changed, 138 insertions, 214 deletions

diff --git a/compare-results.py b/compare-results.py
new file mode 100755
index 00000000..d1adb20c
--- /dev/null
+++ b/compare-results.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+# Copyright 2018 sysmocom - s.f.m.c. GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import re
+
+doc = "Compare TTCN3 test run results with expected results by junit logs."
+
+# The nicest would be to use an XML library, but I don't want to introduce dependencies on the build slaves.
+re_testcase = re.compile(r'''<testcase classname=['"]([^'"]+)['"].* name=['"]([^'"]+)['"].*>''')
+re_testcase_end = re.compile(r'''(</testcase>|<testcase [^>]*/>)''')
+re_failure = re.compile(r'''(<failure\b|<error\b)''')
+
+RESULT_PASS = 'pass'
+RESULT_FAIL = 'pass->FAIL'
+RESULT_SKIP = 'skip'
+RESULT_XFAIL = 'xfail'
+RESULT_FIXED = 'xfail->PASS'
+RESULT_NEW_PASS = 'NEW: PASS'
+RESULT_NEW_FAIL = 'NEW: FAIL'
+
+RESULTS = (
+	RESULT_FAIL,
+	RESULT_NEW_FAIL,
+	RESULT_XFAIL,
+	RESULT_FIXED,
+	RESULT_PASS,
+	RESULT_NEW_PASS,
+	RESULT_SKIP,
+	)
+
+def count(counter, name, result):
+	v = counter.get(result) or 0
+	v += 1
+	counter[result] = v
+	if result != RESULT_SKIP:
+		print('%s %s' % (result, name))
+
+def compare_one(name, expect, result, counter):
+	if result is None:
+		count(counter, name, RESULT_SKIP)
+	elif result == RESULT_PASS:
+		if expect == RESULT_PASS:
+			count(counter, name, RESULT_PASS)
+		elif expect == RESULT_FAIL:
+			count(counter, name, RESULT_FIXED)
+		elif expect is None:
+			count(counter, name, RESULT_NEW_PASS)
+	elif result == RESULT_FAIL:
+		if expect == RESULT_PASS:
+			count(counter, name, RESULT_FAIL)
+		elif expect == RESULT_FAIL:
+			count(counter, name, RESULT_XFAIL)
+		elif expect is None:
+			count(counter, name, RESULT_NEW_FAIL)
+
+def compare(cmdline, f_expected, f_current):
+	expected_list = parse_results(f_expected)
+	current_list = parse_results(f_current)
+
+	expected_dict = dict(expected_list)
+	current_dict = dict(current_list)
+
+	counter = {}
+
+	for expected_name, expected_result in expected_list:
+		compare_one(expected_name, expected_result, current_dict.get(expected_name), counter)
+
+	# Also count new tests
+	for current_name, current_result in current_list:
+		if current_name in expected_dict:
+			continue
+		compare_one(current_name, None, current_result, counter)
+
+
+	print('\nSummary:')
+	for r in RESULTS:
+		v = counter.get(r)
+		if not v:
+			continue
+		print('  %s: %d' % (r, v))
+	print('\n')
+
+def parse_results(f):
+	tests = []
+	name = None
+	result = None
+	for line in f:
+		m = re_testcase.search(line)
+		if m:
+			class_name, test_name = m.groups()
+			name = '%s.%s' % (class_name, test_name)
+
+		m = re_failure.search(line)
+		if m:
+			result = RESULT_FAIL
+
+		m = re_testcase_end.search(line)
+		if m:
+			if not name:
+				continue
+			if result is None:
+				result = RESULT_PASS
+			tests.append((name, result))
+
+			name = None
+			result = None
+
+	return tests
+
+def main(cmdline):
+	with open(cmdline.expected_results, 'r') as f_expected:
+		with open(cmdline.current_results, 'r') as f_current:
+			print('\nComparing expected results %r against results in %r\n--------------------'
+				% (cmdline.expected_results, cmdline.current_results))
+			compare(cmdline, f_expected, f_current)
+
+if __name__ == '__main__':
+	parser = argparse.ArgumentParser(description=doc)
+	parser.add_argument('expected_results', metavar='expected.junit-xml',
+			help='junit XML file listing the expected test results.')
+	parser.add_argument('current_results', metavar='current.junit-xml',
+			help='junit XML file listing the current test results.')
+
+	cmdline = parser.parse_args()
+	main(cmdline)
diff --git a/compare-results.sh b/compare-results.sh
deleted file mode 100755
index 6037174d..00000000
--- a/compare-results.sh
+++ /dev/null
@@ -1,213 +0,0 @@
-#!/usr/bin/env bash
-expected_file="$1"
-results_file="$2"
-
-# Copyright 2018 sysmocom - s.f.m.c. GmbH
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-usage() {
-  echo "
-Usage:
-
-  $(basename "$0") expected_results.junit-log current_results.junit-log [--allow-* [...]]
-
-Return 0 if the expected results match the current results exactly.
-
-  --allow-skip   Allow runnning less tests than are listed in the expected file.
-                 Default is to return failure on any skipped tests.
-  --allow-new    Allow more test results than found in the expected file.
-                 Default is to return failure on any unknown tests.
-  --allow-xpass  If a test was expected to fail but passed, return success.
-                 Default is to return failure on any mismatch.
-"
-}
-
-if [ ! -f "$expected_file" ]; then
-  usage
-  echo "Expected file not found: '$expected_file'"
-  exit 1
-fi
-
-if [ ! -f "$results_file" ]; then
-  usage
-  echo "Current results file not found: '$results_file'"
-  exit 1
-fi
-
-shift
-shift
-
-allow_xpass=0
-allow_skip=0
-allow_new=0
-
-while test -n "$1"; do
-  arg="$1"
-  if [ "x$arg" = "x--allow-xpass" ]; then
-    allow_xpass=1
-  elif [ "x$arg" = "x--allow-skip" ]; then
-    allow_skip=1
-  elif [ "x$arg" = "x--allow-new" ]; then
-    allow_new=1
-  else
-    usage
-    echo "Unknown argument: '$arg'"
-    exit 1
-  fi
-  shift
-done
-
-echo "Comparing expected results $expected_file against results in $results_file
---------------------"
-
-parse_testcase() {
-  line="$1"
-  suite_name="$(echo "$line" | sed 's,.*classname='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
-  test_name="$(echo "$line" | sed 's,.*\<name='"'"'\([^'"'"']*\)'"'"'.*,\1,')"
-  if [ -n "$(echo "$line" | grep '/>$')" ]; then
-    test_result="pass"
-  else
-    test_result="FAIL"
-  fi
-}
-
-pass=0
-xfail=0
-more_failures=0
-more_successes=0
-skipped=0
-new=0
-
-while read line; do
-  parse_testcase "$line"
-  exp_suite_name="$suite_name"
-  exp_test_name="$test_name"
-  exp_test_result="$test_result"
-  matched="0"
-
-  while read line; do
-    parse_testcase "$line"
-    if [ "x$exp_suite_name" != "x$suite_name" ]; then
-      continue
-    fi
-    if [ "x$exp_test_name" != "x$test_name" ]; then
-      continue
-    fi
-
-    if [ "x$exp_test_result" = "x$test_result" ]; then
-      if [ "x$exp_test_result" = "xFAIL" ]; then
-        exp_test_result="xfail"
-	(( xfail += 1 ))
-      else
-        (( pass += 1 ))
-      fi
-      echo "$exp_test_result $suite_name.$test_name"
-    else
-      if [ "x$exp_test_result" = "xFAIL" ]; then
-        exp_test_result="xfail"
-      fi
-      echo "$exp_test_result->$test_result $suite_name.$test_name"
-      if [ "x$test_result" = "xFAIL" ]; then
-        (( more_failures += 1 ))
-      else
-	(( more_successes += 1 ))
-      fi
-    fi
-    matched="1"
-    break
-  done <<< "$(grep "<testcase.*$exp_test_name" "$results_file")"
-
-  if [ "x$matched" = "x0" ]; then
-    echo "skipped $exp_suite_name.$exp_test_name"
-    (( skipped += 1 ))
-  fi
-
-done <<< "$(grep "<testcase" "$expected_file")"
-
-# Also catch all new tests that aren't covered in the expected results
-while read line; do
-  parse_testcase "$line"
-  got_suite_name="$suite_name"
-  got_test_name="$test_name"
-  got_test_result="$test_result"
-  matched="0"
-
-  while read line; do
-    parse_testcase "$line"
-    if [ "x$got_suite_name" != "x$suite_name" ]; then
-      continue
-    fi
-    if [ "x$got_test_name" != "x$test_name" ]; then
-      continue
-    fi
-
-    matched="1"
-    break
-  done <<< "$(grep "<testcase.*$test_name" "$expected_file")"
-
-  if [ "x$matched" = "x0" ]; then
-    echo "NEW-$got_test_result $got_suite_name.$got_test_name"
-    (( new += 1 ))
-  fi
-
-done <<< "$(grep "<testcase" "$results_file")"
-
-echo "--------------------"
-overall_verdict=0
-
-ask_update=""
-
-if [ "x$pass" != x0 ]; then
-  echo "$pass pass"
-fi
-
-if [ "x$xfail" != x0 ]; then
-  echo "$xfail xfail"
-fi
-
-if [ "x$skipped" != x0 ]; then
-  echo "$skipped skipped"
-  ask_update="$ask_update removed=$skipped"
-  if [ "x$allow_skip" = x0 ]; then
-    overall_verdict=4
-  fi
-fi
-
-if [ "x$new" != x0 ]; then
-  echo "$new new"
-  ask_update="$ask_update new=$new"
-  if [ "x$allow_new" = x0 ]; then
-    overall_verdict=3
-  fi
-fi
-
-if [ "x$more_successes" != x0 ]; then
-  echo "$more_successes pass unexpectedly"
-  ask_update="$ask_update xpass=$more_successes"
-  if [ "x$allow_xpass" = x0 ]; then
-    overall_verdict=2
-  fi
-fi
-
-if [ "x$more_failures" != x0 ]; then
-  echo "$more_failures FAIL"
-  overall_verdict=1
-fi
-
-if [ -n "$ask_update" ]; then
-  echo
-  echo "(Please update the expected results:$ask_update)"
-fi
-
-exit $overall_verdict
diff --git a/start-testsuite.sh b/start-testsuite.sh
index e5bc7563..a6380b73 100755
--- a/start-testsuite.sh
+++ b/start-testsuite.sh
@@ -60,7 +60,7 @@ if [ ! -f "$last_log" ]; then
   exit 1
 fi
 
-compare="$SUITE_DIR/../compare-results.sh"
+compare="$SUITE_DIR/../compare-results.py"
 if [ ! -x "$compare" ]; then
   echo "ERROR: cannot find $compare"
   exit 1