diff options
author | Neels Hofmeyr <neels@hofmeyr.de> | 2019-10-22 01:54:43 +0200 |
---|---|---|
committer | laforge <laforge@osmocom.org> | 2019-11-23 07:59:07 +0000 |
commit | b26196be09519fc32a8b348e02678e75f1164e6a (patch) | |
tree | 1399d361a3efa2ecf40405aa6f6c4e5671db87d9 /compare-results.sh | |
parent | 8df6962dec80e90aedd36984ae179d706235569c (diff) |
re-implement compare-results.sh as compare-results.py
The compare-results.sh is annoyingly slow. Since our ttcn3 tests containers
support Python 2, re-implement in Python for much quicker evaluation.
Change-Id: I0747c9d66ffc7e4121497a2416fca78d7b56c8e6
Diffstat (limited to 'compare-results.sh')
-rwxr-xr-x | compare-results.sh | 213 |
1 files changed, 0 insertions, 213 deletions
diff --git a/compare-results.sh b/compare-results.sh deleted file mode 100755 index 6037174d..00000000 --- a/compare-results.sh +++ /dev/null @@ -1,213 +0,0 @@ -#!/usr/bin/env bash -expected_file="$1" -results_file="$2" - -# Copyright 2018 sysmocom - s.f.m.c. GmbH -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -usage() { - echo " -Usage: - - $(basename "$0") expected_results.junit-log current_results.junit-log [--allow-* [...]] - -Return 0 if the expected results match the current results exactly. - - --allow-skip Allow runnning less tests than are listed in the expected file. - Default is to return failure on any skipped tests. - --allow-new Allow more test results than found in the expected file. - Default is to return failure on any unknown tests. - --allow-xpass If a test was expected to fail but passed, return success. - Default is to return failure on any mismatch. -" -} - -if [ ! -f "$expected_file" ]; then - usage - echo "Expected file not found: '$expected_file'" - exit 1 -fi - -if [ ! -f "$results_file" ]; then - usage - echo "Current results file not found: '$results_file'" - exit 1 -fi - -shift -shift - -allow_xpass=0 -allow_skip=0 -allow_new=0 - -while test -n "$1"; do - arg="$1" - if [ "x$arg" = "x--allow-xpass" ]; then - allow_xpass=1 - elif [ "x$arg" = "x--allow-skip" ]; then - allow_skip=1 - elif [ "x$arg" = "x--allow-new" ]; then - allow_new=1 - else - usage - echo "Unknown argument: '$arg'" - exit 1 - fi - shift -done - -echo "Comparing expected results $expected_file against results in $results_file ---------------------" - -parse_testcase() { - line="$1" - suite_name="$(echo "$line" | sed 's,.*classname='"'"'\([^'"'"']*\)'"'"'.*,\1,')" - test_name="$(echo "$line" | sed 's,.*\<name='"'"'\([^'"'"']*\)'"'"'.*,\1,')" - if [ -n "$(echo "$line" | grep '/>$')" ]; then - test_result="pass" - else - test_result="FAIL" - fi -} - -pass=0 -xfail=0 -more_failures=0 -more_successes=0 -skipped=0 -new=0 - -while read line; do - parse_testcase "$line" - exp_suite_name="$suite_name" - exp_test_name="$test_name" - exp_test_result="$test_result" - matched="0" - - while read line; do - parse_testcase "$line" - if [ "x$exp_suite_name" != "x$suite_name" ]; then - continue - fi - if [ "x$exp_test_name" != "x$test_name" ]; then - continue - fi - - if [ "x$exp_test_result" = "x$test_result" ]; then - if [ "x$exp_test_result" = "xFAIL" ]; then - exp_test_result="xfail" - (( xfail += 1 )) - else - (( pass += 1 )) - fi - echo "$exp_test_result $suite_name.$test_name" - else - if [ "x$exp_test_result" = "xFAIL" ]; then - exp_test_result="xfail" - fi - echo "$exp_test_result->$test_result $suite_name.$test_name" - if [ "x$test_result" = "xFAIL" ]; then - (( more_failures += 1 )) - else - (( more_successes += 1 )) - fi - fi - matched="1" - break - done <<< "$(grep "<testcase.*$exp_test_name" "$results_file")" - - if [ "x$matched" = "x0" ]; then - echo "skipped $exp_suite_name.$exp_test_name" - (( skipped += 1 )) - fi - -done <<< "$(grep "<testcase" "$expected_file")" - -# Also catch all new tests that aren't covered in the expected results -while read line; do - parse_testcase "$line" - got_suite_name="$suite_name" - got_test_name="$test_name" - got_test_result="$test_result" - matched="0" - - while read line; do - parse_testcase "$line" - if [ "x$got_suite_name" != "x$suite_name" ]; then - continue - fi - if [ "x$got_test_name" != "x$test_name" ]; then - continue - fi - - matched="1" - break - done <<< "$(grep "<testcase.*$test_name" "$expected_file")" - - if [ "x$matched" = "x0" ]; then - echo "NEW-$got_test_result $got_suite_name.$got_test_name" - (( new += 1 )) - fi - -done <<< "$(grep "<testcase" "$results_file")" - -echo "--------------------" -overall_verdict=0 - -ask_update="" - -if [ "x$pass" != x0 ]; then - echo "$pass pass" -fi - -if [ "x$xfail" != x0 ]; then - echo "$xfail xfail" -fi - -if [ "x$skipped" != x0 ]; then - echo "$skipped skipped" - ask_update="$ask_update removed=$skipped" - if [ "x$allow_skip" = x0 ]; then - overall_verdict=4 - fi -fi - -if [ "x$new" != x0 ]; then - echo "$new new" - ask_update="$ask_update new=$new" - if [ "x$allow_new" = x0 ]; then - overall_verdict=3 - fi -fi - -if [ "x$more_successes" != x0 ]; then - echo "$more_successes pass unexpectedly" - ask_update="$ask_update xpass=$more_successes" - if [ "x$allow_xpass" = x0 ]; then - overall_verdict=2 - fi -fi - -if [ "x$more_failures" != x0 ]; then - echo "$more_failures FAIL" - overall_verdict=1 -fi - -if [ -n "$ask_update" ]; then - echo - echo "(Please update the expected results:$ask_update)" -fi - -exit $overall_verdict |