From f72481a144b1a0e94b0fb053448f9fb6d36d1eb4 Mon Sep 17 00:00:00 2001 From: Gerald Combs Date: Sat, 5 May 2018 15:53:54 -0700 Subject: Test: Make sure we run our display filter tests. Change the test suite list in CMakeLists.txt to a static list. Add a CTest coverage unit test. Change-Id: I8459f320a2d0707618d6d56abdfce80274fddd2d Reviewed-on: https://code.wireshark.org/review/27377 Petri-Dish: Gerald Combs Tested-by: Petri Dish Buildbot Reviewed-by: Gerald Combs --- CMakeLists.txt | 31 +++++++++++++++++++++---------- test/config.py | 2 ++ test/suite_dfilter/group_bytes_ether.py | 2 +- test/suite_dfilter/group_bytes_ipv6.py | 2 +- test/suite_dfilter/group_bytes_type.py | 2 +- test/suite_dfilter/group_double.py | 2 +- test/suite_dfilter/group_integer.py | 2 +- test/suite_dfilter/group_integer_1byte.py | 2 +- test/suite_dfilter/group_ipv4.py | 2 +- test/suite_dfilter/group_membership.py | 2 +- test/suite_dfilter/group_range_method.py | 2 +- test/suite_dfilter/group_scanner.py | 2 +- test/suite_dfilter/group_string_type.py | 2 +- test/suite_dfilter/group_stringz.py | 2 +- test/suite_dfilter/group_time_relative.py | 2 +- test/suite_dfilter/group_time_type.py | 2 +- test/suite_dfilter/group_tvb.py | 2 +- test/suite_dfilter/group_uint64.py | 2 +- test/suite_unittests.py | 23 +++++++++++++++++++++++ test/test.py | 13 ++++++++----- 20 files changed, 70 insertions(+), 31 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index edaaa020ed..369c431b62 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2957,7 +2957,7 @@ if(RPMBUILD_EXECUTABLE) _SET_GITVERSION_CMAKE_VARIABLE(_git_description) - if (NOT _git_description) + if (NOT _git_description) # We're building the rpm outside the source. Guess the version from the dirname. get_filename_component(CMAKE_SOURCE_DIR_NAME ${CMAKE_SOURCE_DIR} NAME) # XXX this assumes the directory to start with "wireshark-" @@ -3160,26 +3160,37 @@ set_target_properties(test-programs PROPERTIES # Test suites enable_testing() -file(GLOB _test_suite_py_list test/suite_*.py) +# We could try to build this list dynamically, but given that we tend to +# go years between adding suites just run +# test/test.py --list-suites | sort +# and paste the output here. +set(_test_suite_list + suite_capture + suite_clopts + suite_decryption + suite_dfilter + suite_dissection + suite_fileformats + suite_io + suite_mergecap + suite_nameres + suite_text2pcap + suite_unittests + suite_wslua +) if(WIN32) set(_test_suite_program_path ./run/$) else() set(_test_suite_program_path ./run) endif() -# We currently don't handle spaces in arguments. On Windows this +# We don't currently handle spaces in arguments. On Windows this # means that you will probably have to pass in an interface index # instead of a name. set(TEST_EXTRA_ARGS "" CACHE STRING "Extra arguments to pass to test/test.py") separate_arguments(TEST_EXTRA_ARGS) -# We can enumerate suites two ways: by probing the filesystem and by -# running `test.py --list-suites`. Probe the filesystem for now, which -# should hopefully give us enough parallelization. If we want to split -# our tests by cases or individual tests we'll have to run and parse -# `test.py --list-cases` or `test.py --list` respectively. -foreach(_suite_py ${_test_suite_py_list}) - get_filename_component(_suite_name ${_suite_py} NAME_WE) +foreach(_suite_name ${_test_suite_list}) add_test( NAME ${_suite_name} COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test.py diff --git a/test/config.py b/test/config.py index cffa59fb69..a68de77d6e 100644 --- a/test/config.py +++ b/test/config.py @@ -63,6 +63,8 @@ key_dir = os.path.join(this_dir, 'keys') lua_dir = os.path.join(this_dir, 'lua') tools_dir = os.path.join(this_dir, '..', 'tools') +all_suites = [] + def canCapture(): # XXX This appears to be evaluated at the wrong time when called # from a unittest.skipXXX decorator. diff --git a/test/suite_dfilter/group_bytes_ether.py b/test/suite_dfilter/group_bytes_ether.py index 71f8d484e0..14fecc585a 100644 --- a/test/suite_dfilter/group_bytes_ether.py +++ b/test/suite_dfilter/group_bytes_ether.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_bytes_ether(dfiltertest.DFTestCase): trace_file = "ipx_rip.pcap" diff --git a/test/suite_dfilter/group_bytes_ipv6.py b/test/suite_dfilter/group_bytes_ipv6.py index 94237aa26c..f2964a09a9 100644 --- a/test/suite_dfilter/group_bytes_ipv6.py +++ b/test/suite_dfilter/group_bytes_ipv6.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_bytes_ipv6(dfiltertest.DFTestCase): trace_file = "ipv6.pcap" diff --git a/test/suite_dfilter/group_bytes_type.py b/test/suite_dfilter/group_bytes_type.py index 4031d27956..91e4271a27 100644 --- a/test/suite_dfilter/group_bytes_type.py +++ b/test/suite_dfilter/group_bytes_type.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_bytes_type(dfiltertest.DFTestCase): trace_file = "arp.pcap" diff --git a/test/suite_dfilter/group_double.py b/test/suite_dfilter/group_double.py index 3bf7cb6083..e9497db9f0 100644 --- a/test/suite_dfilter/group_double.py +++ b/test/suite_dfilter/group_double.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_double(dfiltertest.DFTestCase): diff --git a/test/suite_dfilter/group_integer.py b/test/suite_dfilter/group_integer.py index 42901fb1ea..86dee8fed7 100644 --- a/test/suite_dfilter/group_integer.py +++ b/test/suite_dfilter/group_integer.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_integer(dfiltertest.DFTestCase): trace_file = "ntp.pcap" diff --git a/test/suite_dfilter/group_integer_1byte.py b/test/suite_dfilter/group_integer_1byte.py index be8ba198e8..0043008a9b 100644 --- a/test/suite_dfilter/group_integer_1byte.py +++ b/test/suite_dfilter/group_integer_1byte.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_integer_1_byte(dfiltertest.DFTestCase): diff --git a/test/suite_dfilter/group_ipv4.py b/test/suite_dfilter/group_ipv4.py index 33a996f259..37f8bf13c6 100644 --- a/test/suite_dfilter/group_ipv4.py +++ b/test/suite_dfilter/group_ipv4.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_ipv4(dfiltertest.DFTestCase): trace_file = "nfs.pcap" diff --git a/test/suite_dfilter/group_membership.py b/test/suite_dfilter/group_membership.py index 6cfc9a207b..1c1201aa50 100644 --- a/test/suite_dfilter/group_membership.py +++ b/test/suite_dfilter/group_membership.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_membership(dfiltertest.DFTestCase): trace_file = "http.pcap" diff --git a/test/suite_dfilter/group_range_method.py b/test/suite_dfilter/group_range_method.py index 520b8f261a..2c56854354 100644 --- a/test/suite_dfilter/group_range_method.py +++ b/test/suite_dfilter/group_range_method.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_range(dfiltertest.DFTestCase): trace_file = "ipx_rip.pcap" diff --git a/test/suite_dfilter/group_scanner.py b/test/suite_dfilter/group_scanner.py index 4637f506fc..7ad1b5449c 100644 --- a/test/suite_dfilter/group_scanner.py +++ b/test/suite_dfilter/group_scanner.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_scanner(dfiltertest.DFTestCase): trace_file = "http.pcap" diff --git a/test/suite_dfilter/group_string_type.py b/test/suite_dfilter/group_string_type.py index 9bebfc94ab..b3f9065b08 100644 --- a/test/suite_dfilter/group_string_type.py +++ b/test/suite_dfilter/group_string_type.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_string(dfiltertest.DFTestCase): trace_file = "http.pcap" diff --git a/test/suite_dfilter/group_stringz.py b/test/suite_dfilter/group_stringz.py index 3e21a40fae..594ee215a7 100644 --- a/test/suite_dfilter/group_stringz.py +++ b/test/suite_dfilter/group_stringz.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_stringz(dfiltertest.DFTestCase): trace_file = "tftp.pcap" diff --git a/test/suite_dfilter/group_time_relative.py b/test/suite_dfilter/group_time_relative.py index 24cd79ecc6..963041c5e6 100644 --- a/test/suite_dfilter/group_time_relative.py +++ b/test/suite_dfilter/group_time_relative.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_time_relative(dfiltertest.DFTestCase): trace_file = "nfs.pcap" diff --git a/test/suite_dfilter/group_time_type.py b/test/suite_dfilter/group_time_type.py index 3c77c0cc67..801001d6cf 100644 --- a/test/suite_dfilter/group_time_type.py +++ b/test/suite_dfilter/group_time_type.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_time(dfiltertest.DFTestCase): trace_file = "http.pcap" diff --git a/test/suite_dfilter/group_tvb.py b/test/suite_dfilter/group_tvb.py index e00047b9e3..e257de1a4d 100644 --- a/test/suite_dfilter/group_tvb.py +++ b/test/suite_dfilter/group_tvb.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest import unittest class case_tvb(dfiltertest.DFTestCase): diff --git a/test/suite_dfilter/group_uint64.py b/test/suite_dfilter/group_uint64.py index a4e50d7346..3194a25e2d 100644 --- a/test/suite_dfilter/group_uint64.py +++ b/test/suite_dfilter/group_uint64.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -import dfiltertest +from suite_dfilter import dfiltertest class case_uint64(dfiltertest.DFTestCase): trace_file = "nfs.pcap" diff --git a/test/suite_unittests.py b/test/suite_unittests.py index 8c3fc59ee0..6d896d6b17 100644 --- a/test/suite_unittests.py +++ b/test/suite_unittests.py @@ -12,7 +12,10 @@ '''EPAN unit tests''' import config +import difflib import os.path +import pprint +import re import subprocesstest import unittest @@ -49,6 +52,26 @@ class case_unittests(subprocesstest.SubprocessTestCase): '''fieldcount''' self.assertRun((config.cmd_tshark, '-G', 'fieldcount')) + def test_unit_ctest_coverage(self): + '''Make sure CTest runs all of our tests.''' + with open(os.path.join(config.this_dir, '..', 'CMakeLists.txt')) as cml_fd: + suite_re = re.compile('set *\( *_test_suite_list') + in_list = False + cml_suites = [] + for cml_line in cml_fd: + if suite_re.search(cml_line): + in_list = True + continue + if in_list: + if ')' in cml_line: + break + cml_suites.append(cml_line.strip()) + cml_suites.sort() + if not config.all_suites == cml_suites: + diff = '\n'.join(list(difflib.unified_diff(config.all_suites, cml_suites, 'all test suites', 'CMakeLists.txt test suites'))) + self.fail("CMakeLists.txt doesn't test all available suites:\n" + diff) + + class Proto: """Data for a protocol.""" def __init__(self, line): diff --git a/test/test.py b/test/test.py index 78421bd911..91bddaa5f8 100755 --- a/test/test.py +++ b/test/test.py @@ -87,12 +87,15 @@ def main(): print('\n'.join(run_ids)) sys.exit(0) + all_suites = set() + for aid in all_ids: + aparts = aid.split('.') + all_suites |= {aparts[0]} + config.all_suites = list(all_suites) + config.all_suites.sort() + if args.list_suites: - suites = set() - for rid in run_ids: - rparts = rid.split('.') - suites |= {rparts[0]} - print('\n'.join(list(suites))) + print('\n'.join(config.all_suites)) sys.exit(0) if args.list_cases: -- cgit v1.2.3