Test: Make sure we run our display filter tests.

Change the test suite list in CMakeLists.txt to a static list. Add a
CTest coverage unit test.

Change-Id: I8459f320a2d0707618d6d56abdfce80274fddd2d
Reviewed-on: https://code.wireshark.org/review/27377
Petri-Dish: Gerald Combs <gerald@wireshark.org>
Tested-by: Petri Dish Buildbot
Reviewed-by: Gerald Combs <gerald@wireshark.org>

20 files changed, 70 insertions, 31 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index edaaa020ed..369c431b62 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -2957,7 +2957,7 @@ if(RPMBUILD_EXECUTABLE)
 
 	_SET_GITVERSION_CMAKE_VARIABLE(_git_description)
 
-	if (NOT _git_description)	
+	if (NOT _git_description)
 		# We're building the rpm outside the source. Guess the version from the dirname.
 		get_filename_component(CMAKE_SOURCE_DIR_NAME ${CMAKE_SOURCE_DIR} NAME)
 		# XXX this assumes the directory to start with "wireshark-"
@@ -3160,26 +3160,37 @@ set_target_properties(test-programs PROPERTIES
 
 # Test suites
 enable_testing()
-file(GLOB _test_suite_py_list test/suite_*.py)
+# We could try to build this list dynamically, but given that we tend to
+# go years between adding suites just run
+#     test/test.py --list-suites | sort
+# and paste the output here.
+set(_test_suite_list
+	suite_capture
+	suite_clopts
+	suite_decryption
+	suite_dfilter
+	suite_dissection
+	suite_fileformats
+	suite_io
+	suite_mergecap
+	suite_nameres
+	suite_text2pcap
+	suite_unittests
+	suite_wslua
+)
 if(WIN32)
 	set(_test_suite_program_path ./run/$<CONFIG>)
 else()
 	set(_test_suite_program_path ./run)
 endif()
 
-# We currently don't handle spaces in arguments. On Windows this
+# We don't currently handle spaces in arguments. On Windows this
 # means that you will probably have to pass in an interface index
 # instead of a name.
 set(TEST_EXTRA_ARGS "" CACHE STRING "Extra arguments to pass to test/test.py")
 separate_arguments(TEST_EXTRA_ARGS)
 
-# We can enumerate suites two ways: by probing the filesystem and by
-# running `test.py --list-suites`. Probe the filesystem for now, which
-# should hopefully give us enough parallelization. If we want to split
-# our tests by cases or individual tests we'll have to run and parse
-# `test.py --list-cases` or `test.py --list` respectively.
-foreach(_suite_py ${_test_suite_py_list})
-	get_filename_component(_suite_name ${_suite_py} NAME_WE)
+foreach(_suite_name ${_test_suite_list})
 	add_test(
 		NAME ${_suite_name}
 		COMMAND ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/test/test.py
diff --git a/test/config.py b/test/config.py
index cffa59fb69..a68de77d6e 100644
--- a/test/config.py
+++ b/test/config.py
@@ -63,6 +63,8 @@ key_dir = os.path.join(this_dir, 'keys')
 lua_dir = os.path.join(this_dir, 'lua')
 tools_dir = os.path.join(this_dir, '..', 'tools')
 
+all_suites = []
+
 def canCapture():
     # XXX This appears to be evaluated at the wrong time when called
     # from a unittest.skipXXX decorator.
diff --git a/test/suite_dfilter/group_bytes_ether.py b/test/suite_dfilter/group_bytes_ether.py
index 71f8d484e0..14fecc585a 100644
--- a/test/suite_dfilter/group_bytes_ether.py
+++ b/test/suite_dfilter/group_bytes_ether.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_ether(dfiltertest.DFTestCase):
     trace_file = "ipx_rip.pcap"
diff --git a/test/suite_dfilter/group_bytes_ipv6.py b/test/suite_dfilter/group_bytes_ipv6.py
index 94237aa26c..f2964a09a9 100644
--- a/test/suite_dfilter/group_bytes_ipv6.py
+++ b/test/suite_dfilter/group_bytes_ipv6.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_ipv6(dfiltertest.DFTestCase):
     trace_file = "ipv6.pcap"
diff --git a/test/suite_dfilter/group_bytes_type.py b/test/suite_dfilter/group_bytes_type.py
index 4031d27956..91e4271a27 100644
--- a/test/suite_dfilter/group_bytes_type.py
+++ b/test/suite_dfilter/group_bytes_type.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_bytes_type(dfiltertest.DFTestCase):
     trace_file = "arp.pcap"
diff --git a/test/suite_dfilter/group_double.py b/test/suite_dfilter/group_double.py
index 3bf7cb6083..e9497db9f0 100644
--- a/test/suite_dfilter/group_double.py
+++ b/test/suite_dfilter/group_double.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_double(dfiltertest.DFTestCase):
 
diff --git a/test/suite_dfilter/group_integer.py b/test/suite_dfilter/group_integer.py
index 42901fb1ea..86dee8fed7 100644
--- a/test/suite_dfilter/group_integer.py
+++ b/test/suite_dfilter/group_integer.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_integer(dfiltertest.DFTestCase):
     trace_file = "ntp.pcap"
diff --git a/test/suite_dfilter/group_integer_1byte.py b/test/suite_dfilter/group_integer_1byte.py
index be8ba198e8..0043008a9b 100644
--- a/test/suite_dfilter/group_integer_1byte.py
+++ b/test/suite_dfilter/group_integer_1byte.py
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_integer_1_byte(dfiltertest.DFTestCase):
 
diff --git a/test/suite_dfilter/group_ipv4.py b/test/suite_dfilter/group_ipv4.py
index 33a996f259..37f8bf13c6 100644
--- a/test/suite_dfilter/group_ipv4.py
+++ b/test/suite_dfilter/group_ipv4.py
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_ipv4(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"
diff --git a/test/suite_dfilter/group_membership.py b/test/suite_dfilter/group_membership.py
index 6cfc9a207b..1c1201aa50 100644
--- a/test/suite_dfilter/group_membership.py
+++ b/test/suite_dfilter/group_membership.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_membership(dfiltertest.DFTestCase):
     trace_file = "http.pcap"
diff --git a/test/suite_dfilter/group_range_method.py b/test/suite_dfilter/group_range_method.py
index 520b8f261a..2c56854354 100644
--- a/test/suite_dfilter/group_range_method.py
+++ b/test/suite_dfilter/group_range_method.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_range(dfiltertest.DFTestCase):
     trace_file = "ipx_rip.pcap"
diff --git a/test/suite_dfilter/group_scanner.py b/test/suite_dfilter/group_scanner.py
index 4637f506fc..7ad1b5449c 100644
--- a/test/suite_dfilter/group_scanner.py
+++ b/test/suite_dfilter/group_scanner.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_scanner(dfiltertest.DFTestCase):
     trace_file = "http.pcap"
diff --git a/test/suite_dfilter/group_string_type.py b/test/suite_dfilter/group_string_type.py
index 9bebfc94ab..b3f9065b08 100644
--- a/test/suite_dfilter/group_string_type.py
+++ b/test/suite_dfilter/group_string_type.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_string(dfiltertest.DFTestCase):
     trace_file = "http.pcap"
diff --git a/test/suite_dfilter/group_stringz.py b/test/suite_dfilter/group_stringz.py
index 3e21a40fae..594ee215a7 100644
--- a/test/suite_dfilter/group_stringz.py
+++ b/test/suite_dfilter/group_stringz.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_stringz(dfiltertest.DFTestCase):
     trace_file = "tftp.pcap"
diff --git a/test/suite_dfilter/group_time_relative.py b/test/suite_dfilter/group_time_relative.py
index 24cd79ecc6..963041c5e6 100644
--- a/test/suite_dfilter/group_time_relative.py
+++ b/test/suite_dfilter/group_time_relative.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_time_relative(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"
diff --git a/test/suite_dfilter/group_time_type.py b/test/suite_dfilter/group_time_type.py
index 3c77c0cc67..801001d6cf 100644
--- a/test/suite_dfilter/group_time_type.py
+++ b/test/suite_dfilter/group_time_type.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_time(dfiltertest.DFTestCase):
     trace_file = "http.pcap"
diff --git a/test/suite_dfilter/group_tvb.py b/test/suite_dfilter/group_tvb.py
index e00047b9e3..e257de1a4d 100644
--- a/test/suite_dfilter/group_tvb.py
+++ b/test/suite_dfilter/group_tvb.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 import unittest
 
 class case_tvb(dfiltertest.DFTestCase):
diff --git a/test/suite_dfilter/group_uint64.py b/test/suite_dfilter/group_uint64.py
index a4e50d7346..3194a25e2d 100644
--- a/test/suite_dfilter/group_uint64.py
+++ b/test/suite_dfilter/group_uint64.py
@@ -2,7 +2,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import dfiltertest
+from suite_dfilter import dfiltertest
 
 class case_uint64(dfiltertest.DFTestCase):
     trace_file = "nfs.pcap"
diff --git a/test/suite_unittests.py b/test/suite_unittests.py
index 8c3fc59ee0..6d896d6b17 100644
--- a/test/suite_unittests.py
+++ b/test/suite_unittests.py
@@ -12,7 +12,10 @@
 '''EPAN unit tests'''
 
 import config
+import difflib
 import os.path
+import pprint
+import re
 import subprocesstest
 import unittest
 
@@ -49,6 +52,26 @@ class case_unittests(subprocesstest.SubprocessTestCase):
         '''fieldcount'''
         self.assertRun((config.cmd_tshark, '-G', 'fieldcount'))
 
+    def test_unit_ctest_coverage(self):
+        '''Make sure CTest runs all of our tests.'''
+        with open(os.path.join(config.this_dir, '..', 'CMakeLists.txt')) as cml_fd:
+            suite_re = re.compile('set *\( *_test_suite_list')
+            in_list = False
+            cml_suites = []
+            for cml_line in cml_fd:
+                if suite_re.search(cml_line):
+                    in_list = True
+                    continue
+                if in_list:
+                    if ')' in cml_line:
+                        break
+                    cml_suites.append(cml_line.strip())
+        cml_suites.sort()
+        if not config.all_suites == cml_suites:
+            diff = '\n'.join(list(difflib.unified_diff(config.all_suites, cml_suites, 'all test suites', 'CMakeLists.txt test suites')))
+            self.fail("CMakeLists.txt doesn't test all available suites:\n" + diff)
+
+
 class Proto:
     """Data for a protocol."""
     def __init__(self, line):
diff --git a/test/test.py b/test/test.py
index 78421bd911..91bddaa5f8 100755
--- a/test/test.py
+++ b/test/test.py
@@ -87,12 +87,15 @@ def main():
         print('\n'.join(run_ids))
         sys.exit(0)
 
+    all_suites = set()
+    for aid in all_ids:
+        aparts = aid.split('.')
+        all_suites |= {aparts[0]}
+    config.all_suites = list(all_suites)
+    config.all_suites.sort()
+
     if args.list_suites:
-        suites = set()
-        for rid in run_ids:
-            rparts = rid.split('.')
-            suites |= {rparts[0]}
-        print('\n'.join(list(suites)))
+        print('\n'.join(config.all_suites))
         sys.exit(0)
 
     if args.list_cases: