test: fail tests when programs are missing

Building only a subset of programs is not a very common situation, it is
more likely that some feature was accidentally disabled. For that
reason, fail tests by default unless a program is explicitly permitted
to be missing.

The '-v' test is now dropped from the Travis tests, the sole reason of
adding it was to see which tests got (accidentally) skipped.

Change-Id: I725f4508541d8ed980e17d69fb7aee1ad2875d73
Reviewed-on: https://code.wireshark.org/review/31660
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>

3 files changed, 17 insertions, 5 deletions

diff --git a/test/conftest.py b/test/conftest.py
index 2146cb46a3..3b70e049c3 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -19,6 +19,9 @@ def pytest_addoption(parser):
         help='Capture interface index or name.'
     )
     parser.addoption('--program-path', help='Path to Wireshark executables.')
+    parser.addoption('--skip-missing-programs',
+        help='Skip tests that lack programs from this list instead of failing'
+             ' them. Use "all" to ignore all missing programs.')
 
 _all_test_groups = None
 
diff --git a/test/fixtures_ws.py b/test/fixtures_ws.py
index 9999576c30..4b408acd5b 100644
--- a/test/fixtures_ws.py
+++ b/test/fixtures_ws.py
@@ -72,14 +72,20 @@ def program_path(request):
 
 
 @fixtures.fixture(scope='session')
-def program(program_path):
+def program(program_path, request):
+    skip_if_missing = request.config.getoption('--skip-missing-programs',
+                                               default='')
+    skip_if_missing = skip_if_missing.split(',') if skip_if_missing else []
+    dotexe = ''
+    if sys.platform.startswith('win32'):
+        dotexe = '.exe'
+
     def resolver(name):
-        dotexe = ''
-        if sys.platform.startswith('win32'):
-            dotexe = '.exe'
         path = os.path.abspath(os.path.join(program_path, name + dotexe))
         if not os.access(path, os.X_OK):
-            fixtures.skip('Program %s is not available' % (name,))
+            if skip_if_missing == ['all'] or name in skip_if_missing:
+                fixtures.skip('Program %s is not available' % (name,))
+            raise AssertionError('Program %s is not available' % (name,))
         return path
     return resolver
 
diff --git a/test/test.py b/test/test.py
index c64891bb1c..27d75efc63 100755
--- a/test/test.py
+++ b/test/test.py
@@ -56,6 +56,9 @@ def main():
     cap_group.add_argument('-E', '--disable-capture', action='store_true', help='Disable capture tests')
     cap_group.add_argument('-i', '--capture-interface', help='Capture interface index or name')
     parser.add_argument('-p', '--program-path', default=os.path.curdir, help='Path to Wireshark executables.')
+    parser.add_argument('--skip-missing-programs',
+        help='Skip tests that lack programs from this list instead of failing'
+             ' them. Use "all" to ignore all missing programs.')
     list_group = parser.add_mutually_exclusive_group()
     list_group.add_argument('-l', '--list', action='store_true', help='List tests. One of "all" or a full or partial test name.')
     list_group.add_argument('--list-suites', action='store_true', help='List all suites.')