test/dfilter: be explicit with the expected error message

Instead of just reporting a mismatching error code, include the program
output. This should help tracking down unexpected errors. While at it,
check the expected error message too.

Change-Id: Ib8fe51cc06b795bb54bfe1e6eaa828c6ba1128ef
Reviewed-on: https://code.wireshark.org/review/31714
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>

6 files changed, 47 insertions, 24 deletions

diff --git a/doc/README.display_filter b/doc/README.display_filter
index bea42518d2..5d3a76b9bc 100644
--- a/doc/README.display_filter
+++ b/doc/README.display_filter
@@ -574,11 +574,11 @@ checkDFilterCount(dfilter, expected_count)
     matches is not the same as failure to compile the display
     filter string.
 
-checkDFilterFail(dfilter)
+checkDFilterFail(dfilter, error)
 
-    This will run tshark with the display filter, and
-    assert that tshark fails. This is useful when expecting
-    display filter syntax errors to be caught.
+    This will run dftest with the display filter, and check
+    that it fails with a given error message. This is useful
+    when expecting display filter syntax errors to be caught.
 
 To execute tests:
 
diff --git a/test/suite_dfilter/dfiltertest.py b/test/suite_dfilter/dfiltertest.py
index 501e53a58f..a8d89d599e 100644
--- a/test/suite_dfilter/dfiltertest.py
+++ b/test/suite_dfilter/dfiltertest.py
@@ -20,6 +20,11 @@ def dfilter_cmd(cmd_tshark, capture_file, request):
     return wrapped
 
 
+@fixtures.fixture(scope='session')
+def cmd_dftest(program):
+    return program('dftest')
+
+
 @fixtures.fixture
 def checkDFilterCount(dfilter_cmd, base_env):
     def checkDFilterCount_real(dfilter, expected_count):
@@ -37,12 +42,18 @@ def checkDFilterCount(dfilter_cmd, base_env):
 
 
 @fixtures.fixture
-def checkDFilterFail(dfilter_cmd, base_env):
-    def checkDFilterFail_real(dfilter):
-        """Run a display filter and expect tshark to fail."""
-        exitcode = subprocess.call(dfilter_cmd(dfilter),
-                                   stdout=subprocess.DEVNULL,
-                                   stderr=subprocess.STDOUT,
-                                   env=base_env)
-        assert exitcode == 2, 'Expected process to fail, got %d' % (exitcode,)
+def checkDFilterFail(cmd_dftest, base_env):
+    def checkDFilterFail_real(dfilter, error_message):
+        """Run a display filter and expect dftest to fail."""
+        proc = subprocess.Popen([cmd_dftest, dfilter],
+                                stdout=subprocess.PIPE,
+                                stderr=subprocess.PIPE,
+                                universal_newlines=True,
+                                env=base_env)
+        outs, errs = proc.communicate()
+        assert errs.strip() == 'dftest: %s' % (error_message,), \
+            'Unexpected dftest stderr:\n%s\nstdout:\n%s' % (errs, outs)
+        assert proc.returncode == 2, \
+            'Unexpected dftest exit code: %d. stdout:\n%s\n' % \
+            (proc.returncode, outs)
     return checkDFilterFail_real
diff --git a/test/suite_dfilter/group_dfunction_string.py b/test/suite_dfilter/group_dfunction_string.py
index c1bd17342c..56343739d2 100644
--- a/test/suite_dfilter/group_dfunction_string.py
+++ b/test/suite_dfilter/group_dfunction_string.py
@@ -21,19 +21,23 @@ class case_dfunction_string(unittest.TestCase):
     def test_fail_1(self, checkDFilterFail):
         # Invalid filter (only non-string fields are supported)
         dfilter = "string(dhcp.server) == hostname"
-        checkDFilterFail(dfilter)
+        error = 'To string conversion for this field is not supported'
+        checkDFilterFail(dfilter, error)
 
     def test_fail_2(self, checkDFilterFail):
         # Invalid field: value
         dfilter = "string(123) == \"123\""
-        checkDFilterFail(dfilter)
+        error = 'To string conversion for this field is not supported'
+        checkDFilterFail(dfilter, error)
 
     def test_fail_3(self, checkDFilterFail):
         # Invalid field: protocol
         dfilter = "string(dhcp) == hostname"
-        checkDFilterFail(dfilter)
+        error = 'To string conversion for this field is not supported'
+        checkDFilterFail(dfilter, error)
 
     def test_fail_4(self, checkDFilterFail):
         # Invalid field: bytes
         dfilter = "string(dhcp.option.value) == \"hostname\""
-        checkDFilterFail(dfilter)
+        error = 'To string conversion for this field is not supported'
+        checkDFilterFail(dfilter, error)
diff --git a/test/suite_dfilter/group_integer.py b/test/suite_dfilter/group_integer.py
index 1a6425e4ce..fb38b27ea4 100644
--- a/test/suite_dfilter/group_integer.py
+++ b/test/suite_dfilter/group_integer.py
@@ -22,17 +22,20 @@ class case_integer(unittest.TestCase):
     def test_eq_3(self, checkDFilterFail):
         # Invalid filter (only one equals sign)
         dfilter = "ip.version = 4"
-        checkDFilterFail(dfilter)
+        error = '"=" was unexpected in this context.'
+        checkDFilterFail(dfilter, error)
 
     def test_eq_4(self, checkDFilterFail):
         # Invalid filter
         dfilter = "ip.version == the quick brown fox jumps over the lazy dog"
-        checkDFilterFail(dfilter)
+        error = '"quick" was unexpected in this context.'
+        checkDFilterFail(dfilter, error)
 
     def test_eq_5(self, checkDFilterFail):
         # Invalid filter
         dfilter = "ip.version == 4 the quick brown fox jumps over the lazy dog"
-        checkDFilterFail(dfilter)
+        error = '"the" was unexpected in this context.'
+        checkDFilterFail(dfilter, error)
 
     def test_ne_1(self, checkDFilterCount):
         dfilter = "ip.version != 0"
diff --git a/test/suite_dfilter/group_string_type.py b/test/suite_dfilter/group_string_type.py
index a1c719e509..bf469d35c4 100644
--- a/test/suite_dfilter/group_string_type.py
+++ b/test/suite_dfilter/group_string_type.py
@@ -150,7 +150,8 @@ class case_string(unittest.TestCase):
 
     def test_contains_upper_2(self, checkDFilterFail):
         dfilter = 'upper(tcp.seq) == 4'
-        checkDFilterFail(dfilter)
+        error = 'Only strings can be used in upper() or lower() or len()'
+        checkDFilterFail(dfilter, error)
 
     def test_contains_lower_0(self, checkDFilterCount):
         dfilter = 'lower(http.user_agent) contains "UPDATE"'
@@ -162,7 +163,8 @@ class case_string(unittest.TestCase):
 
     def test_eq_lower_1(self, checkDFilterFail):
         dfilter = 'lower(tcp.seq) == 4'
-        checkDFilterFail(dfilter)
+        error = 'Only strings can be used in upper() or lower() or len()'
+        checkDFilterFail(dfilter, error)
 
     def test_string_len(self, checkDFilterCount):
         dfilter = 'len(http.request.method) == 4'
diff --git a/test/suite_dfilter/group_time_type.py b/test/suite_dfilter/group_time_type.py
index 2447363cfe..daacf3f9f6 100644
--- a/test/suite_dfilter/group_time_type.py
+++ b/test/suite_dfilter/group_time_type.py
@@ -82,14 +82,17 @@ class case_time(unittest.TestCase):
     def test_bad_time_1(self, checkDFilterFail):
         # No text is permitted after the time.
         dfilter = 'frame.time == "Dec 31, 2002 13:56:31.3 UTC"'
-        checkDFilterFail(dfilter)
+        error = '"Dec 31, 2002 13:56:31.3 UTC" is not a valid absolute time. Example: "Nov 12, 1999 08:55:44.123" or "2011-07-04 12:34:56"'
+        checkDFilterFail(dfilter, error)
 
     def test_bad_time_2(self, checkDFilterFail):
         # Miliseconds can only occur after seconds.
         dfilter = 'frame.time == "2002-12-31 13:55.3"'
-        checkDFilterFail(dfilter)
+        error = '"2002-12-31 13:55.3" is not a valid absolute time. Example: "Nov 12, 1999 08:55:44.123" or "2011-07-04 12:34:56"'
+        checkDFilterFail(dfilter, error)
 
     def test_bad_time_3(self, checkDFilterFail):
         # Reject months in a different locale (mrt is March in nl_NL.UTF-8).
         dfilter = 'frame.time == "mrt 1, 2000 00:00:00"'
-        checkDFilterFail(dfilter)
+        error = '"mrt 1, 2000 00:00:00" is not a valid absolute time. Example: "Nov 12, 1999 08:55:44.123" or "2011-07-04 12:34:56"'
+        checkDFilterFail(dfilter, error)