Add a preference for ignoring duplicate frames

Sometimes you have a capture file that has many duplicate frames
because of how the capture was made, and its convenient to ignore
the duplicates so you can concentrate on the data and not all
the TCP warnings.

This adds a preference in the "Protocols" section to ignore
duplicates. This currently only works while reading a capture file
*not* during a live capture.

6 files changed, 409 insertions, 0 deletions

diff --git a/epan/CMakeLists.txt b/epan/CMakeLists.txt
index ad0ae1559c..031231a946 100644
--- a/epan/CMakeLists.txt
+++ b/epan/CMakeLists.txt
@@ -87,6 +87,7 @@ set(LIBWIRESHARK_PUBLIC_HEADERS
 	expert.h
 	export_object.h
 	exported_pdu.h
+	fifo_string_cache.h
 	filter_expressions.h
 	follow.h
 	frame_data.h
@@ -199,6 +200,7 @@ set(LIBWIRESHARK_NONGENERATED_FILES
 	expert.c
 	export_object.c
 	exported_pdu.c
+	fifo_string_cache.c
 	filter_expressions.c
 	follow.c
 	frame_data.c
@@ -401,6 +403,15 @@ set_target_properties(exntest PROPERTIES
 	COMPILE_FLAGS "${WERROR_COMMON_FLAGS}"
 )
 
+add_executable(fifo_string_cache_test EXCLUDE_FROM_ALL fifo_string_cache_test.c)
+target_link_libraries(fifo_string_cache_test epan)
+set_target_properties(fifo_string_cache_test PROPERTIES
+	FOLDER "Tests"
+	EXCLUDE_FROM_DEFAULT_BUILD True
+	COMPILE_DEFINITIONS "WS_BUILD_DLL"
+	COMPILE_FLAGS "${WERROR_COMMON_FLAGS}"
+)
+
 add_executable(oids_test EXCLUDE_FROM_ALL oids_test.c)
 target_link_libraries(oids_test epan)
 set_target_properties(oids_test PROPERTIES
diff --git a/epan/fifo_string_cache.c b/epan/fifo_string_cache.c
new file mode 100644
index 0000000000..eb8034f0c7
--- /dev/null
+++ b/epan/fifo_string_cache.c
@@ -0,0 +1,92 @@
+/* fifo_string_cache.c
+ * A string cache, possibly with a bounded size, using FIFO order to control
+ * the size.
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 1998 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+#include <stdio.h>
+#include "fifo_string_cache.h"
+
+void
+fifo_string_cache_init(fifo_string_cache_t *fcache, guint max_entries, GDestroyNotify string_free_func)
+{
+    fcache->set = g_hash_table_new_full(g_str_hash, g_str_equal, string_free_func, NULL);
+    fcache->head = NULL;
+    fcache->tail = NULL;
+    fcache->max_entries = max_entries;
+}
+
+void
+fifo_string_cache_free(fifo_string_cache_t *fcache)
+{
+    if (fcache->set != NULL) {
+        g_hash_table_destroy(fcache->set);
+        fcache->set = NULL;
+    }
+    if (fcache->head != NULL) {
+        g_slist_free(fcache->head);
+        fcache->head = NULL;
+        fcache->tail = NULL;
+    }
+}
+
+gboolean
+fifo_string_cache_contains(fifo_string_cache_t *fcache, const gchar *entry)
+{
+    return g_hash_table_contains(fcache->set, entry);
+}
+
+gboolean
+fifo_string_cache_insert(fifo_string_cache_t *fcache, const gchar *entry)
+{
+    GSList *prev_head;
+    GSList *new_start_of_tail;
+
+    // In GLIB 2.40, g_hash_table_insert() returns a bool that gives us what we
+    // need (did the entry exist already?). But, if we're not using that
+    // version, we need to first check if the entry exists. So we just check
+    // the hash all the time, regardless of GLIB version.
+    gboolean exists;
+    exists = g_hash_table_contains(fcache->set, entry);
+    if (exists) {
+        return TRUE;
+    }
+
+    // Shall we remove one item?
+    if (fcache->max_entries > 0) {
+        if (g_hash_table_size(fcache->set) == fcache->max_entries) {
+            g_hash_table_remove(fcache->set, fcache->head->data);
+            prev_head = fcache->head;
+            fcache->head = fcache->head->next;
+            g_slist_free_1(prev_head);
+
+            // If max_entries is 1, the head was also the tail. Reset the tail.
+            if (fcache->tail == prev_head) {
+                fcache->tail = NULL;
+            }
+
+            // Now the size of the hash table is max_entries
+        }
+    }
+
+    g_hash_table_insert(fcache->set, (gpointer) entry, /*value=*/NULL);
+    // Do we need to constrain the number of entries?
+    if (fcache->max_entries > 0) {
+        // Keep track of the new entry at the end of the queue
+        new_start_of_tail = g_slist_append(fcache->tail, (gpointer) entry);
+        // Set the new tail
+        if (fcache->tail == NULL) {
+            fcache->tail = new_start_of_tail;
+            // This is the first entry, so head is NULL too. Set it.
+            fcache->head = new_start_of_tail;
+        } else {
+            fcache->tail = new_start_of_tail->next;
+        }
+    }
+
+    return FALSE;
+}
diff --git a/epan/fifo_string_cache.h b/epan/fifo_string_cache.h
new file mode 100644
index 0000000000..9f4fd407c0
--- /dev/null
+++ b/epan/fifo_string_cache.h
@@ -0,0 +1,59 @@
+/* fifo_string_cache.h
+ * A string cache, possibly with a bounded size, using FIFO order to control
+ * the size.
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 1998 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+#ifndef __FIFO_STRING_CACHE_H__
+#define __FIFO_STRING_CACHE_H__
+
+#include <glib.h>
+#include "ws_symbol_export.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+typedef struct {
+    GHashTable          *set;
+    GSList              *head;
+    GSList              *tail;
+    guint               max_entries;
+} fifo_string_cache_t;
+
+// These functions are marked with WS_DLL_PUBLIC so they can be unit-tested
+
+// Initialize an object. If string_free_func is given, then the
+// fifo_string_cache owns the string data, and will call this string_free_func
+// during fifo_string_cache_free().
+// If string_free_func is NULL, then the caller owns the string data, and it is
+// the caller that is responsible for freeing the data.
+WS_DLL_PUBLIC void
+fifo_string_cache_init(fifo_string_cache_t *fcache, guint max_entries, GDestroyNotify string_free_func);
+
+// Free all memory owned by the fifo_string_cache. Whether or not the
+// fifoe_string_cache owns the actual strings depends on whether a
+// string_free_func was passed in during fifo_string_cache_init().
+WS_DLL_PUBLIC void
+fifo_string_cache_free(fifo_string_cache_t *fcache);
+
+// Does the cache contain a specific string?
+WS_DLL_PUBLIC gboolean
+fifo_string_cache_contains(fifo_string_cache_t *fcache, const gchar *entry);
+
+// Insert a string. The return value indicates whether the string was already
+// in the cache before this function was called. If the string was newly
+// inserted, and max_entries is > 0, and inserting the string would have caused
+// max_entries to be exceeded, the oldest inserted key is removed (FIFO order).
+WS_DLL_PUBLIC gboolean
+fifo_string_cache_insert(fifo_string_cache_t *fcache, const gchar *entry);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* __FIFO_STRING_CACHE_H__ */
diff --git a/epan/fifo_string_cache_test.c b/epan/fifo_string_cache_test.c
new file mode 100644
index 0000000000..78a0a3dc2b
--- /dev/null
+++ b/epan/fifo_string_cache_test.c
@@ -0,0 +1,230 @@
+/* fifo_string_cache_test.c
+ *
+ * Wireshark - Network traffic analyzer
+ * By Gerald Combs <gerald@wireshark.org>
+ * Copyright 1998 Gerald Combs
+ *
+ * SPDX-License-Identifier: GPL-2.0-or-later
+ */
+
+#include "config.h"
+#undef G_DISABLE_ASSERT
+
+#include <stdio.h>
+#include <string.h>
+#include <glib.h>
+
+#include "fifo_string_cache.h"
+
+
+// Simple test of insertion and checking its true/false values
+static void
+test_fifo_string_cache_01(void)
+{
+    fifo_string_cache_t fcache;
+    gboolean has;
+
+    fifo_string_cache_init(&fcache, 10, NULL);
+
+    has = fifo_string_cache_insert(&fcache, "alpha");
+    g_assert_false(has);
+
+    has = fifo_string_cache_insert(&fcache, "alpha");
+    g_assert_true(has);
+
+    has = fifo_string_cache_insert(&fcache, "beta");
+    g_assert_false(has);
+
+    has = fifo_string_cache_insert(&fcache, "beta");
+    g_assert_true(has);
+
+    has = fifo_string_cache_insert(&fcache, "alpha");
+    g_assert_true(has);
+
+    fifo_string_cache_free(&fcache);
+}
+
+// Is the max_entries honored?
+static void
+test_fifo_string_cache_02(void)
+{
+    fifo_string_cache_t fcache;
+    gboolean has;
+    fifo_string_cache_init(&fcache, 4, NULL);
+
+    // Insert 4 items
+    has = fifo_string_cache_insert(&fcache, "alpha");
+    g_assert_false(has);
+    has = fifo_string_cache_insert(&fcache, "beta");
+    g_assert_false(has);
+    has = fifo_string_cache_insert(&fcache, "gamma");
+    g_assert_false(has);
+    has = fifo_string_cache_insert(&fcache, "delta");
+    g_assert_false(has);
+
+    // They should all be there
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "gamma");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "delta");
+    g_assert_true(has);
+
+    // Add a 5th item
+    has = fifo_string_cache_insert(&fcache, "epsilon");
+    g_assert_false(has);
+
+    // The first one should no longer be there
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_false(has); // FALSE
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "gamma");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "delta");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "epsilon");
+    g_assert_true(has);
+
+    // Add a 6th item
+    has = fifo_string_cache_insert(&fcache, "zeta");
+    g_assert_false(has);
+
+    // The first two should no longer be there
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_false(has); // FALSE
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_false(has); // FALSE
+    has = fifo_string_cache_contains(&fcache, "gamma");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "delta");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "epsilon");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "zeta");
+    g_assert_true(has);
+
+    fifo_string_cache_free(&fcache);
+}
+
+// Check a max_entries == 1, to ensure we don't have any mistakes
+// at that end of the range
+static void
+test_fifo_string_cache_03(void)
+{
+    fifo_string_cache_t fcache;
+    gboolean has;
+    fifo_string_cache_init(&fcache, 1, NULL);
+
+    // Insert
+    has = fifo_string_cache_insert(&fcache, "alpha");
+    g_assert_false(has);
+
+    // Check
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_true(has);
+
+    // Insert
+    has = fifo_string_cache_insert(&fcache, "beta");
+    g_assert_false(has);
+
+    // Check
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_false(has);
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_true(has);
+
+    // Insert
+    has = fifo_string_cache_insert(&fcache, "gamma");
+    g_assert_false(has);
+
+    // Check
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_false(has);
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_false(has);
+    has = fifo_string_cache_contains(&fcache, "gamma");
+    g_assert_true(has);
+
+    fifo_string_cache_free(&fcache);
+}
+
+// Test an unbounded maximum (max_entries == 0)
+static void
+test_fifo_string_cache_04(void)
+{
+    fifo_string_cache_t fcache;
+    gboolean has;
+    fifo_string_cache_init(&fcache, 0, g_free);
+
+    // Insert; we call g_strdup because in this test, the cache owns the string
+    has = fifo_string_cache_insert(&fcache, g_strdup("alpha"));
+    g_assert_false(has);
+
+    // Check
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_true(has);
+
+    // Insert; we call g_strdup because in this test, the cache owns the string
+    has = fifo_string_cache_insert(&fcache, g_strdup("beta"));
+    g_assert_false(has);
+
+    // Check
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_true(has);
+
+    // Insert many
+    int i;
+    char *s;
+    for (i = 0; i < 1000 ; i++) {
+        s = g_strdup_printf("%d", i);
+        has = fifo_string_cache_insert(&fcache, s);
+        g_assert_false(has);
+    }
+
+    // Check everything
+    has = fifo_string_cache_contains(&fcache, "alpha");
+    g_assert_true(has);
+    has = fifo_string_cache_contains(&fcache, "beta");
+    g_assert_true(has);
+    for (i = 0; i < 1000 ; i++) {
+        s = g_strdup_printf("%d", i);
+        has = fifo_string_cache_contains(&fcache, s);
+        g_assert_true(has);
+    }
+    fifo_string_cache_free(&fcache);
+}
+
+int
+main(int argc, char **argv)
+{
+    int result;
+
+    g_test_init(&argc, &argv, NULL);
+
+    g_test_add_func("/fifo_string_cache/01",    test_fifo_string_cache_01);
+    g_test_add_func("/fifo_string_cache/02",    test_fifo_string_cache_02);
+    g_test_add_func("/fifo_string_cache/03",    test_fifo_string_cache_03);
+    g_test_add_func("/fifo_string_cache/04",    test_fifo_string_cache_04);
+
+    result = g_test_run();
+
+    return result;
+}
+
+/*
+ * Editor modelines  -  https://www.wireshark.org/tools/modelines.html
+ *
+ * Local variables:
+ * c-basic-offset: 4
+ * tab-width: 8
+ * indent-tabs-mode: nil
+ * End:
+ *
+ * vi: set shiftwidth=4 tabstop=8 expandtab:
+ * :indentSize=4:tabSize=8:noTabs=true:
+ */
diff --git a/epan/prefs.c b/epan/prefs.c
index c17ba2ad4b..2ff177ec86 100644
--- a/epan/prefs.c
+++ b/epan/prefs.c
@@ -3768,6 +3768,17 @@ prefs_register_modules(void)
                                    "Currently ICMP and ICMPv6 use this preference to add VLAN ID to conversation tracking, and IPv4 uses this preference to take VLAN ID into account during reassembly",
                                    &prefs.strict_conversation_tracking_heuristics);
 
+    prefs_register_bool_preference(protocols_module, "ignore_dup_frames",
+                                   "Ignore duplicate frames",
+                                   "Ignore frames that are exact duplicates of any previous frame.",
+                                   &prefs.ignore_dup_frames);
+
+    prefs_register_uint_preference(protocols_module, "ignore_dup_frames_cache_entries",
+            "The max number of hashes to keep in memory for determining duplicates frames",
+            "If \"Ignore duplicate frames\" is set, this setting sets the maximum number "
+            "of cache entries to maintain. A 0 means no limit.",
+            10, &prefs.ignore_dup_frames_cache_entries);
+
     /* Obsolete preferences
      * These "modules" were reorganized/renamed to correspond to their GUI
      * configuration screen within the preferences dialog
@@ -4182,8 +4193,12 @@ pre_init_prefs(void)
     prefs.st_sort_defcolflag = ST_SORT_COL_COUNT;
     prefs.st_sort_defdescending = TRUE;
     prefs.st_sort_showfullname = FALSE;
+
+    /* protocols */
     prefs.display_hidden_proto_items = FALSE;
     prefs.display_byte_fields_with_spaces = FALSE;
+    prefs.ignore_dup_frames = FALSE;
+    prefs.ignore_dup_frames_cache_entries = 10000;
 
     /* set the default values for the io graph dialog */
     prefs.gui_io_graph_automatic_update = TRUE;
diff --git a/epan/prefs.h b/epan/prefs.h
index 79a652a0ca..50ea5fc03d 100644
--- a/epan/prefs.h
+++ b/epan/prefs.h
@@ -206,6 +206,8 @@ typedef struct _e_prefs {
   gboolean     enable_incomplete_dissectors_check;
   gboolean     incomplete_dissectors_check_debug;
   gboolean     strict_conversation_tracking_heuristics;
+  gboolean     ignore_dup_frames;
+  guint        ignore_dup_frames_cache_entries;
   gboolean     filter_expressions_old;  /* TRUE if old filter expressions preferences were loaded. */
   gboolean     gui_update_enabled;
   software_update_channel_e gui_update_channel;