wiretap: camins, vwr: Stop heuristics after 1GiB

Very large 64 bit files are supported, so the CAM Inspector and
Ixia Veriwave heuristics, which are fairly weak and either always
(CAM Inspector) or possibly (Veriwave) try to read the entire file
should stop their heuristics and make a decision after some reasonable
length.

Without this, the GUI freezes for seconds, minutes, or even hours
by merely clicking on a large file in the file chooser, as
wtap_open_offline attempts to determine the file type. The same issue
occurs in capinfos, captype, tshark, editcap, etc.

In addition, previously the CAM Inspector heuristics could give the wrong
result on very large files, because 10 * invalid_pairs could overflow
its guint32 and then end up comparing as less than valid_pairs.

Fix #17620

2 files changed, 25 insertions, 3 deletions

diff --git a/wiretap/camins.c b/wiretap/camins.c
index 5d9680e3a9..f26b5910a5 100644
--- a/wiretap/camins.c
+++ b/wiretap/camins.c
@@ -106,6 +106,9 @@ typedef enum {
 #define DVB_CI_PSEUDO_HDR_CAM_TO_HOST 0xFF
 #define DVB_CI_PSEUDO_HDR_HOST_TO_CAM 0xFE
 
+/* Maximum number of bytes to read before making a heuristic decision
+ * of whether this is our file type or not. Arbitrary. */
+#define CAMINS_BYTES_TO_CHECK 0x3FFFFFFFU
 
 static int camins_file_type_subtype = -1;
 
@@ -115,8 +118,8 @@ void register_camins(void);
    size register. The matching blocks to access the upper and lower 8bit
    must be no further than 5 blocks apart.
    A file may have errors that affect the size blocks. Therefore, we
-   read the entire file and require that we have much more valid pairs
-   than errors. */
+   read CAMINS_BYTES_TO_CHECK bytes and require that we have many more
+   valid pairs than errors. */
 static wtap_open_return_val detect_camins_file(FILE_T fh)
 {
     int      err;
@@ -125,6 +128,7 @@ static wtap_open_return_val detect_camins_file(FILE_T fh)
     guint8   search_block = 0;
     guint8   gap_count = 0;
     guint32  valid_pairs = 0, invalid_pairs = 0;
+    guint64  read_bytes = 0;
 
     while (wtap_read_bytes(fh, block, sizeof(block), &err, &err_info)) {
        if (search_block != 0) {
@@ -167,9 +171,14 @@ static wtap_open_return_val detect_camins_file(FILE_T fh)
                 gap_count = 0;
             }
         }
+        read_bytes += sizeof(block);
+        if (read_bytes > CAMINS_BYTES_TO_CHECK) {
+            err = 0;
+            break;
+        }
     }
 
-    if (err != WTAP_ERR_SHORT_READ) {
+    if ((err != 0) && (err != WTAP_ERR_SHORT_READ)) {
         /* A real read error. */
         return WTAP_OPEN_ERROR;
     }
diff --git a/wiretap/vwr.c b/wiretap/vwr.c
index 7ecf018150..0620074802 100644
--- a/wiretap/vwr.c
+++ b/wiretap/vwr.c
@@ -68,6 +68,12 @@
  */
 #define VW_RECORD_HEADER_LENGTH 16
 
+/*
+ * Maximum number of bytes to read looking for a valid frame starting
+ * with a command byte to determine if this is our file type. Arbitrary.
+ */
+#define VW_BYTES_TO_CHECK 0x3FFFFFFFU
+
 /* Command byte values */
 #define COMMAND_RX   0x21
 #define COMMAND_TX   0x31
@@ -967,6 +973,7 @@ static int vwr_get_fpga_version(wtap *wth, int *err, gchar **err_info)
     guint8  *s_510024_ptr = NULL;
     guint8  *s_510012_ptr = NULL; /* stats pointers */
     gint64   filePos      = -1;
+    guint64  bytes_read   = 0;
     guint32  frame_type   = 0;
     int      f_len, v_type;
     guint16  data_length  = 0;
@@ -1104,6 +1111,12 @@ static int vwr_get_fpga_version(wtap *wth, int *err, gchar **err_info)
                 }
             }
         }
+        bytes_read += VW_RECORD_HEADER_LENGTH;
+        if (bytes_read > VW_BYTES_TO_CHECK) {
+            /* no frame found in VW_BYTES_TO_CHECK - not a vwr file */
+            g_free(rec);
+            return UNKNOWN_FPGA;
+        }
     }
 
     /* Is this a valid but empty file?  If so, claim it's the S3_W_FPGA FPGA. */