Improve file merging for mergecap and wireshark

Refactor the file merging code by removing the duplicate logic from mergecap.c
and file.c's cf_merge_files(), into a new merge_files() function in merge.c.
Also the following user-visible changes:

 * Removed the '-T' encap type option from mergecap, as it's illogical for
   mergecap and would complicate common merge code.

 * Input files with IDBs of different name, speed, tsprecision, etc., will produce
   an output PCAPNG file with separate IDBs, even if their encap types are the same.

 * Added a '-I' IDB merge mode option for mergecap, to control how IDBs are merged.

 * Changed Wireshark's drag-and-drop merging to use PCAPNG instead of PCAP.

Bug: 8795
Bug: 7381
Change-Id: Icc30d217e093d6f40114422204afd2e332834f71
Reviewed-on: https://code.wireshark.org/review/10058
Petri-Dish: Hadriel Kaplan <hadrielk@yahoo.com>
Tested-by: Petri Dish Buildbot <buildbot-no-reply@wireshark.org>
Reviewed-by: Anders Broman <a.broman58@gmail.com>

4 files changed, 840 insertions, 9 deletions

diff --git a/wiretap/merge.c b/wiretap/merge.c
index 15e7b4183b..1b2c6f85d0 100644
--- a/wiretap/merge.c
+++ b/wiretap/merge.c
@@ -36,11 +36,76 @@
 #include <string.h>
 #include "merge.h"
 
+#include <wsutil/filesystem.h>
+#include "wsutil/os_version_info.h"
+
+
+static const char* idb_merge_mode_strings[] = {
+    /* IDB_MERGE_MODE_NONE */
+    "none",
+    /* IDB_MERGE_MODE_ALL_SAME */
+    "all",
+    /* IDB_MERGE_MODE_ANY_SAME */
+    "any",
+    /* IDB_MERGE_MODE_MAX */
+    "UNKNOWN"
+};
+
+idb_merge_mode
+merge_string_to_idb_merge_mode(const char *name)
+{
+    int i;
+    for (i = 0; i < IDB_MERGE_MODE_MAX; i++) {
+        if (g_strcmp0(name, idb_merge_mode_strings[i]) == 0) {
+            return (idb_merge_mode) i;
+        }
+    }
+    return IDB_MERGE_MODE_MAX;
+}
+
+const char *
+merge_idb_merge_mode_to_string(const int mode)
+{
+    if (mode >= 0 && mode < IDB_MERGE_MODE_MAX) {
+        return idb_merge_mode_strings[mode];
+    }
+    return idb_merge_mode_strings[(int)IDB_MERGE_MODE_MAX];
+}
+
+
+static void
+cleanup_in_file(merge_in_file_t *in_file)
+{
+    g_assert(in_file != NULL);
+
+    wtap_close(in_file->wth);
+    in_file->wth = NULL;
+
+    g_array_free(in_file->idb_index_map, TRUE);
+    in_file->idb_index_map = NULL;
+}
+
+static void
+add_idb_index_map(merge_in_file_t *in_file, const guint orig_index, const guint found_index)
+{
+    g_assert(in_file != NULL);
+    g_assert(in_file->idb_index_map != NULL);
+
+    /*
+     * we didn't really need the orig_index, since just appending to the array
+     * should result in the orig_index being its location in the array; but we
+     * pass it into this function to do a sanity check here
+     */
+    g_assert(orig_index == in_file->idb_index_map->len);
+
+    g_array_append_val(in_file->idb_index_map, found_index);
+}
+
 /*
  * Scan through the arguments and open the input files
  */
 gboolean
-merge_open_in_files(int in_file_count, char *const *in_file_names,
+merge_open_in_files(int in_file_count, const char *const *in_file_names,
                     merge_in_file_t **in_files, int *err, gchar **err_info,
                     int *err_fileno)
 {
@@ -50,7 +115,7 @@ merge_open_in_files(int in_file_count, char *const *in_file_names,
     merge_in_file_t *files;
     gint64 size;
 
-    files = (merge_in_file_t *)g_malloc(files_size);
+    files = (merge_in_file_t *)g_malloc0(files_size);
     *in_files = files;
 
     for (i = 0; i < in_file_count; i++) {
@@ -59,21 +124,23 @@ merge_open_in_files(int in_file_count, char *const *in_file_names,
         files[i].data_offset = 0;
         files[i].state       = PACKET_NOT_PRESENT;
         files[i].packet_num  = 0;
+
         if (!files[i].wth) {
             /* Close the files we've already opened. */
             for (j = 0; j < i; j++)
-                wtap_close(files[j].wth);
+                cleanup_in_file(&files[j]);
             *err_fileno = i;
             return FALSE;
         }
         size = wtap_file_size(files[i].wth, err);
         if (size == -1) {
             for (j = 0; j + 1 > j && j <= i; j++)
-                wtap_close(files[j].wth);
+                cleanup_in_file(&files[j]);
             *err_fileno = i;
             return FALSE;
         }
         files[i].size = size;
+        files[i].idb_index_map = g_array_new(FALSE, FALSE, sizeof(guint));
     }
     return TRUE;
 }
@@ -86,7 +153,7 @@ merge_close_in_files(int count, merge_in_file_t in_files[])
 {
     int i;
     for (i = 0; i < count; i++) {
-        wtap_close(in_files[i].wth);
+        cleanup_in_file(&in_files[i]);
     }
 }
 
@@ -277,7 +344,638 @@ merge_append_read_packet(int in_file_count, merge_in_file_t in_files[],
      */
     *err = 0;
     return &in_files[i];
-  }
+}
+
+
+/* creates a section header block for the new output file */
+static wtapng_section_t*
+create_shb_header(const merge_in_file_t *in_files, const guint in_file_count,
+                  const gchar *app_name)
+{
+    wtapng_section_t *shb_hdr = NULL;
+    GString *comment_gstr;
+    GString *os_info_str;
+    guint i;
+
+    shb_hdr = wtap_file_get_shb_for_new_file(in_files[0].wth);
+
+    comment_gstr = g_string_new("");
+
+    /* TODO: merge comments from all files */
+
+    /* very lame way to save comments - does not save them from the other files */
+    if (shb_hdr->opt_comment && strlen(shb_hdr->opt_comment) > 0) {
+        g_string_append_printf(comment_gstr, "%s \n",shb_hdr->opt_comment);
+    }
+    g_free(shb_hdr->opt_comment);
+    shb_hdr->opt_comment = NULL;
+
+    g_string_append_printf(comment_gstr, "File created by merging: \n");
+
+    for (i = 0; i < in_file_count; i++) {
+        g_string_append_printf(comment_gstr, "File%d: %s \n",i+1,in_files[i].filename);
+    }
+
+    os_info_str = g_string_new("");
+    get_os_version_info(os_info_str);
+
+    shb_hdr->section_length = -1;
+    /* TODO: handle comments from each file being merged */
+    shb_hdr->opt_comment   = g_string_free(comment_gstr, FALSE);  /* section comment */
+    shb_hdr->shb_hardware  = NULL;        /* NULL if not available, UTF-8 string containing the        */
+                                          /*  description of the hardware used to create this section. */
+    shb_hdr->shb_os        = g_string_free(os_info_str, FALSE); /* UTF-8 string containing the name   */
+                                          /*  of the operating system used to create this section.     */
+    shb_hdr->shb_user_appl = g_strdup(app_name); /* NULL if not available, UTF-8 string containing the name */
+                                          /*  of the application used to create this section.          */
+
+    return shb_hdr;
+}
+
+static gboolean
+is_duplicate_idb(const wtapng_if_descr_t *idb1, const wtapng_if_descr_t *idb2)
+{
+    g_assert(idb1 && idb2);
+
+    /* does not compare filters nor interface statistics */
+    return (idb1->wtap_encap == idb2->wtap_encap &&
+            idb1->time_units_per_second == idb2->time_units_per_second &&
+            idb1->tsprecision == idb2->tsprecision &&
+            idb1->link_type == idb2->link_type &&
+            /* XXX: should snaplen not be compared? */
+            idb1->snap_len == idb2->snap_len &&
+            idb1->if_speed == idb2->if_speed &&
+            idb1->if_tsresol == idb2->if_tsresol &&
+            idb1->if_fcslen == idb2->if_fcslen &&
+            g_strcmp0(idb1->opt_comment, idb2->opt_comment) == 0 &&
+            g_strcmp0(idb1->if_name, idb2->if_name) == 0 &&
+            g_strcmp0(idb1->if_description, idb2->if_description) == 0 &&
+            g_strcmp0(idb1->if_os, idb2->if_os) == 0);
+}
+
+/*
+ * Returns true if all of the input files have duplicate IDBs to the other files.
+ */
+static gboolean
+all_idbs_are_duplicates(const merge_in_file_t *in_files, const guint in_file_count)
+{
+    const wtapng_iface_descriptions_t *first_idb_list = NULL;
+    const wtapng_iface_descriptions_t *other_idb_list = NULL;
+    guint first_idb_list_size, other_idb_list_size;
+    const wtapng_if_descr_t *first_file_idb, *other_file_idb;
+    guint i, j;
+
+    g_assert(in_files != NULL);
+
+    /* get the first file's info */
+    first_idb_list = wtap_file_get_idb_info(in_files[0].wth);
+    g_assert(first_idb_list->interface_data);
+
+    first_idb_list_size = first_idb_list->interface_data->len;
+
+    /* now compare the other input files with that */
+    for (i = 1; i < in_file_count; i++) {
+        other_idb_list = wtap_file_get_idb_info(in_files[i].wth);
+        g_assert(other_idb_list->interface_data);
+        other_idb_list_size = other_idb_list->interface_data->len;
+
+        if (other_idb_list_size != first_idb_list_size) {
+            return FALSE;
+        }
+
+        for (j = 0; j < other_idb_list_size; j++) {
+            first_file_idb = &g_array_index(first_idb_list->interface_data, wtapng_if_descr_t, j);
+            other_file_idb = &g_array_index(other_idb_list->interface_data, wtapng_if_descr_t, j);
+
+            if (!is_duplicate_idb(first_file_idb, other_file_idb)) {
+                return FALSE;
+            }
+        }
+    }
+
+    return TRUE;
+}
+
+/*
+ * Returns true if the given input_file_idb is a duplicate of an existing one
+ * in the merged_idb_list; it's a duplicate if the interface description data
+ * is all identical to a previous one in another input file. For this
+ * function, the input file IDB's index does NOT need to match the index
+ * location of a previous one to be considered a duplicate; any match is
+ * considered a success. That means it will even match another IDB from its
+ * own (same) input file.
+ */
+static gboolean
+find_duplicate_idb(const wtapng_if_descr_t *input_file_idb,
+               const wtapng_iface_descriptions_t *merged_idb_list,
+               guint *found_index)
+{
+    const wtapng_if_descr_t *merged_idb;
+    guint i;
+
+    g_assert(input_file_idb != NULL);
+    g_assert(merged_idb_list != NULL);
+    g_assert(merged_idb_list->interface_data != NULL);
+    g_assert(found_index != NULL);
+
+    for (i = 0; i < merged_idb_list->interface_data->len; i++) {
+        merged_idb = &g_array_index(merged_idb_list->interface_data, wtapng_if_descr_t, i);
+
+        if (is_duplicate_idb(input_file_idb, merged_idb)) {
+            *found_index = i;
+            return TRUE;
+        }
+    }
+
+    return FALSE;
+}
+
+/* adds IDB to merged file info, returns its index */
+static guint
+add_idb_to_merged_file(wtapng_iface_descriptions_t *merged_idb_list,
+                       const wtapng_if_descr_t *input_file_idb)
+{
+    wtapng_if_descr_t idb;
+
+    g_assert(merged_idb_list != NULL);
+    g_assert(merged_idb_list->interface_data != NULL);
+    g_assert(input_file_idb != NULL);
+
+    idb.wtap_encap            = input_file_idb->wtap_encap;
+    idb.time_units_per_second = input_file_idb->time_units_per_second;
+    idb.tsprecision           = input_file_idb->tsprecision;
+    idb.link_type             = input_file_idb->link_type;
+    idb.snap_len              = input_file_idb->snap_len;
+    idb.if_name               = g_strdup(input_file_idb->if_name);
+    idb.opt_comment           = g_strdup(input_file_idb->opt_comment);;
+    idb.if_description        = g_strdup(input_file_idb->if_description);
+    idb.if_speed              = input_file_idb->if_speed;
+    idb.if_tsresol            = input_file_idb->if_tsresol;
+    idb.if_filter_str         = NULL;
+    idb.bpf_filter_len        = 0;
+    idb.if_filter_bpf_bytes   = NULL;
+    idb.if_os                 = g_strdup(input_file_idb->if_os);
+    idb.if_fcslen             = input_file_idb->if_fcslen;
+    idb.num_stat_entries      = 0;          /* Number of ISB:s */
+    idb.interface_statistics  = NULL;
+
+    g_array_append_val(merged_idb_list->interface_data, idb);
+
+    return merged_idb_list->interface_data->len - 1;
+}
+
+/*
+ * Create clone IDBs for the merge file, based on the input files and mode.
+ */
+static wtapng_iface_descriptions_t *
+generate_merged_idb(merge_in_file_t *in_files, const guint in_file_count, const idb_merge_mode mode)
+{
+    wtapng_iface_descriptions_t *merged_idb_list = NULL;
+    wtapng_iface_descriptions_t *input_file_idb_list = NULL;
+    const wtapng_if_descr_t     *input_file_idb = NULL;
+    guint                        itf_count, merged_index;
+    guint                        i;
+
+    /* create new IDB info */
+    merged_idb_list = g_new(wtapng_iface_descriptions_t,1);
+    merged_idb_list->interface_data = g_array_new(FALSE, FALSE, sizeof(wtapng_if_descr_t));
+
+    if (mode == IDB_MERGE_MODE_ALL_SAME && all_idbs_are_duplicates(in_files, in_file_count)) {
+        guint num_idbs;
+        /* they're all the same, so just get the first file's IDBs */
+        input_file_idb_list = wtap_file_get_idb_info(in_files[0].wth);
+        /* this is really one more than number of IDBs, but that's good for the for-loops */
+        num_idbs = input_file_idb_list->interface_data->len;
+
+        /* put them in the merged file */
+        for (itf_count = 0; itf_count < num_idbs; itf_count++) {
+            input_file_idb = &g_array_index(input_file_idb_list->interface_data,
+                                            wtapng_if_descr_t, itf_count);
+            merged_index = add_idb_to_merged_file(merged_idb_list, input_file_idb);
+            add_idb_index_map(&in_files[0], itf_count, merged_index);
+        }
+
+        /* and set all the other file index maps the same way */
+        for (i = 1; i < in_file_count; i++) {
+            for (itf_count = 0; itf_count < num_idbs; itf_count++) {
+                add_idb_index_map(&in_files[i], itf_count, itf_count);
+            }
+        }
+
+        g_free(input_file_idb_list);
+    }
+    else {
+        for (i = 0; i < in_file_count; i++) {
+            input_file_idb_list = wtap_file_get_idb_info(in_files[i].wth);
+
+            for (itf_count = 0; itf_count < input_file_idb_list->interface_data->len; itf_count++) {
+                input_file_idb = &g_array_index(input_file_idb_list->interface_data,
+                                                wtapng_if_descr_t, itf_count);
+
+                if (mode == IDB_MERGE_MODE_ANY_SAME &&
+                    find_duplicate_idb(input_file_idb, merged_idb_list, &merged_index))
+                {
+                    /*
+                     * It's the same as a previous IDB, so we're going to "merge"
+                     * them into one by adding a map from its old IDB index to the new
+                     * one. This will be used later to change the phdr interface_id.
+                     */
+                    add_idb_index_map(&in_files[i], itf_count, merged_index);
+                }
+                else {
+                    /*
+                     * This IDB does not match a previous (or we want to save all IDBs),
+                     * so add the IDB to the merge file, and add a map of the indeces.
+                     */
+                    merged_index = add_idb_to_merged_file(merged_idb_list, input_file_idb);
+                    add_idb_index_map(&in_files[i], itf_count, merged_index);
+                }
+            }
+
+            g_free(input_file_idb_list);
+        }
+    }
+
+    return merged_idb_list;
+}
+
+static gboolean
+map_phdr_interface_id(struct wtap_pkthdr *phdr, const merge_in_file_t *in_file)
+{
+    guint current_interface_id = 0;
+    g_assert(phdr != NULL);
+    g_assert(in_file != NULL);
+    g_assert(in_file->idb_index_map != NULL);
+
+    if (phdr->presence_flags & WTAP_HAS_INTERFACE_ID) {
+        current_interface_id = phdr->interface_id;
+    }
+
+    if (current_interface_id >= in_file->idb_index_map->len) {
+        /* this shouldn't happen, but in a malformed input file it could */
+        return FALSE;
+    }
+
+    phdr->interface_id = g_array_index(in_file->idb_index_map, guint, current_interface_id);
+    phdr->presence_flags |= WTAP_HAS_INTERFACE_ID;
+
+    return TRUE;
+}
+
+static gchar*
+get_read_error_string(const merge_in_file_t *in_files, const guint in_file_count,
+                      const int *err, gchar **err_info)
+{
+    GString *err_message = g_string_new("");
+    gchar   *display_basename = NULL;
+    guint    i;
+
+    g_assert(in_files != NULL);
+    g_assert(err != NULL);
+    g_assert(err_info != NULL);
+
+    if (*err_info == NULL) {
+        *err_info = g_strdup("no information supplied");
+    }
+
+    /*
+     * Find the file on which we got the error, and report the error.
+     */
+    for (i = 0; i < in_file_count; i++) {
+        if (in_files[i].state == GOT_ERROR) {
+            display_basename = g_filename_display_basename(in_files[i].filename);
+
+            switch (*err) {
+
+                case WTAP_ERR_SHORT_READ:
+                    g_string_printf(err_message,
+                         "The capture file %s appears to have been cut short"
+                          " in the middle of a packet.", display_basename);
+                    break;
+
+                case WTAP_ERR_BAD_FILE:
+                    g_string_printf(err_message,
+                         "The capture file %s appears to be damaged or corrupt.\n(%s)",
+                         display_basename, *err_info);
+                    break;
+
+                case WTAP_ERR_DECOMPRESS:
+                    g_string_printf(err_message,
+                         "The compressed capture file %s appears to be damaged or corrupt.\n"
+                         "(%s)", display_basename, *err_info);
+                    break;
+
+                default:
+                    g_string_printf(err_message,
+                         "An error occurred while reading the"
+                         " capture file %s: %s.",
+                         display_basename,  wtap_strerror(*err));
+                    break;
+            }
+
+            g_free(display_basename);
+            break;
+        }
+    }
+
+    g_free(*err_info);
+    *err_info = g_string_free(err_message, FALSE);
+
+    return *err_info;
+}
+
+static gchar*
+get_write_error_string(const merge_in_file_t *in_file, const int file_type,
+                       const gchar* out_filename, const int *err, gchar **err_info)
+{
+    GString *err_message = g_string_new("");
+    gchar *display_basename = NULL;
+    int write_err;
+
+    /* in_file may be NULL */
+    g_assert(err != NULL);
+    g_assert(err_info != NULL);
+
+    if (*err_info == NULL) {
+        *err_info = g_strdup("no information supplied");
+    }
+
+    write_err = *err;
+
+    display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
+
+    if (write_err < 0) {
+
+        switch (write_err) {
+
+            case WTAP_ERR_UNWRITABLE_ENCAP:
+                /*
+                 * This is a problem with the particular frame we're writing and
+                 * the file type and subtype we're wwriting; note that, and
+                 * report the frame number and file type/subtype.
+                 */
+                g_string_printf(err_message,
+                    "Frame %u of \"%s\" has a network type that can't be saved in a \"%s\" file.\n",
+                    in_file ? in_file->packet_num : 0, display_basename,
+                    wtap_file_type_subtype_string(file_type));
+                break;
+
+            case WTAP_ERR_PACKET_TOO_LARGE:
+                /*
+                 * This is a problem with the particular frame we're writing and
+                 * the file type and subtype we're writing; note that, and report
+                 * the frame number and file type/subtype.
+                 */
+                g_string_printf(err_message,
+                    "Frame %u of \"%s\" is too large for a \"%s\" file.",
+                    in_file ? in_file->packet_num : 0, display_basename,
+                    wtap_file_type_subtype_string(file_type));
+                break;
+
+            case WTAP_ERR_UNWRITABLE_REC_TYPE:
+                /*
+                 * This is a problem with the particular record we're writing and
+                 * the file type and subtype we're writing; note that, and report
+                 * the record number and file type/subtype.
+                 */
+                g_string_printf(err_message,
+                    "Record %u of \"%s\" has a record type that can't be saved in a \"%s\" file.",
+                    in_file ? in_file->packet_num : 0, display_basename,
+                    wtap_file_type_subtype_string(file_type));
+                break;
+
+            case WTAP_ERR_UNWRITABLE_REC_DATA:
+                /*
+                 * This is a problem with the particular record we're writing and
+                 * the file type and subtype we're writing; note that, and report
+                 * the frame number and file type/subtype.
+                 */
+                g_string_printf(err_message,
+                    "Record %u of \"%s\" has data that can't be saved in a \"%s\" file.\n(%s)",
+                    in_file ? in_file->packet_num : 0, display_basename,
+                    wtap_file_type_subtype_string(file_type), *err_info);
+                break;
+
+            default:
+                g_string_printf(err_message,
+                    "An error occurred while writing to the file \"%s\": %s.",
+                    out_filename, wtap_strerror(write_err));
+                break;
+        }
+    }
+    else {
+        /* OS error. */
+        g_string_printf(err_message, file_write_error_message(write_err), out_filename);
+    }
+
+    g_free(display_basename);
+    g_free(*err_info);
+    *err_info = g_string_free(err_message, FALSE);
+
+    return *err_info;
+}
+
+
+/*
+ * Merges the files base don given input, and invokes callback during
+ * execution. Returns MERGE_OK on success, or a MERGE_ERR_XXX on failure; note
+ * that the passed-in 'err' variable will be more specific to what failed, and
+ * err_info will have pretty output.
+ */
+merge_result
+merge_files(int out_fd, const gchar* out_filename, const int file_type,
+            const char *const *in_filenames, const guint in_file_count,
+            const gboolean do_append, const idb_merge_mode mode,
+            guint snaplen, const gchar *app_name, merge_progress_callback_t* cb,
+            int *err, gchar **err_info, int *err_fileno)
+{
+    merge_in_file_t    *in_files = NULL, *in_file = NULL;
+    int                 frame_type = WTAP_ENCAP_PER_PACKET;
+    merge_result        status = MERGE_OK;
+    wtap_dumper        *pdh;
+    struct wtap_pkthdr *phdr, snap_phdr;
+    int                 count = 0;
+    gboolean            stop_flag = FALSE;
+
+    g_assert(out_fd > 0);
+    g_assert(in_file_count > 0);
+    g_assert(in_filenames != NULL);
+    g_assert(err != NULL);
+    g_assert(err_info != NULL);
+    g_assert(err_fileno != NULL);
+
+    /* if a callback was given, it has to have a callback function ptr */
+    g_assert((cb != NULL) ? (cb->callback_func != NULL) : TRUE);
+
+    /* open the input files */
+    if (!merge_open_in_files(in_file_count, in_filenames, &in_files,
+                             err, err_info, err_fileno)) {
+        return MERGE_ERR_CANT_OPEN_INFILE;
+    }
+
+    if (cb)
+        cb->callback_func(MERGE_EVENT_INPUT_FILES_OPENED, 0, in_files, in_file_count, cb->data);
+
+    if (snaplen == 0) {
+        /* Snapshot length not specified - default to the maximum. */
+        snaplen = WTAP_MAX_PACKET_SIZE;
+    }
+
+    /*
+     * This doesn't tell us that much. It tells us what to set the outfile's
+     * encap type to, but that's all - for example, it does *not* tells us
+     * whether the input files had the same number of IDBs, for the same exact
+     * interfaces, and only one IDB each, so it doesn't actually tell us
+     * whether we can merge IDBs into one or not.
+     */
+    frame_type = merge_select_frame_type(in_file_count, in_files);
+
+    if (cb)
+        cb->callback_func(MERGE_EVENT_FRAME_TYPE_SELECTED, frame_type, in_files, in_file_count, cb->data);
+
+    /* prepare the outfile */
+    if (file_type == WTAP_FILE_TYPE_SUBTYPE_PCAPNG) {
+        wtapng_section_t            *shb_hdr = NULL;
+        wtapng_iface_descriptions_t *idb_inf = NULL;
+
+        shb_hdr = create_shb_header(in_files, in_file_count, app_name);
+
+        idb_inf = generate_merged_idb(in_files, in_file_count, mode);
+
+        pdh = wtap_dump_fdopen_ng(out_fd, file_type, frame_type, snaplen,
+                                  FALSE /* compressed */, shb_hdr, idb_inf,
+                                  NULL, err);
+    }
+    else {
+        pdh = wtap_dump_fdopen(out_fd, file_type, frame_type, snaplen, FALSE /* compressed */, err);
+    }
+
+    if (pdh == NULL) {
+        merge_close_in_files(in_file_count, in_files);
+        g_free(in_files);
+        return MERGE_ERR_CANT_OPEN_OUTFILE;
+    }
+
+    if (cb)
+        cb->callback_func(MERGE_EVENT_READY_TO_MERGE, 0, in_files, in_file_count, cb->data);
+
+    for (;;) {
+        *err = 0;
+
+        if (do_append) {
+            in_file = merge_append_read_packet(in_file_count, in_files, err,
+                                               err_info);
+        }
+        else {
+            in_file = merge_read_packet(in_file_count, in_files, err,
+                                        err_info);
+        }
+
+        if (in_file == NULL) {
+            /* EOF */
+            break;
+        }
+
+        if (*err != 0) {
+            /* I/O error reading from in_file */
+            status = MERGE_ERR_CANT_READ_INFILE;
+            break;
+        }
+
+        count++;
+        if (cb)
+            stop_flag = cb->callback_func(MERGE_EVENT_PACKET_WAS_READ, count, in_files, in_file_count, cb->data);
+
+        if (stop_flag) {
+            /* The user decided to abort the merge. */
+            status = MERGE_USER_ABORTED;
+            break;
+        }
+
+        phdr = wtap_phdr(in_file->wth);
+
+        if (snaplen != 0 && phdr->caplen > snaplen) {
+            /*
+             * The dumper will only write up to caplen bytes out, so we only
+             * need to change that value, instead of cloning the whole packet
+             * with fewer bytes.
+             *
+             * XXX: but do we need to change the IDBs' snap_len?
+             */
+            snap_phdr = *phdr;
+            snap_phdr.caplen = snaplen;
+            phdr = &snap_phdr;
+        }
+
+        if (file_type == WTAP_FILE_TYPE_SUBTYPE_PCAPNG) {
+            if (!map_phdr_interface_id(phdr, in_file)) {
+                status = MERGE_ERR_BAD_PHDR_INTERFACE_ID;
+                break;
+            }
+        }
+
+        if (!wtap_dump(pdh, phdr, wtap_buf_ptr(in_file->wth), err, err_info)) {
+            status = MERGE_ERR_CANT_WRITE_OUTFILE;
+            break;
+        }
+    }
+
+    if (cb)
+        cb->callback_func(MERGE_EVENT_DONE, count, in_files, in_file_count, cb->data);
+
+    merge_close_in_files(in_file_count, in_files);
+
+    if (status == MERGE_OK || status == MERGE_USER_ABORTED) {
+        if (!wtap_dump_close(pdh, err))
+            status = MERGE_ERR_CANT_CLOSE_OUTFILE;
+    } else {
+        /*
+         * We already got some error; no need to report another error on
+         * close.
+         *
+         * Don't overwrite the earlier error.
+         */
+        int close_err = 0;
+        (void)wtap_dump_close(pdh, &close_err);
+    }
+
+    if (status != MERGE_OK) {
+        GString *err_message = NULL;
+        gchar   *display_basename = NULL;
+
+        switch(status) {
+
+            case MERGE_ERR_CANT_READ_INFILE:
+                *err_info = get_read_error_string(in_files, in_file_count, err, err_info);
+                break;
+
+            case MERGE_ERR_CANT_WRITE_OUTFILE: /* fall through */
+            case MERGE_ERR_CANT_CLOSE_OUTFILE:
+                *err_info = get_write_error_string(in_file, file_type, out_filename, err, err_info);
+                break;
+
+            case MERGE_ERR_BAD_PHDR_INTERFACE_ID:
+                display_basename = g_filename_display_basename(in_file ? in_file->filename : "UNKNOWN");
+                if (*err_info != NULL)
+                    g_free(*err_info);
+                err_message = g_string_new("");
+                g_string_printf(err_message,
+                    "Record %u of \"%s\" has an interface ID which does not match any IDB in its file.",
+                    in_file ? in_file->packet_num : 0, display_basename);
+                g_free(display_basename);
+                *err_info = g_string_free(err_message, FALSE);
+                break;
+
+            case MERGE_USER_ABORTED: /* not really an error */
+            default:
+                break;
+        }
+    }
+
+    g_free(in_files);
+
+    return status;
+}
 
 /*
  * Editor modelines  -  http://www.wireshark.org/tools/modelines.html
diff --git a/wiretap/merge.h b/wiretap/merge.h
index d78275248e..69162191bb 100644
--- a/wiretap/merge.h
+++ b/wiretap/merge.h
@@ -46,8 +46,7 @@ typedef struct merge_in_file_s {
     in_file_state_e state;
     guint32         packet_num;     /* current packet number */
     gint64          size;           /* file size */
-    guint32         interface_id;   /* identifier of the interface.
-                                     * Used for fake interfaces when writing WTAP_ENCAP_PER_PACKET */
+    GArray         *idb_index_map;  /* used for mapping the old phdr interface_id values to new during merge */
 } merge_in_file_t;
 
 /** Open a number of input files to merge.
@@ -61,7 +60,7 @@ typedef struct merge_in_file_s {
  * @return TRUE if all files could be opened, FALSE otherwise
  */
 WS_DLL_PUBLIC gboolean
-merge_open_in_files(int in_file_count, char *const *in_file_names,
+merge_open_in_files(int in_file_count, const char *const *in_file_names,
                     merge_in_file_t **in_files, int *err, gchar **err_info,
                     int *err_fileno);
 
@@ -120,6 +119,104 @@ WS_DLL_PUBLIC merge_in_file_t *
 merge_append_read_packet(int in_file_count, merge_in_file_t in_files[],
                          int *err, gchar **err_info);
 
+
+/** Return values from merge_files(). */
+typedef enum {
+    MERGE_OK,
+    MERGE_USER_ABORTED,
+    /* below here are true errors */
+    MERGE_ERR_CANT_OPEN_INFILE,
+    MERGE_ERR_CANT_OPEN_OUTFILE,
+    MERGE_ERR_CANT_READ_INFILE,
+    MERGE_ERR_BAD_PHDR_INTERFACE_ID,
+    MERGE_ERR_CANT_WRITE_OUTFILE,
+    MERGE_ERR_CANT_CLOSE_OUTFILE
+} merge_result;
+
+
+/** Merge events, used as an arg in the callback function - indicates when the callback was invoked. */
+typedef enum {
+    MERGE_EVENT_INPUT_FILES_OPENED,
+    MERGE_EVENT_FRAME_TYPE_SELECTED,
+    MERGE_EVENT_READY_TO_MERGE,
+    MERGE_EVENT_PACKET_WAS_READ,
+    MERGE_EVENT_DONE
+} merge_event;
+
+
+/** Merge mode for IDB info. */
+typedef enum {
+    IDB_MERGE_MODE_NONE = 0,    /**< no merging of IDBs is done, all IDBs are copied into merged file */
+    IDB_MERGE_MODE_ALL_SAME,/**< duplicate IDBs merged only if all the files have the same set of IDBs */
+    IDB_MERGE_MODE_ANY_SAME, /**< any and all duplicate IDBs are merged into one IDB, even within a file */
+    IDB_MERGE_MODE_MAX
+} idb_merge_mode;
+
+
+/** Returns the idb_merge_mode for the given string name.
+ *
+ * @param name The name of the mode.
+ * @return The idb_merge_mode, or IDB_MERGE_MODE_MAX on failure.
+ */
+WS_DLL_PUBLIC idb_merge_mode
+merge_string_to_idb_merge_mode(const char *name);
+
+
+/** Returns the string name for the given number.
+ *
+ * @param mode The number of the mode, representing the idb_merge_mode enum value.
+ * @return The string name, or "UNKNOWN" on failure.
+ */
+WS_DLL_PUBLIC const char*
+merge_idb_merge_mode_to_string(const int mode);
+
+
+/** @struct merge_progress_callback_t
+ *
+ * @brief Callback information for merging.
+ *
+ * @details The merge_files() routine can invoke a callback during its execution,
+ * to enable verbose printing or progress bar updating, for example. This struct
+ * provides merge_files() with the callback routine to invoke, and optionally
+ * private data to pass through to the callback each time it is invoked.
+ * For the callback_func routine's arguments: the event is when the callback
+ * was invoked, the num is an int specific to the event, in_files is an array
+ * of the created merge info, in_file_count is the size of the array, data is
+ * whatever was passed in the data member of this struct. The callback_func
+ * routine's return value should be TRUE if merging should be aborted.
+ */
+typedef struct {
+    gboolean (*callback_func)(merge_event event, int num,
+                              const merge_in_file_t in_files[], const guint in_file_count,
+                              void *data);
+    void *data; /**< private data to use for passing through to the callback function */
+} merge_progress_callback_t;
+
+
+/** Merge the given input files to the output file descriptor.
+ *
+ * @param out_fd The already opened output file decriptor
+ * @param out_filename The output filename, used in error messages
+ * @param file_type The WTAP_FILE_TYPE_SUBTYPE_XXX output file type
+ * @param in_filenames An array of input filenames to merge from
+ * @param in_file_count The number of entries in in_filenames
+ * @param do_append Whether to append by file order instead of chronological order
+ * @param mode The IDB_MERGE_MODE_XXX merge mode for interface data
+ * @param snaplen The snaplen to limit it to, or 0 to leave as it is in the files
+ * @param app_name The application name performing the merge, used in SHB info
+ * @param cb The callback information to use during execution
+ * @param[out] err Set to the internal WTAP_ERR_XXX error code if it failed
+ * @param[out] err_info Set to a descriptive error string, which must be g_free'd
+ * @param[out] err_fileno Set to the input file number which failed, if it failed
+ * @return the frame type
+ */
+WS_DLL_PUBLIC merge_result
+merge_files(int out_fd, const gchar* out_filename, const int file_type,
+            const char *const *in_filenames, const guint in_file_count,
+            const gboolean do_append, const idb_merge_mode mode,
+            guint snaplen, const gchar *app_name, merge_progress_callback_t* cb,
+            int *err, gchar **err_info, int *err_fileno);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */
diff --git a/wiretap/wtap.c b/wiretap/wtap.c
index e5a6af7021..ea8004febf 100644
--- a/wiretap/wtap.c
+++ b/wiretap/wtap.c
@@ -263,6 +263,30 @@ wtap_file_get_idb_info(wtap *wth)
 	return idb_info;
 }
 
+gchar *
+wtap_get_debug_if_descr(const wtapng_if_descr_t *if_descr)
+{
+	GString *info = g_string_new("");
+
+	g_assert(if_descr);
+
+	g_string_printf(info,
+		"Interface description: wtap_encap=%d, time_units_per_second=%" G_GINT64_MODIFIER "u"
+		", tsprecision=%d, link_type=%u, snap_len=%u, opt_comment='%s', if_name='%s'"
+		", if_description='%s', if_speed=%" G_GINT64_MODIFIER "u, if_tsresol=%d"
+		", if_filter_str='%s', bpf_filter_len=%u, if_fcslen=%d, num_stat_entries=%u",
+		if_descr->wtap_encap, if_descr->time_units_per_second, if_descr->tsprecision,
+		if_descr->link_type, if_descr->snap_len,
+		if_descr->opt_comment ? if_descr->opt_comment : "NONE",
+		if_descr->if_name ? if_descr->if_name : "NONE",
+		if_descr->if_description ? if_descr->if_description : "NONE",
+		if_descr->if_speed, if_descr->if_tsresol,
+		if_descr->if_filter_str ? if_descr->if_filter_str : "NONE",
+		if_descr->bpf_filter_len, if_descr->if_fcslen, if_descr->num_stat_entries);
+
+	return g_string_free(info, FALSE);
+}
+
 wtapng_name_res_t *
 wtap_file_get_nrb_for_new_file(wtap *wth)
 {
diff --git a/wiretap/wtap.h b/wiretap/wtap.h
index 1f9b4e1e84..e3b4e5166d 100644
--- a/wiretap/wtap.h
+++ b/wiretap/wtap.h
@@ -1735,6 +1735,18 @@ WS_DLL_PUBLIC
 wtapng_iface_descriptions_t *wtap_file_get_idb_info(wtap *wth);
 
 /**
+ * @brief Gets a debug string of an interface description.
+ * @details Returns a newly allocated string of debug information about
+ *          the given interface descrption, useful for debugging.
+ * @note The returned pointer must be g_free'd.
+ *
+ * @param if_descr The interface description.
+ * @return A newly allocated gcahr array string, which must be g_free'd.
+ */
+WS_DLL_PUBLIC
+gchar *wtap_get_debug_if_descr(const wtapng_if_descr_t *if_descr);
+
+/**
  * @brief Gets new name resolution info for new file, based on existing info.
  * @details Creates a new wtapng_name_res_t name resolution info and only
  *          copies appropriate members for a new file.