tshark: improve -G elastic-mapping command by adding filters.

The generated elastic mapping file is huge and it can hassle softwares
like Kibana. This change adds the ability to append desired filters
that will appear in the mapping file.

This change adds the option --elastic-mapping-filter <protocols> to tshark.

Example: tshark -G elastic-mapping --elastic-mapping-filter ip,udp,dns

make only those 3 protocols to appear in the mapping file.

Change-Id: Ie2dcd6e44be2d084e8e50cd6554bd90178da4e38
Reviewed-on: https://code.wireshark.org/review/27001
Reviewed-by: Peter Wu <peter@lekensteyn.nl>
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Dario Lombardo <lomato@gmail.com>

4 files changed, 71 insertions, 10 deletions

diff --git a/doc/tshark.pod b/doc/tshark.pod
index 9ccfdbca7b..e44fa7dfa6 100644
--- a/doc/tshark.pod
+++ b/doc/tshark.pod
@@ -66,7 +66,7 @@ S<[ B<--disable-heuristic> E<lt>short_nameE<gt> ]>
 S<[ E<lt>capture filterE<gt> ]>
 
 B<tshark>
-B<-G> [ E<lt>report typeE<gt> ]
+B<-G> [ E<lt>report typeE<gt> ] [ --elastic-mapping-filter E<lt>protocolsE<gt> ]
 
 =head1 DESCRIPTION
 
@@ -839,7 +839,12 @@ Example of usage to import data into Elasticsearch:
 
 Elastic requires a mapping file to be loaded as template for packets-*
 index in order to convert wireshark types to elastic types. This file
-can be auto-generated with the command "tshark -G elastic-mapping".
+can be auto-generated with the command "tshark -G elastic-mapping". Since
+the mapping file can be huge, protocols can be selected by using the option
+--elastic-mapping-filter:
+
+  tshark -G elastic-mapping --elastic-mapping-filter ip,udp,dns
+
 
 B<fields> The values of fields specified with the B<-e> option, in a
 form specified by the B<-E> option.  For example,
@@ -1743,6 +1748,14 @@ If a key appears multiple times in an object, only write it a single time with
 as value a json array containing all the separate values. (Only works with
 -T json)
 
+=item --elastic-mapping-filter E<lt>protocolE<gt>,E<lt>protocolE<gt>,...
+
+When generating the ElasticSearch mapping file, only put the specified protocols
+in it, to avoid a huge mapping file that can choke some software (such as Kibana).
+The option takes a list of wanted protocol abbreviations, separated by comma.
+
+Example: ip,udp,dns puts only those three protocols in the mapping file.
+
 =item --export-objects E<lt>protocolE<gt>,E<lt>destdirE<gt>
 
 Export all objects within a protocol into directory B<destdir>. The available
diff --git a/epan/proto.c b/epan/proto.c
index 762bd48d8e..d41a8294ef 100644
--- a/epan/proto.c
+++ b/epan/proto.c
@@ -10052,7 +10052,7 @@ dot_to_underscore(gchar* str)
 /* Dumps a mapping file for ElasticSearch
  */
 void
-proto_registrar_dump_elastic(void)
+proto_registrar_dump_elastic(const gchar* filter)
 {
 	header_field_info *hfinfo;
 	header_field_info *parent_hfinfo;
@@ -10065,11 +10065,20 @@ proto_registrar_dump_elastic(void)
 	const char* prev_proto = NULL;
 	gchar* data;
 	gchar* str;
+	gchar** protos = NULL;
+	gchar* proto;
+	gboolean found;
+	guint j;
+
+	/* We have filtering protocols. Extract them. */
+	if (filter) {
+		protos = g_strsplit(filter, ",", -1);
+	}
 
 	/*
-		To help traking down the json tree, objects have been appended with a comment:
-		n.label -> where n is the indentation level and label the name of the object
-	*/
+	 * To help tracking down the json tree, objects have been appended with a comment:
+	 * n.label -> where n is the indentation level and label the name of the object
+	 */
 
 	builder = json_builder_new();
 	json_builder_begin_object(builder); // 1.root
@@ -10104,7 +10113,7 @@ proto_registrar_dump_elastic(void)
 
 		/*
 		 * Skip the pseudo-field for "proto_tree_add_text()" since
-		 * we don't want it in the list of filterable fields.
+		 * we don't want it in the list of filterable protocols.
 		 */
 		if (hfinfo->id == hf_text_only)
 			continue;
@@ -10112,6 +10121,26 @@ proto_registrar_dump_elastic(void)
 		if (!proto_registrar_is_protocol(i)) {
 			PROTO_REGISTRAR_GET_NTH(hfinfo->parent, parent_hfinfo);
 
+			/*
+			 * Skip the field if filter protocols have been set and this one's
+			 * parent is not listed.
+			 */
+			if (protos) {
+				found = FALSE;
+				j = 0;
+				proto = protos[0];
+				while(proto) {
+					if (!g_strcmp0(proto, parent_hfinfo->abbrev)) {
+						found = TRUE;
+						break;
+					}
+					j++;
+					proto = protos[j];
+				}
+				if (!found)
+					continue;
+			}
+
 			if (prev_proto && g_strcmp0(parent_hfinfo->abbrev, prev_proto)) {
 				json_builder_end_object(builder); // 8.properties
 				json_builder_end_object(builder); // 7.parent_hfinfo->abbrev
@@ -10159,6 +10188,7 @@ proto_registrar_dump_elastic(void)
 	g_object_unref(generator);
 	ws_debug_printf("%s\n", data);
 	g_free(data);
+	g_strfreev(protos);
 }
 #endif
 
diff --git a/epan/proto.h b/epan/proto.h
index d618a49e36..95c0fe8fcc 100644
--- a/epan/proto.h
+++ b/epan/proto.h
@@ -2455,7 +2455,7 @@ WS_DLL_PUBLIC void proto_registrar_dump_values(void);
 
 #ifdef HAVE_JSONGLIB
 /** Dumps a mapping file for loading tshark output into ElasticSearch */
-WS_DLL_PUBLIC void proto_registrar_dump_elastic(void);
+WS_DLL_PUBLIC void proto_registrar_dump_elastic(const gchar* filter);
 #endif
 
 /** Dumps the number of protocol and field registrations to STDOUT.
diff --git a/tshark.c b/tshark.c
index fb8c6ede12..504aae4226 100644
--- a/tshark.c
+++ b/tshark.c
@@ -142,6 +142,9 @@
  */
 #define LONGOPT_COLOR (65536+1000)
 #define LONGOPT_NO_DUPLICATE_KEYS (65536+1001)
+#ifdef HAVE_JSONGLIB
+#define LONGOPT_ELASTIC_MAPPING_FILTER (65536+1002)
+#endif
 
 #if 0
 #define tshark_debug(...) g_warning(__VA_ARGS__)
@@ -439,7 +442,11 @@ print_usage(FILE *output)
   fprintf(output, "                           (Note that attributes are nonstandard)\n");
   fprintf(output, "  --no-duplicate-keys      If -T json is specified, merge duplicate keys in an object\n");
   fprintf(output, "                           into a single key with as value a json array containing all\n");
-  fprintf(output, "                           values");
+  fprintf(output, "                           values\n");
+#ifdef HAVE_JSONGLIB
+  fprintf(output, "  --elastic-mapping-filter <protocols> If -G elastic-mapping is specified, put only the\n");
+  fprintf(output, "                           specified protocols within the mapping file\n");
+#endif
 
   fprintf(output, "\n");
   fprintf(output, "Miscellaneous:\n");
@@ -680,6 +687,9 @@ main(int argc, char *argv[])
     {"export-objects", required_argument, NULL, LONGOPT_EXPORT_OBJECTS},
     {"color", no_argument, NULL, LONGOPT_COLOR},
     {"no-duplicate-keys", no_argument, NULL, LONGOPT_NO_DUPLICATE_KEYS},
+#ifdef HAVE_JSONGLIB
+    {"elastic-mapping-filter", required_argument, NULL, LONGOPT_ELASTIC_MAPPING_FILTER},
+#endif
     {0, 0, 0, 0 }
   };
   gboolean             arg_error = FALSE;
@@ -723,6 +733,9 @@ main(int argc, char *argv[])
   gchar               *volatile pdu_export_arg = NULL;
   const char          *volatile exp_pdu_filename = NULL;
   exp_pdu_t            exp_pdu_tap_data;
+#ifdef HAVE_JSONGLIB
+  const gchar*         elastic_mapping_filter = NULL;
+#endif
 
 /*
  * The leading + ensures that getopt_long() does not permute the argv[]
@@ -864,6 +877,11 @@ main(int argc, char *argv[])
     case 'X':
       ex_opt_add(optarg);
       break;
+#ifdef HAVE_JSONGLIB
+    case LONGOPT_ELASTIC_MAPPING_FILTER:
+      elastic_mapping_filter = optarg;
+      break;
+#endif
     default:
       break;
     }
@@ -969,7 +987,7 @@ main(int argc, char *argv[])
         dissector_dump_dissector_tables();
 #ifdef HAVE_JSONGLIB
       else if (strcmp(argv[2], "elastic-mapping") == 0)
-        proto_registrar_dump_elastic();
+        proto_registrar_dump_elastic(elastic_mapping_filter);
 #endif
       else if (strcmp(argv[2], "fieldcount") == 0) {
         /* return value for the test suite */