Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

3 files changed, 81 insertions, 39 deletions

diff --git a/epan/dfilter/dfilter-int.h b/epan/dfilter/dfilter-int.h
index 40e55488b5..f94c78f139 100644
--- a/epan/dfilter/dfilter-int.h
+++ b/epan/dfilter/dfilter-int.h
@@ -56,10 +56,12 @@ typedef struct {
 } dfwork_t;
 
 /*
- * XXX - if we're using a version of Flex that supports reentrant lexical
- * analyzers, we should put this into the lexical analyzer's state.
+ * State kept by the scanner.
  */
-extern dfwork_t *global_dfw;
+typedef struct {
+	dfwork_t *dfw;
+	GString* quoted_string;
+} df_scanner_state_t;
 
 /* Constructor/Destructor prototypes for Lemon Parser */
 void *DfilterAlloc(void* (*)(gsize));
diff --git a/epan/dfilter/dfilter.c b/epan/dfilter/dfilter.c
index df24b1ced7..151bb5ec7d 100644
--- a/epan/dfilter/dfilter.c
+++ b/epan/dfilter/dfilter.c
@@ -31,14 +31,10 @@
 #include <epan/epan_dissect.h>
 #include "dfilter.h"
 #include "dfilter-macro.h"
+#include "scanner_lex.h"
 
 #define DFILTER_TOKEN_ID_OFFSET	1
 
-/* From scanner.c */
-void df_scanner_text(const char *text);
-void    df_scanner_cleanup(void);
-int     df_lex(void);
-
 /* Holds the singular instance of our Lemon parser object */
 static void*	ParserObj = NULL;
 
@@ -214,6 +210,9 @@ dfilter_compile(const gchar *text, dfilter_t **dfp, gchar **err_msg)
 	int		token;
 	dfilter_t	*dfilter;
 	dfwork_t	*dfw;
+	df_scanner_state_t state;
+	yyscan_t	scanner;
+	YY_BUFFER_STATE in_buffer;
 	gboolean failure = FALSE;
 	const char	*depr_test;
 	guint		i;
@@ -233,21 +232,28 @@ dfilter_compile(const gchar *text, dfilter_t **dfp, gchar **err_msg)
 		return FALSE;
 	}
 
+	if (df_lex_init(&scanner) != 0) {
+		*dfp = NULL;
+		if (err_msg != NULL)
+			*err_msg = g_strdup_printf("Can't initialize scanner: %s",
+			    g_strerror(errno));
+		return FALSE;
+	}
+
+	in_buffer = df__scan_string(expanded_text, scanner);
+
 	dfw = dfwork_new();
 
-	/*
-	 * XXX - if we're using a version of Flex that supports reentrant lexical
-	 * analyzers, we should put this into the lexical analyzer's state.
-	 */
-	global_dfw = dfw;
+	state.dfw = dfw;
+	state.quoted_string = NULL;
 
-	df_scanner_text(expanded_text);
+	df_set_extra(&state, scanner);
 
 	deprecated = g_ptr_array_new();
 
 	while (1) {
 		df_lval = stnode_new(STTYPE_UNINITIALIZED, NULL);
-		token = df_lex();
+		token = df_lex(scanner);
 
 		/* Check for scanner failure */
 		if (token == SCAN_FAILED) {
@@ -307,8 +313,11 @@ dfilter_compile(const gchar *text, dfilter_t **dfp, gchar **err_msg)
 	if (dfw->syntax_error)
 		failure = TRUE;
 
-	/* Reset flex */
-	df_scanner_cleanup();
+	/* Free scanner state */
+	if (state.quoted_string != NULL)
+		g_string_free(state.quoted_string, TRUE);
+	df__delete_buffer(in_buffer, scanner);
+	df_lex_destroy(scanner);
 
 	if (failure)
 		goto FAILURE;
diff --git a/epan/dfilter/scanner.l b/epan/dfilter/scanner.l
index c213ac8351..ce138da8b6 100644
--- a/epan/dfilter/scanner.l
+++ b/epan/dfilter/scanner.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -24,6 +29,24 @@
  */
 %option noyywrap
 
+/*
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="df_scanner_state_t *"
+
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
 %{
 /*
  * Wireshark - Network traffic analyzer
@@ -54,7 +77,6 @@
 #include "syntax-tree.h"
 #include "grammar.h"
 #include "dfunctions.h"
-#include "scanner_lex.h"
 
 #ifdef _WIN32
 /* disable Windows VC compiler warning "signed/unsigned mismatch" associated  */
@@ -76,9 +98,20 @@ static int set_lval(int token, gpointer data);
 static int set_lval_int(dfwork_t *dfw, int token, char *s);
 static int simple(int token);
 static gboolean str_to_gint32(dfwork_t *dfw, char *s, gint32* pint);
-GString* quoted_string = NULL;
 static void mark_lval_deprecated(const char *s);
 
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define df_alloc(size, yyscanner)		(void *)malloc(size)
+#define df_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define df_free(ptr, yyscanner)			free((char *)ptr)
+
 %}
 
 %x RANGE_INT
@@ -136,12 +169,12 @@ static void mark_lval_deprecated(const char *s);
 
 <RANGE_INT>[+-]?[[:digit:]]+		{
 	BEGIN(RANGE_PUNCT);
-	return set_lval_int(global_dfw, TOKEN_INTEGER, yytext);
+	return set_lval_int(yyextra->dfw, TOKEN_INTEGER, yytext);
 }
 
 <RANGE_INT>[+-]?0x[[:xdigit:]]+		{
 	BEGIN(RANGE_PUNCT);
-	return set_lval_int(global_dfw, TOKEN_INTEGER, yytext);
+	return set_lval_int(yyextra->dfw, TOKEN_INTEGER, yytext);
 }
 
 <RANGE_INT,RANGE_PUNCT>":"		{
@@ -167,7 +200,7 @@ static void mark_lval_deprecated(const char *s);
 	/* Error if none of the above while scanning a range (slice) */
 
 <RANGE_PUNCT>[^:\-,\]]+		{
-	dfilter_fail(global_dfw, "Invalid string \"%s\" found while scanning slice.", yytext);
+	dfilter_fail(yyextra->dfw, "Invalid string \"%s\" found while scanning slice.", yytext);
 	return SCAN_FAILED;
 }
 
@@ -176,7 +209,7 @@ static void mark_lval_deprecated(const char *s);
 	 */
 
 <RANGE_INT>.	{
-	dfilter_fail(global_dfw, "Invalid character \"%s\" found while scanning slice; expected integer.", yytext);
+	dfilter_fail(yyextra->dfw, "Invalid character \"%s\" found while scanning slice; expected integer.", yytext);
 	return SCAN_FAILED;
 }
 
@@ -192,13 +225,13 @@ static void mark_lval_deprecated(const char *s);
 	a missing end quote will have left quoted_string set
 	to something. Clear it now that we are starting
 	a new quoted string. */
-	if (quoted_string) {
-		g_string_free(quoted_string, TRUE);
+	if (yyextra->quoted_string) {
+		g_string_free(yyextra->quoted_string, TRUE);
 		/* Don't set quoted_string to NULL, as we
 		do in other quoted_string-cleanup code, as we're
 		about to set it in the next line. */
 	}
-	quoted_string = g_string_new("");
+	yyextra->quoted_string = g_string_new("");
 }
 
 <DQUOTE><<EOF>>				{
@@ -208,7 +241,7 @@ static void mark_lval_deprecated(const char *s);
 	See:
 	http://www.gnu.org/software/flex/manual/html_node/flex_13.html */
 
-	dfilter_fail(global_dfw, "The final quote was missing from a quoted string.");
+	dfilter_fail(yyextra->dfw, "The final quote was missing from a quoted string.");
 	return SCAN_FAILED;
 }
 
@@ -216,9 +249,9 @@ static void mark_lval_deprecated(const char *s);
 	/* end quote */
 	int token;
 	BEGIN(INITIAL);
-	token = set_lval(TOKEN_STRING, quoted_string->str);
-	g_string_free(quoted_string, TRUE);
-	quoted_string = NULL;
+	token = set_lval(TOKEN_STRING, yyextra->quoted_string->str);
+	g_string_free(yyextra->quoted_string, TRUE);
+	yyextra->quoted_string = NULL;
 	return token;
 }
 
@@ -227,30 +260,30 @@ static void mark_lval_deprecated(const char *s);
 	unsigned long result;
 	result = strtoul(yytext + 1, NULL, 8);
 	if (result > 0xff) {
-		g_string_free(quoted_string, TRUE);
-		quoted_string = NULL;
-		dfilter_fail(global_dfw, "%s is larger than 255.", yytext);
+		g_string_free(yyextra->quoted_string, TRUE);
+		yyextra->quoted_string = NULL;
+		dfilter_fail(yyextra->dfw, "%s is larger than 255.", yytext);
 		return SCAN_FAILED;
 	}
-	g_string_append_c(quoted_string, (gchar) result);
+	g_string_append_c(yyextra->quoted_string, (gchar) result);
 }
 
 <DQUOTE>\\x[[:xdigit:]]{1,2} {
 	/* hex sequence */
 	unsigned long result;
 	result = strtoul(yytext + 2, NULL, 16);
-	g_string_append_c(quoted_string, (gchar) result);
+	g_string_append_c(yyextra->quoted_string, (gchar) result);
 }
 
 
 <DQUOTE>\\.				{
 	/* escaped character */
-	g_string_append_c(quoted_string, yytext[1]);
+	g_string_append_c(yyextra->quoted_string, yytext[1]);
 }
 
 <DQUOTE>[^\\\042]+			{
 	/* non-escaped string */
-	g_string_append(quoted_string, yytext);
+	g_string_append(yyextra->quoted_string, yytext);
 }
 
 
@@ -429,5 +462,3 @@ mark_lval_deprecated(const char *s)
 {
 	df_lval->deprecated_token = s;
 }
-
-#include <lemonflex-tail.inc>