Make the Flex scanners and YACC parser in libraries reentrant.

master-branch libpcap now generates a reentrant Flex scanner and
Bison/Berkeley YACC parser for capture filter expressions, so it
requires versions of Flex and Bison/Berkeley YACC that support that.

We might as well do the same.  For libwiretap, it means we could
actually have multiple K12 text or Ascend/Lucent text files open at the
same time.  For libwireshark, it might not be as useful, as we only read
configuration files at startup (which should only happen once, in one
thread) or on demand (in which case, if we ever support multiple threads
running libwireshark, we'd need a mutex to ensure that only one file
reads it), but it's still the right thing to do.

We also require a version of Flex that can write out a header file, so
we change the runlex script to generate the header file ourselves. This
means we require a version of Flex new enough to support --header-file.

Clean up some other stuff encountered in the process.

Change-Id: Id23078c6acea549a52fc687779bb55d715b55c16
Reviewed-on: https://code.wireshark.org/review/14719
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 175 insertions, 92 deletions

diff --git a/epan/uat_load.l b/epan/uat_load.l
index 149c03226b..a27fb2ff2d 100644
--- a/epan/uat_load.l
+++ b/epan/uat_load.l
@@ -1,4 +1,9 @@
 /*
+ * We want a reentrant scanner.
+ */
+%option reentrant
+
+/*
  * We don't use input, so don't generate code for it.
  */
 %option noinput
@@ -19,11 +24,41 @@
 %option noyywrap
 
 /*
+ * The type for the state we keep for a scanner.
+ */
+%option extra-type="uat_load_scanner_state_t *"
+
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide our own versions of the routines generated by Flex, which
+ * have a "yes, this is unused, don't complain about it" flag for yyscanner.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
+/*
  * Prefix scanner routines with "uat_load_" rather than "yy", so this scanner
  * can coexist with other scanners.
  */
 %option prefix="uat_load_"
 
+/*
+ * We have to override the memory allocators so that we don't get
+ * "unused argument" warnings from the yyscanner argument (which
+ * we don't use, as we have a global memory allocator).
+ *
+ * We provide, as macros, our own versions of the routines generated by Flex,
+ * which just call malloc()/realloc()/free() (as the Flex versions do),
+ * discarding the extra argument.
+ */
+%option noyyalloc
+%option noyyrealloc
+%option noyyfree
+
 %{
 	/*
 	 * uat_load.l
@@ -62,7 +97,6 @@
 #include <glib.h>
 
 #include "uat-int.h"
-#include "uat_load_lex.h"
 #include <wsutil/file_util.h>
 
 #ifdef _WIN32
@@ -71,41 +105,44 @@
 #pragma warning (disable:4018)
 #endif
 
-static uat_t* uat;
-static gboolean valid_record;
-static guint colnum;
-static gchar* ptrx;
-static guint len;
-static gchar* error;
-static void* record;
-static guint linenum;
-static gchar *parse_str;
-static size_t parse_str_pos;
+typedef struct {
+	uat_t* uat;
+	gchar *parse_str;
+
+	gchar* error;
+	gboolean valid_record;
+	guint colnum;
+	gchar* ptrx;
+	guint len;
+	void* record;
+	guint linenum;
+	size_t parse_str_pos;
+} uat_load_scanner_state_t;
 
 #define ERROR(fmtd) do { \
     char* fmt_str = g_strdup_printf fmtd; \
-    error = g_strdup_printf("%s:%d: %s",uat->filename,linenum,fmt_str); \
+    yyextra->error = g_strdup_printf("%s:%d: %s",yyextra->uat->filename,yyextra->linenum,fmt_str); \
     g_free(fmt_str); \
     yyterminate(); \
 } while(0)
 
 #define SET_FIELD() \
 	{ gchar* errx; \
-	if (uat->fields[colnum].cb.chk) { \
-		if ( ! uat->fields[colnum].cb.chk(record, ptrx, len, uat->fields[colnum].cbdata.chk, uat->fields[colnum].fld_data, &errx) ) { \
-			error = g_strdup_printf("%s:%d: %s",uat->filename,linenum,errx); \
+	if (yyextra->uat->fields[yyextra->colnum].cb.chk) { \
+		if ( ! yyextra->uat->fields[yyextra->colnum].cb.chk(yyextra->record, yyextra->ptrx, yyextra->len, yyextra->uat->fields[yyextra->colnum].cbdata.chk, yyextra->uat->fields[yyextra->colnum].fld_data, &errx) ) { \
+			yyextra->error = g_strdup_printf("%s:%d: %s",yyextra->uat->filename,yyextra->linenum,errx); \
 			g_free(errx); \
-			valid_record = FALSE; \
+			yyextra->valid_record = FALSE; \
 		}\
 	}\
-	uat->fields[colnum].cb.set(record, ptrx, len, uat->fields[colnum].cbdata.chk, uat->fields[colnum].fld_data);\
-	g_free(ptrx);\
-	colnum++; \
+	yyextra->uat->fields[yyextra->colnum].cb.set(yyextra->record, yyextra->ptrx, yyextra->len, yyextra->uat->fields[yyextra->colnum].cbdata.chk, yyextra->uat->fields[yyextra->colnum].fld_data);\
+	g_free(yyextra->ptrx);\
+	yyextra->colnum++; \
 	} while(0)
 
 #ifdef DEBUG_UAT_LOAD
 #define DUMP_FIELD(str) \
-		{ guint i; printf("%s: %s='",str,uat->fields[colnum].name); for(i=0;i<len;i++) if (uat->fields[colnum].mode == PT_TXTMOD_HEXBYTES) { printf("%.2x ",((guint8*)ptrx)[i]); } else putc(ptrx[i],stdout); printf("'[%d]\n",len); }
+		{ guint i; printf("%s: %s='",str,yyextra->uat->fields[yyextra->colnum].name); for(i=0;i<yyextra->len;i++) if (yyextra->uat->fields[yyextra->colnum].mode == PT_TXTMOD_HEXBYTES) { printf("%.2x ",((guint8*)yyextra->ptrx)[i]); } else putc(yyextra->ptrx[i],stdout); printf("'[%d]\n",yyextra->len); }
 
 #define DUMP(str) printf("%s\n",str)
 #else
@@ -115,16 +152,16 @@ static size_t parse_str_pos;
 
 /* Modified version of YY_INPUT generated by Flex 2.91 */
 #define YY_INPUT(buf,result,max_size) \
-	if ( parse_str ) \
+	if ( yyextra->parse_str ) \
 		{ \
 		size_t n = 0; \
-		size_t pslen = strlen(parse_str); \
-		if (parse_str_pos < pslen) \
+		size_t pslen = strlen(yyextra->parse_str); \
+		if (yyextra->parse_str_pos < pslen) \
 			{ \
-			n = pslen - parse_str_pos; \
+			n = pslen - yyextra->parse_str_pos; \
 			if (n > max_size) n = max_size; \
-			memcpy(buf, parse_str + parse_str_pos, n); \
-			parse_str_pos += n; \
+			memcpy(buf, yyextra->parse_str + yyextra->parse_str_pos, n); \
+			yyextra->parse_str_pos += n; \
 			} \
 		result = n; \
 		} \
@@ -148,6 +185,21 @@ static size_t parse_str_pos;
 		 * quoted_string below fails badly on "...\\"
 		 * workarround in uat_save(), using /x5c and /x22
 		 */
+
+#define YY_USER_INIT BEGIN START_OF_LINE;
+
+/*
+ * Sleazy hack to suppress compiler warnings in yy_fatal_error().
+ */
+#define YY_EXIT_FAILURE ((void)yyscanner, 2)
+
+/*
+ * Macros for the allocators, to discard the extra argument.
+ */
+#define uat_load_alloc(size, yyscanner)		(void *)malloc(size)
+#define uat_load_realloc(ptr, size, yyscanner)	(void *)realloc((char *)(ptr), (size))
+#define uat_load_free(ptr, yyscanner)		free((char *)ptr)
+
 %}
 
 quoted_string \042([^\042]|\134\042)*\042
@@ -160,18 +212,18 @@ comment #[^\n]*\n
 %x START_OF_LINE NEXT_FIELD SEPARATOR END_OF_RECORD ERRORED
 %%
 <START_OF_LINE,NEXT_FIELD>{ws} ;
-<START_OF_LINE>{newline} linenum++;
-<START_OF_LINE>{comment} linenum++;
+<START_OF_LINE>{newline} yyextra->linenum++;
+<START_OF_LINE>{comment} yyextra->linenum++;
 
 <START_OF_LINE,NEXT_FIELD>{separator} {
-	ptrx = g_strdup("");
-	len = 0;
+	yyextra->ptrx = g_strdup("");
+	yyextra->len = 0;
 
 	DUMP_FIELD("empty->next");
 
 	SET_FIELD();
 
-	if ( colnum >= uat->ncols ) {
+	if ( yyextra->colnum >= yyextra->uat->ncols ) {
 		ERROR(("more fields than required"));
 	}
 
@@ -179,8 +231,8 @@ comment #[^\n]*\n
 }
 
 <START_OF_LINE,NEXT_FIELD>{newline}   {
-	ptrx = g_strdup("");
-	len = 0;
+	yyextra->ptrx = g_strdup("");
+	yyextra->len = 0;
 
 	BEGIN END_OF_RECORD;
 
@@ -188,10 +240,10 @@ comment #[^\n]*\n
 }
 
 <START_OF_LINE,NEXT_FIELD>{quoted_string} {
-	ptrx = uat_undquote(yytext, (guint) yyleng, &len);
+	yyextra->ptrx = uat_undquote(yytext, (guint) yyleng, &yyextra->len);
 
 
-	if (colnum < uat->ncols - 1) {
+	if (yyextra->colnum < yyextra->uat->ncols - 1) {
 		DUMP("quoted_str->s");
 		BEGIN SEPARATOR;
 	} else {
@@ -201,13 +253,13 @@ comment #[^\n]*\n
 }
 
 <START_OF_LINE,NEXT_FIELD>{binstring} {
-	ptrx = uat_unbinstring(yytext,  (guint) yyleng, &len);
+	yyextra->ptrx = uat_unbinstring(yytext,  (guint) yyleng, &yyextra->len);
 
-	if (!ptrx) {
-		ERROR(("uneven hexstring for field %s",uat->fields[colnum].name));
+	if (!yyextra->ptrx) {
+		ERROR(("uneven hexstring for field %s",yyextra->uat->fields[yyextra->colnum].name));
 	}
 
-	if ( colnum < uat->ncols - 1 ) {
+	if ( yyextra->colnum < yyextra->uat->ncols - 1 ) {
 		DUMP("binstring->s");
 		BEGIN SEPARATOR;
 	} else {
@@ -222,7 +274,7 @@ comment #[^\n]*\n
 
 	SET_FIELD();
 
-	if ( colnum >= uat->ncols ) {
+	if ( yyextra->colnum >= yyextra->uat->ncols ) {
 		ERROR(("more fields than required"));
 	}
 
@@ -230,12 +282,12 @@ comment #[^\n]*\n
 }
 
 <SEPARATOR>{newline} {
-	linenum++;
-	ERROR(("expecting field %s in previous line",uat->fields[colnum].name));
+	yyextra->linenum++;
+	ERROR(("expecting field %s in previous line",yyextra->uat->fields[yyextra->colnum].name));
 }
 
 <SEPARATOR>. {
-	ERROR(("unexpected char '%s' while looking for field %s",yytext,uat->fields[colnum].name));
+	ERROR(("unexpected char '%s' while looking for field %s",yytext,yyextra->uat->fields[yyextra->colnum].name));
 }
 
 <END_OF_RECORD>{separator} {
@@ -246,28 +298,28 @@ comment #[^\n]*\n
 	void* rec;
 	char* err = NULL;
 
-	linenum++;
+	yyextra->linenum++;
 
 	DUMP_FIELD("newline->start");
 
 	SET_FIELD();
 
-	rec = uat_add_record(uat, record, valid_record);
+	rec = uat_add_record(yyextra->uat, yyextra->record, yyextra->valid_record);
 
-	if ((uat->update_cb) && (rec != NULL)) {
-		if (!uat->update_cb(rec,&err)) {
-			error = err;
+	if ((yyextra->uat->update_cb) && (rec != NULL)) {
+		if (!yyextra->uat->update_cb(rec,&err)) {
+			yyextra->error = err;
 			yyterminate();
 		}
 	}
 
-	valid_record = TRUE;
-	colnum = 0;
-	ptrx = NULL;
-	len = 0;
+	yyextra->valid_record = TRUE;
+	yyextra->colnum = 0;
+	yyextra->ptrx = NULL;
+	yyextra->len = 0;
 
 	/* XXX is this necessary since we free it before reusing anyway? */
-	memset(record,0,uat->record_size);
+	memset(yyextra->record,0,yyextra->uat->record_size);
 
 	BEGIN START_OF_LINE;
  }
@@ -276,10 +328,10 @@ comment #[^\n]*\n
 	ERROR(("unexpected char while looking for end of line"));
 }
 
-<ERRORED>{newline} { linenum++; BEGIN START_OF_LINE; }
+<ERRORED>{newline} { yyextra->linenum++; BEGIN START_OF_LINE; }
 <ERRORED>. ;
 
-{newline} { linenum++; ERROR(("incomplete record")); }
+{newline} { yyextra->linenum++; ERROR(("incomplete record")); }
 . { ERROR(("unexpected input")); }
 
 %%
@@ -288,12 +340,12 @@ comment #[^\n]*\n
 
 
 gboolean
-uat_load(uat_t *uat_in, char **errx)
+uat_load(uat_t *uat, char **errx)
 {
-	gchar *fname = uat_get_actual_filename(uat_in, FALSE);
-
-	uat = uat_in;
-	parse_str = NULL;
+	gchar *fname = uat_get_actual_filename(uat, FALSE);
+	FILE *in;
+	yyscan_t scanner;
+	uat_load_scanner_state_t state;
 
 	if (!fname) {
 		UAT_UPDATE(uat);
@@ -305,33 +357,51 @@ uat_load(uat_t *uat_in, char **errx)
 	}
 
 
-	if (!(yyin = ws_fopen(fname,"r"))) {
+	if (!(in = ws_fopen(fname,"r"))) {
 		*errx = g_strdup(g_strerror(errno));
 		g_free(fname);
 		return FALSE;
 	}
 
-	error = NULL;
-	valid_record = TRUE;
-	colnum = 0;
-	g_free(record);
-	record = g_malloc0(uat->record_size);
-	linenum = 1;
+	if (uat_load_lex_init(&scanner) != 0) {
+		*errx = g_strdup(g_strerror(errno));
+		fclose(in);
+		g_free(fname);
+		return FALSE;
+	}
+
+	uat_load_set_in(in, scanner);
+
+	state.uat = uat;
+	state.parse_str = NULL;	/* we're reading from a file */
+
+	state.error = NULL;
+	state.valid_record = TRUE;
+	state.colnum = 0;
+	state.ptrx = NULL;
+	state.len = 0;
+	state.record = g_malloc0(uat->record_size);
+	state.linenum = 1;
+	state.parse_str_pos = 0;
 
-	BEGIN START_OF_LINE;
 	DUMP(fname);
 	g_free(fname);	/* we're done with the file name now */
 
-	yylex();
-	fclose(yyin);
-	yyrestart(NULL);
+	/* Associate the state with the scanner */
+	uat_load_set_extra(&state, scanner);
+
+	uat_load_lex(scanner);
+
+	uat_load_lex_destroy(scanner);
+	g_free(state.record);
+	fclose(in);
 
 	uat->changed = FALSE;
 	uat->loaded = TRUE;
 	UAT_UPDATE(uat);
 
-	if (error) {
-		*errx = error;
+	if (state.error) {
+		*errx = state.error;
 		return FALSE;
 	}
 
@@ -343,34 +413,47 @@ uat_load(uat_t *uat_in, char **errx)
 }
 
 gboolean
-uat_load_str(uat_t *uat_in, char *entry, char **err)
+uat_load_str(uat_t *uat, char *entry, char **err)
 {
-	uat = uat_in;
-	parse_str = g_strdup_printf("%s\n", entry); /* Records must end with a newline */
-	parse_str_pos = 0;
-	yyin = NULL;
-
-	error = NULL;
-	valid_record = TRUE;
-	colnum = 0;
-	g_free(record);
-	record = g_malloc0(uat->record_size);
-	linenum = 1;
+	yyscan_t scanner;
+	uat_load_scanner_state_t state;
+
+	state.uat = uat;
+	state.parse_str = g_strdup_printf("%s\n", entry); /* Records must end with a newline */
+
+	state.error = NULL;
+	state.valid_record = TRUE;
+	state.colnum = 0;
+	state.ptrx = NULL;
+	state.len = 0;
+	state.record = g_malloc0(uat->record_size);
+	state.linenum = 1;
+	state.parse_str_pos = 0;
+
+	if (uat_load_lex_init(&scanner) != 0) {
+		*err = g_strdup(g_strerror(errno));
+		g_free(state.parse_str);
+		g_free(state.record);
+		return FALSE;
+	}
 
-	BEGIN START_OF_LINE;
 	DUMP(entry);
 
-	yylex();
-	yyrestart(NULL);
-	g_free(parse_str);
-	parse_str = NULL;
+	/* Associate the state with the scanner */
+	uat_load_set_extra(&state, scanner);
+
+	uat_load_lex(scanner);
+
+	uat_load_lex_destroy(scanner);
+	g_free(state.record);
+	g_free(state.parse_str);
 
 	uat->changed = TRUE;
 	uat->loaded = TRUE;
 	UAT_UPDATE(uat);
 
-	if (error) {
-		*err = error;
+	if (state.error) {
+		*err = state.error;
 		return FALSE;
 	}