(closes issue #13357)

Reported by: pj
Tested by: murf

(closes issue #13416)
Reported by: yarns
Tested by: murf

If you find this message overly verbose, relax, it's probably
not meant for you. This message is meant for probably only
two people in the whole world: me, or the poor schnook that
has to maintain this code because I'm either dead or unavailable
at the moment.

This fix solves two reports, both having to do with embedding
a function call in a ${} construct. It was tricky because the
funccall syntax has parenthesis () in it. And up till now,
the 'word' token in the flex stuff didn't allow that, because
it would tend to steal the LP and RP tokens. To be truthful,
the "word" token was the trickiest, most unstable thing in
the whole lexer. I was lucky it made this long without complaints.
I had to choose every character in the pattern with extreme
care, and I knew that someday I'd have to revisit it. Well,
the day has come.

So, my brilliant idea (and I'm being modest), was to use the 
surrounding ${} construct to make a state machine and capture 
everything in it, no matter what it contains. But, I have to now
treat the word token like I did with comments, in that I turn
the whole thing into a state-machine sort of spec, with new
contexts "curlystate", "wordstate", and "brackstate".

Wait a minute, "brackstate"? Yes, well, it didn't take very many
regression tests to point out if I do this for ${} constructs,
I also have to do it with the $[] constructs, too.

I had to create a separate pcbstack2 and pcbstack3 because
these constructs can occur inside macro argument lists, and
when we have two state machines operating on the same structures
we'd get problems otherwise. I guess I could have stopped at
pcbstack2 and had the brackstate stuff share it, but it doesn't
hurt to be safe. So, the pcbpush and pcbpop routines also now
have versions for "2" and "3".

I had to add the {KEYWORD} construct to the initial pattern for
"word", because previously word would match stuff like "default7",
because it was a longer match than the keyword "default". But,
not any more, because the word pattern only matches only one or
two characters now, and it will always lose. So, I made it the
winner again by making an optional match on any of the keywords
before it's normal pattern.

I added another regression test to make sure we don't
lose this in future edits, and had to fix just one regression,
where it no longer reports a 'cascaded' error, which I guess
is a plus.

I've given some thought as to whether to apply these fixes to
1.4 and the 1.6.x releases, vs trunk; I decided to put it in
1.4 because one of the bug reports was against 1.4; and it
is unexpected that AEL cannot handle this situation. It actually
reduced the amount of useless "cascade" error messages that
appeared in the regressions (by one line, ehhem). There is
a possible side-effect in that it does now do more careful
checking of what's in those ${} constructs, as far as matching
parens, and brackets are concerned. Some users may find a an
insidious problem and correct it this way. This should be 
exceedingly rare, I hope.



git-svn-id: http://svn.digium.com/svn/asterisk/branches/1.4@141094 f38db490-d61c-443f-a65b-d21fe96a405b

1 files changed, 176 insertions, 4 deletions

diff --git a/pbx/ael/ael.flex b/pbx/ael/ael.flex
index 866e36170..2bf00695b 100644
--- a/pbx/ael/ael.flex
+++ b/pbx/ael/ael.flex
@@ -26,7 +26,7 @@
  *
  * %x describes the contexts we have: paren, semic and argg, plus INITIAL
  */
-%x paren semic argg  comment
+%x paren semic argg  comment curlystate wordstate brackstate
 
 /* prefix used for various globally-visible functions and variables.
  * This renames also yywrap, but since we do not use it, we just
@@ -83,10 +83,28 @@ static char pbcstack[400];	/* XXX missing size checks */
 static int pbcpos = 0;
 static void pbcpush(char x);
 static int pbcpop(char x);
-
 static int parencount = 0;
 
 /*
+ * A similar stack to keep track of matching brackets ( [ { } ] ) in word tokens surrounded by ${ ... }
+ */
+static char pbcstack2[400];	/* XXX missing size checks */
+static int pbcpos2 = 0;
+static void pbcpush2(char x);
+static int pbcpop2(char x);
+static int parencount2 = 0;
+
+/*
+ * A similar stack to keep track of matching brackets ( [ { } ] ) in word tokens surrounded by $[ ... ]
+ */
+static char pbcstack3[400];	/* XXX missing size checks */
+static int pbcpos3 = 0;
+static void pbcpush3(char x);
+static int pbcpop3(char x);
+static int parencount3 = 0;
+
+
+/*
  * current line, column and filename, updated as we read the input.
  */
 static int my_lineno = 1;	/* current line in the source */
@@ -175,6 +193,7 @@ static void pbcwhere(const char *text, int *line, int *col )
 #endif
 %}
 
+KEYWORD     (context|abstract|extend|macro|globals|local|ignorepat|switch|if|ifTime|random|regexten|hint|else|goto|jump|return|break|continue|for|while|case|default|pattern|catch|switches|eswitches|includes)
 
 NOPARENS	([^()\[\]\{\}]|\\[()\[\]\{\}])*
 
@@ -230,19 +249,140 @@ includes	{ STORE_POS; return KW_INCLUDES;}
 <comment>[^*\n]*\n	{ ++my_lineno; my_col=1;}
 <comment>"*"+[^*/\n]*	{ my_col += yyleng; }
 <comment>"*"+[^*/\n]*\n 	{ ++my_lineno; my_col=1;}
-<comment>"*/"		{ my_col += 2; BEGIN(INITIAL); }
+<comment>"*/"		{ my_col += 2; BEGIN(INITIAL); } /* the nice thing about comments is that you know exactly what ends them */
 
 \n		{ my_lineno++; my_col = 1; }
 [ ]+		{ my_col += yyleng; }
 [\t]+		{ my_col += (yyleng*8)-(my_col%8); }
 
-[-a-zA-Z0-9'"_/.\<\>\*\+!$#\[\]][-a-zA-Z0-9'"_/.!\*\+\<\>\{\}$#\[\]]*	{
+({KEYWORD}?[-a-zA-Z0-9'"_/.\<\>\*\+!$#\[\]]|(\\.)|(\$\{)|(\$\[)) {
+		/* boy did I open a can of worms when I changed the lexical token "word".
+		all the above keywords can be used as a beginning to a "word".-
+		before, a "word" would match a longer sequence than the above
+		keywords, and all would be well. But now "word" is a single char
+		and feeds into a statemachine sort of sequence from there on. So...
+		I added the {KEYWORD}? to the beginning of the word match sequence */
+
+		if (!strcmp(yytext,"${")) {
+			parencount2 = 0;
+			pbcpos2 = 0;
+			pbcpush2('{');  /* push '{' so the last pcbpop (parencount2 = -1) will succeed */
+			BEGIN(curlystate);
+			yymore();
+		} else if (!strcmp(yytext,"$[")) { 
+			parencount3 = 0; 
+			pbcpos3 = 0; 
+			pbcpush3('[');  /* push '[' so the last pcbpop (parencount3 = -1) will succeed */ 
+			BEGIN(brackstate); 
+			yymore(); 
+		} else { 
+			BEGIN(wordstate); 
+			yymore(); 
+		} 
+	} 
+<wordstate>[-a-zA-Z0-9'"_/.\<\>\*\+!$#\[\]] { yymore(); /* Keep going */ }
+<wordstate>(\\.)  { yymore(); /* Keep Going */ }
+<wordstate>(\$\{)  { /* the beginning of a ${} construct. prepare and pop into curlystate */
+		parencount2 = 0;
+		pbcpos2 = 0;
+		pbcpush2('{');  /* push '{' so the last pcbpop (parencount2 = -1) will succeed */
+		BEGIN(curlystate);
+		yymore();
+      }
+<wordstate>(\$\[)  { /* the beginning of a $[] construct. prepare and pop into brackstate */
+		parencount3 = 0;
+		pbcpos3 = 0;
+		pbcpush3('[');  /* push '[' so the last pcbpop (parencount3 = -1) will succeed */
+		BEGIN(brackstate);
+		yymore();
+	}
+<wordstate>([^a-zA-Z0-9\x80-\xff\x2d'"_/.\<\>\*\+!$#\[\]]) {
+		/* a non-word constituent char, like a space, tab, curly, paren, etc */
+		char c = yytext[yyleng-1];
 		STORE_POS;
 		yylval->str = strdup(yytext);
+		yylval->str[yyleng-1] = 0;
+		unput(c);  /* put this ending char back in the stream */
+		BEGIN(0);
 		prev_word = yylval->str;
 		return word;
 	}
+<curlystate>{NOPARENS}\}	{
+		if ( pbcpop2('}') ) {	/* error */
+			STORE_LOC;
+			ast_log(LOG_ERROR,"File=%s, line=%d, column=%d: Mismatched ')' in expression: %s !\n", my_file, my_lineno, my_col, yytext);
+			BEGIN(0);
+			yylval->str = strdup(yytext);
+			return word;
+		}
+		parencount2--;
+		if ( parencount2 >= 0) {
+			yymore();
+		} else {
+			BEGIN(wordstate); /* Finished with the current ${} construct. Return to word gathering state */
+			yymore();
+		}
+	}
 
+<curlystate>{NOPARENS}[\(\[\{]	{ 
+		char c = yytext[yyleng-1];
+		if (c == '{')
+			parencount2++;
+		pbcpush2(c);
+		yymore();
+	}
+
+<curlystate>{NOPARENS}[\]\)]	{ 
+		char c = yytext[yyleng-1];
+		if ( pbcpop2(c))  { /* error */
+			STORE_LOC;
+			ast_log(LOG_ERROR,"File=%s, line=%d, column=%d: Mismatched '%c' in expression!\n",
+				my_file, my_lineno, my_col, c);
+			BEGIN(0);
+			yylval->str = strdup(yytext);
+			return word;
+		}
+		yymore();
+	}
+
+
+<brackstate>{NOPARENS}\]	{
+		if ( pbcpop3(']') ) {	/* error */
+			STORE_LOC;
+			ast_log(LOG_ERROR,"File=%s, line=%d, column=%d: Mismatched ')' in expression: %s !\n", my_file, my_lineno, my_col, yytext);
+			BEGIN(0);
+			yylval->str = strdup(yytext);
+			return word;
+		}
+		parencount3--;
+		if ( parencount3 >= 0) {
+			yymore();
+		} else {
+			BEGIN(wordstate); /* Finished with the current ${} construct. Return to word gathering state */
+			yymore();
+		}
+	}
+
+<brackstate>{NOPARENS}[\(\[\{]	{ 
+		char c = yytext[yyleng-1];
+		if (c == '[')
+			parencount3++;
+		pbcpush3(c);
+		yymore();
+	}
+
+<brackstate>{NOPARENS}[\}\)]	{ 
+		char c = yytext[yyleng-1];
+		if ( pbcpop3(c))  { /* error */
+			STORE_LOC;
+			ast_log(LOG_ERROR,"File=%s, line=%d, column=%d: Mismatched '%c' in expression!\n",
+				my_file, my_lineno, my_col, c);
+			BEGIN(0);
+			yylval->str = strdup(yytext);
+			return word;
+		}
+		yymore();
+	}
 
 
 	/*
@@ -491,6 +631,38 @@ static int pbcpop(char x)
 	return 1; /* error */
 }
 
+static void pbcpush2(char x)
+{
+	pbcstack2[pbcpos2++] = x;
+}
+
+static int pbcpop2(char x)
+{
+	if (   ( x == ')' && pbcstack2[pbcpos2-1] == '(' )
+		|| ( x == ']' && pbcstack2[pbcpos2-1] == '[' )
+		|| ( x == '}' && pbcstack2[pbcpos2-1] == '{' )) {
+		pbcpos2--;
+		return 0;
+	}
+	return 1; /* error */
+}
+
+static void pbcpush3(char x)
+{
+	pbcstack3[pbcpos3++] = x;
+}
+
+static int pbcpop3(char x)
+{
+	if (   ( x == ')' && pbcstack3[pbcpos3-1] == '(' )
+		|| ( x == ']' && pbcstack3[pbcpos3-1] == '[' )
+		|| ( x == '}' && pbcstack3[pbcpos3-1] == '{' )) {
+		pbcpos3--;
+		return 0;
+	}
+	return 1; /* error */
+}
+
 static int c_prevword(void)
 {
 	char *c = prev_word;