1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
%top {
/* Include this before everything else, for various large-file definitions */
#include "config.h"
}
/*
* We want a reentrant scanner.
*/
%option reentrant
/*
* We want to generate code that can be used by a reentrant parser
* generated by Bison or Berkeley YACC.
*/
%option bison-bridge
/*
* We don't read interactively from the terminal.
*/
%option never-interactive
/*
* We want to stop processing when we get to the end of the input.
*/
%option noyywrap
/*
* The type for the state we keep for the scanner (and parser).
*/
%option extra-type="protobuf_lang_state_t *"
/*
* Prefix scanner routines with "protobuf_lang" rather than "yy", so this scanner
* can coexist with other scanners.
*/
%option prefix="protobuf_lang"
/*
* We have to override the memory allocators so that we don't get
* "unused argument" warnings from the yyscanner argument (which
* we don't use, as we have a global memory allocator).
*
* We provide, as macros, our own versions of the routines generated by Flex,
* which just call malloc()/realloc()/free() (as the Flex versions do),
* discarding the extra argument.
*/
%option noyyalloc
%option noyyrealloc
%option noyyfree
%option yylineno
%option noinput
%option nounput
%{
/* protobuf_lang_scanner.l
*
* C Protocol Buffers Language Lexer (for *.proto files)
* Copyright 2019, Huang Qiangxiong <qiangxiong.huang@qq.com>
*
* SPDX-License-Identifier: GPL-2.0-or-later
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <glib.h>
#include "protobuf_lang_tree.h"
#include "protobuf_lang.h"
/*
* Disable diagnostics in the code generated by Flex.
*/
DIAG_OFF_FLEX
/*
* Sleazy hack to suppress compiler warnings in yy_fatal_error().
*/
#define YY_EXIT_FAILURE ((void)yyscanner, 2)
/*
* Macros for the allocators, to discard the extra argument.
*/
#define protobuf_langalloc(size, yyscanner) (void *)malloc(size)
#define protobuf_langrealloc(ptr, size, yyscanner) (void *)realloc((char *)(ptr), (size))
#define protobuf_langfree(ptr, yyscanner) free((char *)ptr)
int old_status;
/* error handling function defined in bison (*.y) file */
extern void
protobuf_langerrorv(void* yyscanner, protobuf_lang_state_t *state, const char *fmt, ...);
/* duplicate the text and keep the pointer in parser state for freeing later automatically */
static gchar*
strdup_and_store(void* yyscanner, const char* text);
%}
%x COMMENT
%%
/* operations or symbols (PT_ means PBL Token) */
\" return PT_QUOTE;
"(" return PT_LPAREN;
")" return PT_RPAREN;
"[" return PT_LBRACKET;
"]" return PT_RBRACKET;
"{" return PT_LCURLY;
"}" return PT_RCURLY;
"==" return PT_EQUAL;
"!=" return PT_NOTEQUAL;
"<>" return PT_NOTEQUAL2;
">=" return PT_GEQUAL;
"<=" return PT_LEQUAL;
"+=" return PT_ASSIGN_PLUS;
"=" return PT_ASSIGN;
"+" return PT_PLUS;
"-" return PT_MINUS;
"*" return PT_MULTIPLY;
"/" return PT_DIV;
"||" return PT_LOGIC_OR;
"|" return PT_OR;
"&&" return PT_LOGIC_AND;
"&" return PT_AND;
"!" return PT_NOT;
"~" return PT_NEG;
"^" return PT_XOR;
"<<" return PT_SHL;
">>" return PT_SHR;
"%" return PT_PERCENT;
"$" return PT_DOLLAR;
"?" return PT_COND;
";" return PT_SEMICOLON;
"." return PT_DOT;
"," return PT_COMMA;
":" return PT_COLON;
"<" return PT_LESS;
">" return PT_GREATER;
/* key words */
syntax yylval->sval = strdup_and_store(yyscanner, yytext); return PT_SYNTAX;
import yylval->sval = strdup_and_store(yyscanner, yytext); return PT_IMPORT;
weak yylval->sval = strdup_and_store(yyscanner, yytext); return PT_WEAK;
public yylval->sval = strdup_and_store(yyscanner, yytext); return PT_PUBLIC;
package yylval->sval = strdup_and_store(yyscanner, yytext); return PT_PACKAGE;
option yylval->sval = strdup_and_store(yyscanner, yytext); return PT_OPTION;
required yylval->sval = strdup_and_store(yyscanner, yytext); return PT_REQUIRED;
optional yylval->sval = strdup_and_store(yyscanner, yytext); return PT_OPTIONAL;
repeated yylval->sval = strdup_and_store(yyscanner, yytext); return PT_REPEATED;
oneof yylval->sval = strdup_and_store(yyscanner, yytext); return PT_ONEOF;
map yylval->sval = strdup_and_store(yyscanner, yytext); return PT_MAP;
reserved yylval->sval = strdup_and_store(yyscanner, yytext); return PT_RESERVED;
enum yylval->sval = strdup_and_store(yyscanner, yytext); return PT_ENUM;
group yylval->sval = strdup_and_store(yyscanner, yytext); return PT_GROUP;
extend yylval->sval = strdup_and_store(yyscanner, yytext); return PT_EXTEND;
extensions yylval->sval = strdup_and_store(yyscanner, yytext); return PT_EXTENSIONS;
message yylval->sval = strdup_and_store(yyscanner, yytext); return PT_MESSAGE;
service yylval->sval = strdup_and_store(yyscanner, yytext); return PT_SERVICE;
rpc yylval->sval = strdup_and_store(yyscanner, yytext); return PT_RPC;
stream yylval->sval = strdup_and_store(yyscanner, yytext); return PT_STREAM;
returns yylval->sval = strdup_and_store(yyscanner, yytext); return PT_RETURNS;
to yylval->sval = strdup_and_store(yyscanner, yytext); return PT_TO;
/* key values */
["']proto2["'] yylval->sval = strdup_and_store(yyscanner, yytext); return PT_PROTO2;
["']proto3["'] yylval->sval = strdup_and_store(yyscanner, yytext); return PT_PROTO3;
/* intLit values */
0|[1-9][0-9]* yylval->ival = atoi(yytext); return PT_DECIMALLIT;
0[0-7]* sscanf(yytext+1, "%o", &yylval->ival); return PT_OCTALLIT;
0[xX][0-9a-fA-F]+ sscanf(yytext+2, "%x", &yylval->ival); return PT_HEXLIT;
/* Using extended identifier because we care only about position */
[a-zA-Z0-9_.+-]+ yylval->sval = strdup_and_store(yyscanner, yytext); return PT_IDENT;
\"(\\.|\"\"|[^"\n"])*\" yylval->sval = g_strndup(yytext + 1, strlen(yytext) - 2); return PT_STRLIT;
\'(\\.|\'\'|[^"\n"])*\' yylval->sval = g_strndup(yytext + 1, strlen(yytext) - 2); return PT_STRLIT;
/* comments */
"//"[^\r\n]*
"/*" { old_status = YY_START; BEGIN COMMENT; }
<COMMENT>"*/" { BEGIN old_status; }
<COMMENT>([^*]|\n)+|.
/* space & tab */
[ \t\r\n]
/* prevent flex jam */
. { protobuf_langerrorv(yyscanner, protobuf_langget_extra(yyscanner), "unexpected token in proto file!\n"); }
%%
static gchar*
strdup_and_store(void* yyscanner, const char* text) {
return pbl_store_string_token(protobuf_langget_extra(yyscanner), g_strdup(text));
}
/*
* Turn diagnostics back on, so we check the code that we've written.
*/
DIAG_ON_FLEX
|
