diff options
author | Huang Qiangxiong <qiangxiong.huang@qq.com> | 2020-12-27 13:57:17 +0800 |
---|---|---|
committer | AndersBroman <a.broman58@gmail.com> | 2020-12-27 11:32:10 +0000 |
commit | cd2d35c1d2b10fe2916469544cc2951a8979a4dc (patch) | |
tree | 12472adbf303c2e4b2d052c7740924d3a5655cb2 | |
parent | 5778b2403ea4fec9762f961a6ba4dbf33fd2ee05 (diff) |
Protobuf: fix bugs that parsing complex syntax .proto files
Some .proto files contain complex syntax that does not be described in protobuf official site
(https://developers.google.com/protocol-buffers/docs/reference/proto3-spec).
1. Update 'epan/protobuf_lang_parser.lemon' to:
1) Support complex option names format (EBNF):
optionName = ( ident | "(" fullIdent ")" ) { "." ( ident | "(" fullIdent ")" ) }
for example, "option (complex_opt2).(grault) = 654;".
2) Make enum body support 'reserved' section (EBNF):
enumBody = "{" { reserved | option | enumField | emptyStatement } "}"
3) Allow the value of field or enumValue option to be "{ ... }" other than constant:
enumValueOption = optionName "=" ( constant | customOptionValue ) ";"
fieldOption = optionName "=" ( constant | customOptionValue ) ";"
4) Allow 'group' section missing 'label' (for example, in 'oneof' section).
5) Make 'oneof' section support 'option' and 'group' sections (BNF):
oneof = "oneof" oneofName "{" { oneofField | option | group | emptyStatement } "}"
6) Ignore unused 'extend' section.
7) Fix the bug of one string being splitted into multi-lines.
2. Update 'epan/protobuf_lang_tree.c' to:
8) Fix the bug of parsing repeated option.
3. Update 'test/suite_dissection.py' to add test case for parsing complex syntax .proto files:
test/protobuf_lang_files/complex_proto_files/unittest_custom_options.proto
test/protobuf_lang_files/complex_proto_files/complex_syntax.proto
and dependency files:
test/protobuf_lang_files/well_know_types/google/protobuf/any.proto
test/protobuf_lang_files/well_know_types/google/protobuf/descriptor.proto
Refer to issue #17046
-rw-r--r-- | epan/protobuf_lang_parser.lemon | 62 | ||||
-rw-r--r-- | epan/protobuf_lang_tree.c | 12 | ||||
-rw-r--r-- | test/protobuf_lang_files/complex_proto_files/complex_syntax.proto | 82 | ||||
-rw-r--r-- | test/protobuf_lang_files/complex_proto_files/unittest_custom_options.proto | 395 | ||||
-rw-r--r-- | test/protobuf_lang_files/well_know_types/google/protobuf/any.proto | 22 | ||||
-rw-r--r-- | test/protobuf_lang_files/well_know_types/google/protobuf/descriptor.proto | 282 | ||||
-rw-r--r-- | test/suite_dissection.py | 20 |
7 files changed, 865 insertions, 10 deletions
diff --git a/epan/protobuf_lang_parser.lemon b/epan/protobuf_lang_parser.lemon index 36f9edb506..4aa1b9b85d 100644 --- a/epan/protobuf_lang_parser.lemon +++ b/epan/protobuf_lang_parser.lemon @@ -190,11 +190,17 @@ option ::= PT_OPTION optionName PT_ASSIGN constant PT_SEMICOLON. option ::= PT_OPTION optionName PT_ASSIGN customOptionValue PT_SEMICOLON. /* v2/v3: optionName = ( ident | "(" fullIdent ")" ) { "." ident } */ -optionName ::= exIdent. -optionName(A) ::= PT_LPAREN exIdent(B) PT_RPAREN. +/* Offical PBL bugfix: optionName = ( ident | "(" fullIdent ")" ) { "." ( ident | "(" fullIdent ")" ) } */ +extIdentInParentheses(A) ::= PT_LPAREN exIdent(B) PT_RPAREN. { A = B; A->v = pbl_store_string_token(state, g_strconcat("(", B->v, ")", NULL)); } -optionName(A) ::= PT_LPAREN exIdent(B) PT_RPAREN exIdent(C). /* Note that the exIdent contains "." */ - { A = B; A->v = pbl_store_string_token(state, g_strconcat("(", B->v, ")", C->v, NULL)); } +optionName ::= exIdent. +optionName ::= extIdentInParentheses. +optionName(A) ::= optionName(B) exIdent(C). // Note that the exIdent contains "." + { A = B; A->v = pbl_store_string_token(state, g_strconcat(B->v, C->v, NULL)); } +optionName(A) ::= optionName(B) PT_DOT extIdentInParentheses(C). + { A = B; A->v = pbl_store_string_token(state, g_strconcat(B->v, ".", C->v, NULL)); } +optionName(A) ::= optionName(B) extIdentInParentheses(C). + { A = B; A->v = pbl_store_string_token(state, g_strconcat(B->v, ".", C->v, NULL)); } /* Allow format which not defined in offical PBL specification like: option (google.api.http) = { post: "/v3alpha/kv/put" body: "*" }; @@ -203,9 +209,30 @@ optionName(A) ::= PT_LPAREN exIdent(B) PT_RPAREN exIdent(C). /* Note that the e */ customOptionValue ::= PT_LCURLY customOptionBody PT_RCURLY. +/* The formal EBNF of customOptionBody seems to be */ +/* +customOptionBody ::= . +customOptionBody ::= customOptionBody optionField. +customOptionBody ::= customOptionBody PT_COMMA optionField. +customOptionBody ::= customOptionBody PT_SEMICOLON optionField. + +optionField ::= optionName PT_COLON constant. +optionField ::= optionName PT_COLON customOptionValue. +optionField ::= optionName customOptionValue. +optionField ::= optionName PT_COLON array. + +array ::= PT_LBRACKET arrayBody PT_RBRACKET. +arrayBodyConst ::= constant. +arrayBodyConst ::= arrayBody PT_COMMA constant. +arrayBodyCustom ::= customOptionValue. +arrayBodyCustom ::= arrayBody PT_COMMA customOptionValue. +arrayBody ::= arrayBodyConst. +arrayBody ::= arrayBodyCustom. +*/ +/* but for handling unexpected situations, we still use following EBNF */ customOptionBody ::= . customOptionBody ::= customOptionBody exIdent. -customOptionBody ::= customOptionBody strLit. +customOptionBody ::= customOptionBody PT_STRLIT. customOptionBody ::= customOptionBody symbolsWithoutCurly. customOptionBody ::= customOptionBody intLit. customOptionBody ::= customOptionBody customOptionValue. @@ -275,7 +302,9 @@ enum(A) ::= PT_ENUM enumName(B) PT_LCURLY enumBody(C) PT_RCURLY. { A = C; pbl_set_node_name(A, B->ln, B->v); } /* v2/v3: enumBody = "{" { option | enumField | emptyStatement } "}" */ +/* Offical PBL bugfix: enumBody = "{" { reserved | option | enumField | emptyStatement } "}" */ enumBody(A) ::= . { A = pbl_create_node(state->file, CUR_LINENO, PBL_ENUM, NAME_TO_BE_SET); } +enumBody ::= enumBody reserved. enumBody ::= enumBody option. enumBody(A) ::= enumBody(B) enumField(C). { A = B; pbl_add_child(A, C); } enumBody ::= enumBody emptyStatement. @@ -296,7 +325,9 @@ enumValueOptions ::= enumValueOption. enumValueOptions ::= enumValueOptions PT_COMMA enumValueOption. /* v2/v3: enumValueOption = optionName "=" constant */ +/* Offical PBL bugfix: enumValueOption = optionName "=" ( constant | customOptionValue ) ";" */ enumValueOption ::= optionName PT_ASSIGN constant. +enumValueOption ::= optionName PT_ASSIGN customOptionValue. /* v2: service = "service" serviceName "{" { option | rpc | stream | emptyStatement } "}" */ /* v3: service = "service" serviceName "{" { option | rpc | emptyStatement } "}" */ @@ -373,21 +404,30 @@ fieldOptions(A) ::= fieldOptions(B) PT_COMMA fieldOption(C). { A = B; pbl_add_child(A, C); } /* v2/v3: fieldOption = optionName "=" constant */ +/* Offical PBL bugfix: fieldOption = optionName "=" ( constant | customOptionValue ) ";" */ fieldOption(A) ::= optionName(B) PT_ASSIGN constant(C). { A = pbl_create_option_node(state->file, B->ln, B->v, C); } +fieldOption(A) ::= optionName(B) PT_ASSIGN customOptionValue. + { A = pbl_create_option_node(state->file, B->ln, B->v, pbl_store_string_token(state, g_strdup("{ ... }"))); } /* v2 only: group = label "group" groupName "=" fieldNumber messageBody */ +/* Offical PBL bugfix: there is no label if the 'group' is a member of oneof body */ +group(A) ::= PT_GROUP groupName(B) PT_ASSIGN fieldNumber PT_LCURLY messageBody(C) PT_RCURLY. + { A = C; pbl_set_node_name(A, B->ln, B->v); } group(A) ::= label PT_GROUP groupName(B) PT_ASSIGN fieldNumber PT_LCURLY messageBody(C) PT_RCURLY. { A = C; pbl_set_node_name(A, B->ln, B->v); } groupName ::= exIdent. /* v2/v3: oneof = "oneof" oneofName "{" { oneofField | emptyStatement } "}" */ +/* Offical PBL bugfix: oneof = "oneof" oneofName "{" { oneofField | option | group | emptyStatement } "}" */ oneof(A) ::= PT_ONEOF oneofName(B) PT_LCURLY oneofBody(C) PT_RCURLY. { A = C; pbl_set_node_name(A, B->ln, B->v); } oneofBody(A) ::= . { A = pbl_create_node(state->file, CUR_LINENO, PBL_ONEOF, NAME_TO_BE_SET); } oneofBody(A) ::= oneofBody(B) oneofField(C). { A = B; pbl_add_child(A, C); } +oneofBody ::= oneofBody option. +oneofBody ::= oneofBody group. oneofBody ::= oneofBody emptyStatement. /* v2/v3: oneofField = type fieldName "=" fieldNumber [ "[" fieldOptions "]" ] ";" */ @@ -436,11 +476,13 @@ Note that there is an error in BNF definition about reserved fieldName. It's str quoteFieldNames ::= strLit. quoteFieldNames ::= quoteFieldNames PT_COMMA strLit. -/* v2 only: extend = "extend" messageType "{" {field | group | emptyStatement} "}" */ -extend(A) ::= PT_EXTEND(X) messageType(B) PT_LCURLY extendBody(C) PT_RCURLY. - { A = C; pbl_set_node_name(A, X->ln, pbl_store_string_token(state, g_strconcat(B, "Extend", NULL))); } +/* v2/v3: extend = "extend" messageType "{" {field | group | emptyStatement} "}" +Note that creating custom options uses extensions, which are permitted only for custom options in proto3. +We don't use custom options while parsing packet, so we just ignore the 'extend'. +*/ +extend(A) ::= PT_EXTEND messageType PT_LCURLY extendBody(B) PT_RCURLY. + { A = NULL; pbl_free_node(B); } -/* v2 only */ extendBody(A) ::= . { A = pbl_create_node(state->file, CUR_LINENO, PBL_MESSAGE, NAME_TO_BE_SET); } extendBody(A) ::= extendBody(B) field(C). { A = B; pbl_add_child(A, C); } extendBody(A) ::= extendBody(B) group(C). { A = B; pbl_add_child(A, C); } @@ -481,6 +523,8 @@ constant(A) ::= PT_MINUS exIdent(B). { A = pbl_store_string_token(state, g_strco exIdent ::= PT_IDENT. strLit(A) ::= PT_STRLIT(B). { A = pbl_store_string_token(state, g_strndup(B->v + 1, strlen(B->v) - 2)); } +/* support one string being splitted into multi-lines */ +strLit(A) ::= strLit(B) PT_STRLIT(C). { A = pbl_store_string_token(state, g_strconcat(B, g_strndup(C->v + 1, strlen(C->v) - 2), NULL)); } %code { diff --git a/epan/protobuf_lang_tree.c b/epan/protobuf_lang_tree.c index 25c9d2e1ac..b269ad3439 100644 --- a/epan/protobuf_lang_tree.c +++ b/epan/protobuf_lang_tree.c @@ -1009,7 +1009,17 @@ pbl_add_child(pbl_node_t* parent, pbl_node_t* child) } node = (pbl_node_t*) g_hash_table_lookup(parent->children_by_name, child->name); - if (node && child->file && parent->file + if (node && node->nodetype == PBL_OPTION && child->nodetype == PBL_OPTION + && ((pbl_option_descriptor_t*)node)->value && ((pbl_option_descriptor_t*)child)->value) { + /* repeated option can be set many times like: + string fieldWithComplexOption5 = 5 [(rules).repeated_int = 1, (rules).repeated_int = 2]; + we just merge the old value and new value in format /old_value "," new_value/. + */ + gchar* oval = ((pbl_option_descriptor_t*)node)->value; + gchar* nval = ((pbl_option_descriptor_t*)child)->value; + ((pbl_option_descriptor_t*)child)->value = g_strconcat(oval, ",", nval, NULL); + g_free(nval); + } else if (node && child->file && parent->file && child->file->pool && child->file->pool->error_cb) { child->file->pool->error_cb( "Protobuf: Warning: \"%s\" of [%s:%d] is already defined in file [%s:%d].\n", diff --git a/test/protobuf_lang_files/complex_proto_files/complex_syntax.proto b/test/protobuf_lang_files/complex_proto_files/complex_syntax.proto new file mode 100644 index 0000000000..359ead4aa6 --- /dev/null +++ b/test/protobuf_lang_files/complex_proto_files/complex_syntax.proto @@ -0,0 +1,82 @@ +// Test more complex syntax of *.proto files. +syntax = "proto3"; +package wireshark.protobuf.test.complex.syntax; +import "google/protobuf/descriptor.proto"; + +// equal to "testing.multiline.strings" +option java_package = "testing." + 'multiline.' +"strings"; + +// user defined options for messages +extend google.protobuf.MessageOptions { + bool disabled = 1071; + bool ignored = 1072; + TestMultiLinesOption mlinemsg = 1073; +} + +// user defined options for oneof types +extend google.protobuf.OneofOptions { + bool required = 1071; +} + +// user defined options for fields +extend google.protobuf.FieldOptions { + FieldRules rules = 1071; +} +// test extend google.protobuf.FieldOptions twice +extend google.protobuf.FieldOptions { + string multilines = 1072; +} + +message FieldRules { + oneof type { + BoolRules bool = 13; + StringRules string = 14; + } + repeated uint32 repeated_uint = 15; +} + +message StringRules { + uint64 min_bytes = 2; + BoolRules morebool = 3; + string astr = 4; +} + +message BoolRules { + bool const = 1; + repeated bool repeated_bool = 2; + repeated int32 repeated_int = 3; +} + +message TestMultiLinesOption { + string mlines = 1; +} + +message ComplexDefinedMessage { + option (mlinemsg).mlines = "first line" + "second line"; + + // test complex field options + string fieldWithComplexOption1 = 1 [(wireshark.protobuf.test.complex.syntax.rules).string = {min_bytes: 1}]; + string fieldWithComplexOption2 = 2 [(rules).string = {min_bytes: 2 astr: "abc" }]; + string fieldWithComplexOption3 = 3 [(rules).string.morebool = {const: true, repeated_bool: [false, true], repeated_int: [1, 2]}]; + string fieldWithComplexOption4 = 4 [(rules).string = {min_bytes: 1; morebool { const: true }}]; + string fieldWithComplexOption5 = 5 [(rules).repeated_uint = 1, (rules).repeated_uint = 2]; + + // test oneof custom option + oneof oneofWithOption { + option (wireshark.protobuf.test.complex.syntax.required) = true; + int32 field1 = 11; + string field2 = 12; + } + + // test multilines strings + uint32 fieldWithMultilineStringOption = 20 [(wireshark.protobuf.test.complex.syntax.multilines) = "first line" + 'Second line' ]; +} + +// add this message for testing whether this file was successfully parsed +message TestFileParsed { + optional int32 last_field_for_wireshark_test = 1; +} diff --git a/test/protobuf_lang_files/complex_proto_files/unittest_custom_options.proto b/test/protobuf_lang_files/complex_proto_files/unittest_custom_options.proto new file mode 100644 index 0000000000..d04c9b7289 --- /dev/null +++ b/test/protobuf_lang_files/complex_proto_files/unittest_custom_options.proto @@ -0,0 +1,395 @@ +// This file is from https://github.com/protocolbuffers/protobuf/blob/3.14.x/src/google/protobuf/unittest_custom_options.proto +// To reduce the file size, some comments have been removed. +// Message 'TestFileParsed' is added at the end of file for testing whether this file was successfully parsed. + +syntax = "proto2"; + +option cc_generic_services = true; +option java_generic_services = true; +option py_generic_services = true; + +option (file_opt1) = 9876543210; + +import "google/protobuf/any.proto"; +import "google/protobuf/descriptor.proto"; + +package protobuf_unittest; + +// Some simple test custom options of various types. +extend google.protobuf.FileOptions { + optional uint64 file_opt1 = 7736974; +} + +extend google.protobuf.MessageOptions { + optional int32 message_opt1 = 7739036; +} + +extend google.protobuf.FieldOptions { + optional fixed64 field_opt1 = 7740936; + optional int32 field_opt2 = 7753913 [default = 42]; +} + +extend google.protobuf.OneofOptions { + optional int32 oneof_opt1 = 7740111; +} + +extend google.protobuf.EnumOptions { + optional sfixed32 enum_opt1 = 7753576; +} + +extend google.protobuf.EnumValueOptions { + optional int32 enum_value_opt1 = 1560678; +} + +extend google.protobuf.ServiceOptions { + optional sint64 service_opt1 = 7887650; +} + +enum MethodOpt1 { + METHODOPT1_VAL1 = 1; + METHODOPT1_VAL2 = 2; +} + +extend google.protobuf.MethodOptions { + optional MethodOpt1 method_opt1 = 7890860; +} + +message TestMessageWithCustomOptions { + option message_set_wire_format = false; + option (message_opt1) = -56; + + optional string field1 = 1 [ctype = CORD, (field_opt1) = 8765432109]; + + oneof AnOneof { + option (oneof_opt1) = -99; + + int32 oneof_field = 2; + } + + enum AnEnum { + option (enum_opt1) = -789; + + ANENUM_VAL1 = 1; + ANENUM_VAL2 = 2 [(enum_value_opt1) = 123]; + } +} + +message CustomOptionFooRequest {} + +message CustomOptionFooResponse {} + +message CustomOptionFooClientMessage {} + +message CustomOptionFooServerMessage {} + +service TestServiceWithCustomOptions { + option (service_opt1) = -9876543210; + + rpc Foo(CustomOptionFooRequest) returns (CustomOptionFooResponse) { + option (method_opt1) = METHODOPT1_VAL2; + } +} + +message DummyMessageContainingEnum { + enum TestEnumType { + TEST_OPTION_ENUM_TYPE1 = 22; + TEST_OPTION_ENUM_TYPE2 = -23; + } +} + +message DummyMessageInvalidAsOptionType {} + +extend google.protobuf.MessageOptions { + optional bool bool_opt = 7706090; + optional int32 int32_opt = 7705709; + optional int64 int64_opt = 7705542; + optional uint32 uint32_opt = 7704880; + optional uint64 uint64_opt = 7702367; + optional sint32 sint32_opt = 7701568; + optional sint64 sint64_opt = 7700863; + optional fixed32 fixed32_opt = 7700307; + optional fixed64 fixed64_opt = 7700194; + optional sfixed32 sfixed32_opt = 7698645; + optional sfixed64 sfixed64_opt = 7685475; + optional float float_opt = 7675390; + optional double double_opt = 7673293; + optional string string_opt = 7673285; + optional bytes bytes_opt = 7673238; + optional DummyMessageContainingEnum.TestEnumType enum_opt = 7673233; + optional DummyMessageInvalidAsOptionType message_type_opt = 7665967; +} + +message CustomOptionMinIntegerValues { + option (bool_opt) = false; + option (int32_opt) = -0x80000000; + option (int64_opt) = -0x8000000000000000; + option (uint32_opt) = 0; + option (uint64_opt) = 0; + option (sint32_opt) = -0x80000000; + option (sint64_opt) = -0x8000000000000000; + option (fixed32_opt) = 0; + option (fixed64_opt) = 0; + option (sfixed32_opt) = -0x80000000; + option (sfixed64_opt) = -0x8000000000000000; +} + +message CustomOptionMaxIntegerValues { + option (bool_opt) = true; + option (int32_opt) = 0x7FFFFFFF; + option (int64_opt) = 0x7FFFFFFFFFFFFFFF; + option (uint32_opt) = 0xFFFFFFFF; + option (uint64_opt) = 0xFFFFFFFFFFFFFFFF; + option (sint32_opt) = 0x7FFFFFFF; + option (sint64_opt) = 0x7FFFFFFFFFFFFFFF; + option (fixed32_opt) = 0xFFFFFFFF; + option (fixed64_opt) = 0xFFFFFFFFFFFFFFFF; + option (sfixed32_opt) = 0x7FFFFFFF; + option (sfixed64_opt) = 0x7FFFFFFFFFFFFFFF; +} + +message CustomOptionOtherValues { + option (int32_opt) = -100; // To test sign-extension. + option (float_opt) = 12.3456789; + option (double_opt) = 1.234567890123456789; + option (string_opt) = "Hello, \"World\""; + option (bytes_opt) = "Hello\0World"; + option (enum_opt) = TEST_OPTION_ENUM_TYPE2; +} + +message SettingRealsFromPositiveInts { + option (float_opt) = 12; + option (double_opt) = 154; +} + +message SettingRealsFromNegativeInts { + option (float_opt) = -12; + option (double_opt) = -154; +} + +message ComplexOptionType1 { + optional int32 foo = 1; + optional int32 foo2 = 2; + optional int32 foo3 = 3; + repeated int32 foo4 = 4; + + extensions 100 to max; +} + +message ComplexOptionType2 { + optional ComplexOptionType1 bar = 1; + optional int32 baz = 2; + + message ComplexOptionType4 { + optional int32 waldo = 1; + + extend google.protobuf.MessageOptions { + optional ComplexOptionType4 complex_opt4 = 7633546; + } + } + + optional ComplexOptionType4 fred = 3; + repeated ComplexOptionType4 barney = 4; + + extensions 100 to max; +} + +message ComplexOptionType3 { + optional int32 qux = 1; + + optional group ComplexOptionType5 = 2 { + optional int32 plugh = 3; + } +} + +extend ComplexOptionType1 { + optional int32 quux = 7663707; + optional ComplexOptionType3 corge = 7663442; +} + +extend ComplexOptionType2 { + optional int32 grault = 7650927; + optional ComplexOptionType1 garply = 7649992; +} + +extend google.protobuf.MessageOptions { + optional protobuf_unittest.ComplexOptionType1 complex_opt1 = 7646756; + optional ComplexOptionType2 complex_opt2 = 7636949; + optional ComplexOptionType3 complex_opt3 = 7636463; + optional group ComplexOpt6 = 7595468 { + optional int32 xyzzy = 7593951; + } +} + +message VariousComplexOptions { + option (.protobuf_unittest.complex_opt1).foo = 42; + option (protobuf_unittest.complex_opt1).(.protobuf_unittest.quux) = 324; + option (.protobuf_unittest.complex_opt1).(protobuf_unittest.corge).qux = 876; + option (protobuf_unittest.complex_opt1).foo4 = 99; + option (protobuf_unittest.complex_opt1).foo4 = 88; + option (complex_opt2).baz = 987; + option (complex_opt2).(grault) = 654; + option (complex_opt2).bar.foo = 743; + option (complex_opt2).bar.(quux) = 1999; + option (complex_opt2).bar.(protobuf_unittest.corge).qux = 2008; + option (complex_opt2).(garply).foo = 741; + option (complex_opt2).(garply).(.protobuf_unittest.quux) = 1998; + option (complex_opt2).(protobuf_unittest.garply).(corge).qux = 2121; + option (ComplexOptionType2.ComplexOptionType4.complex_opt4).waldo = 1971; + option (complex_opt2).fred.waldo = 321; + option (complex_opt2).barney = { + waldo: 101 + }; + option (complex_opt2).barney = { + waldo: 212 + }; + option (protobuf_unittest.complex_opt3).qux = 9; + option (complex_opt3).complexoptiontype5.plugh = 22; + option (complexopt6).xyzzy = 24; +} + +message AggregateMessageSet { + option message_set_wire_format = true; + + extensions 4 to max; +} + +message AggregateMessageSetElement { + extend AggregateMessageSet { + optional AggregateMessageSetElement message_set_extension = 15447542; + } + optional string s = 1; +} + +message Aggregate { + optional int32 i = 1; + optional string s = 2; + + optional Aggregate sub = 3; + + optional google.protobuf.FileOptions file = 4; + extend google.protobuf.FileOptions { + optional Aggregate nested = 15476903; + } + + optional AggregateMessageSet mset = 5; + + optional google.protobuf.Any any = 6; +} + +extend google.protobuf.FileOptions { + optional Aggregate fileopt = 15478479; +} +extend google.protobuf.MessageOptions { + optional Aggregate msgopt = 15480088; +} +extend google.protobuf.FieldOptions { + optional Aggregate fieldopt = 15481374; +} +extend google.protobuf.EnumOptions { + optional Aggregate enumopt = 15483218; +} +extend google.protobuf.EnumValueOptions { + optional Aggregate enumvalopt = 15486921; +} +extend google.protobuf.ServiceOptions { + optional Aggregate serviceopt = 15497145; +} +extend google.protobuf.MethodOptions { + optional Aggregate methodopt = 15512713; +} + +option (fileopt) = { + s: 'FileAnnotation' + i: 100 + + sub { s: 'NestedFileAnnotation' } + + file { + [protobuf_unittest.fileopt] { s: 'FileExtensionAnnotation' } + } + + mset { + [protobuf_unittest.AggregateMessageSetElement.message_set_extension] { + s: 'EmbeddedMessageSetElement' + } + } + + any { + [type.googleapis.com/protobuf_unittest.AggregateMessageSetElement] { + s: 'EmbeddedMessageSetElement' + } + } +}; + +message AggregateMessage { + option (msgopt) = { + i: 101 + s: 'MessageAnnotation' + }; + + optional int32 fieldname = 1 [(fieldopt) = { s: 'FieldAnnotation' }]; +} + +service AggregateService { + option (serviceopt) = { + s: 'ServiceAnnotation' + }; + + rpc Method(AggregateMessage) returns (AggregateMessage) { + option (methodopt) = { + s: 'MethodAnnotation' + }; + } +} + +enum AggregateEnum { + option (enumopt) = { + s: 'EnumAnnotation' + }; + + VALUE = 1 [(enumvalopt) = { s: 'EnumValueAnnotation' }]; +} + +message NestedOptionType { + message NestedMessage { + option (message_opt1) = 1001; + + optional int32 nested_field = 1 [(field_opt1) = 1002]; + } + enum NestedEnum { + option (enum_opt1) = 1003; + + NESTED_ENUM_VALUE = 1 [(enum_value_opt1) = 1004]; + } + extend google.protobuf.FileOptions { + optional int32 nested_extension = 7912573 [(field_opt2) = 1005]; + } +} + +message OldOptionType { + enum TestEnum { OLD_VALUE = 0; } + required TestEnum value = 1; +} + +message NewOptionType { + enum TestEnum { + OLD_VALUE = 0; + NEW_VALUE = 1; + } + required TestEnum value = 1; +} + +extend google.protobuf.MessageOptions { + optional OldOptionType required_enum_opt = 106161807; +} + +message TestMessageWithRequiredEnumOption { + option (required_enum_opt) = { + value: OLD_VALUE + }; +} + +// add this message for testing whether this file was successfully parsed +message TestFileParsed { + optional int32 last_field_for_wireshark_test = 1; +} diff --git a/test/protobuf_lang_files/well_know_types/google/protobuf/any.proto b/test/protobuf_lang_files/well_know_types/google/protobuf/any.proto new file mode 100644 index 0000000000..8ff870dd82 --- /dev/null +++ b/test/protobuf_lang_files/well_know_types/google/protobuf/any.proto @@ -0,0 +1,22 @@ +// This file is from https://github.com/protocolbuffers/protobuf/blob/3.14.x/src/google/protobuf/any.proto +// To reduce the file size, some comments have been removed. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option go_package = "google.golang.org/protobuf/types/known/anypb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "AnyProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +message Any { + // A URL/resource name that uniquely identifies the type of the serialized + // protocol buffer message. + string type_url = 1; + + // Must be a valid serialized protocol buffer of the above specified type. + bytes value = 2; +} diff --git a/test/protobuf_lang_files/well_know_types/google/protobuf/descriptor.proto b/test/protobuf_lang_files/well_know_types/google/protobuf/descriptor.proto new file mode 100644 index 0000000000..ece32e1571 --- /dev/null +++ b/test/protobuf_lang_files/well_know_types/google/protobuf/descriptor.proto @@ -0,0 +1,282 @@ +// This file is from https://github.com/protocolbuffers/protobuf/blob/3.14.x/src/google/protobuf/descriptor.proto +// To reduce the file size, some comments have been removed. + +syntax = "proto2"; + +package google.protobuf; + +option go_package = "google.golang.org/protobuf/types/descriptorpb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; + +option optimize_for = SPEED; + +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +message FileDescriptorProto { + optional string name = 1; + optional string package = 2; + repeated string dependency = 3; + repeated int32 public_dependency = 10; + repeated int32 weak_dependency = 11; + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + optional FileOptions options = 8; + optional SourceCodeInfo source_code_info = 9; + optional string syntax = 12; +} + +message DescriptorProto { + optional string name = 1; + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + optional ExtensionRangeOptions options = 3; + } + repeated ExtensionRange extension_range = 5; + repeated OneofDescriptorProto oneof_decl = 8; + optional MessageOptions options = 7; + + message ReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } + repeated ReservedRange reserved_range = 9; + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message FieldDescriptorProto { + enum Type { + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; + TYPE_MESSAGE = 11; + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; + TYPE_SINT64 = 18; + } + + enum Label { + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + } + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + optional Type type = 5; + optional string type_name = 6; + optional string extendee = 2; + optional string default_value = 7; + optional int32 oneof_index = 9; + optional string json_name = 10; + optional FieldOptions options = 8; + optional bool proto3_optional = 17; +} + +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + +message EnumDescriptorProto { + optional string name = 1; + repeated EnumValueDescriptorProto value = 2; + optional EnumOptions options = 3; + + message EnumReservedRange { + optional int32 start = 1; + optional int32 end = 2; + } + + repeated EnumReservedRange reserved_range = 4; + repeated string reserved_name = 5; +} + +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + optional EnumValueOptions options = 3; +} + +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + optional ServiceOptions options = 3; +} + +message MethodDescriptorProto { + optional string name = 1; + optional string input_type = 2; + optional string output_type = 3; + optional MethodOptions options = 4; + optional bool client_streaming = 5 [default = false]; + optional bool server_streaming = 6 [default = false]; +} + +message FileOptions { + optional string java_package = 1; + optional string java_outer_classname = 8; + optional bool java_multiple_files = 10 [default = false]; + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + optional bool java_string_check_utf8 = 27 [default = false]; + enum OptimizeMode { + SPEED = 1; + CODE_SIZE = 2; + LITE_RUNTIME = 3; + } + optional OptimizeMode optimize_for = 9 [default = SPEED]; + optional string go_package = 11; + optional bool cc_generic_services = 16 [default = false]; + optional bool java_generic_services = 17 [default = false]; + optional bool py_generic_services = 18 [default = false]; + optional bool php_generic_services = 42 [default = false]; + optional bool deprecated = 23 [default = false]; + optional bool cc_enable_arenas = 31 [default = true]; + optional string objc_class_prefix = 36; + optional string csharp_namespace = 37; + optional string swift_prefix = 39; + optional string php_class_prefix = 40; + optional string php_namespace = 41; + optional string php_metadata_namespace = 44; + optional string ruby_package = 45; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + reserved 38; +} + +message MessageOptions { + optional bool message_set_wire_format = 1 [default = false]; + optional bool no_standard_descriptor_accessor = 2 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool map_entry = 7; + reserved 8; + reserved 9; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message FieldOptions { + optional CType ctype = 1 [default = STRING]; + enum CType { + STRING = 0; + CORD = 1; + STRING_PIECE = 2; + } + optional bool packed = 2; + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + JS_NORMAL = 0; + JS_STRING = 1; + JS_NUMBER = 2; + } + optional bool lazy = 5 [default = false]; + optional bool deprecated = 3 [default = false]; + optional bool weak = 10 [default = false]; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; + + reserved 4; +} + +message OneofOptions { + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message EnumOptions { + optional bool allow_alias = 2; + optional bool deprecated = 3 [default = false]; + reserved 5; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message EnumValueOptions { + optional bool deprecated = 1 [default = false]; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message ServiceOptions { + optional bool deprecated = 33 [default = false]; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message MethodOptions { + optional bool deprecated = 33 [default = false]; + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; + IDEMPOTENT = 2; + } + optional IdempotencyLevel idempotency_level = 34 + [default = IDEMPOTENCY_UNKNOWN]; + repeated UninterpretedOption uninterpreted_option = 999; + extensions 1000 to max; +} + +message UninterpretedOption { + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +message SourceCodeInfo { + repeated Location location = 1; + message Location { + repeated int32 path = 1 [packed = true]; + repeated int32 span = 2 [packed = true]; + optional string leading_comments = 3; + optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +message GeneratedCodeInfo { + repeated Annotation annotation = 1; + message Annotation { + repeated int32 path = 1 [packed = true]; + optional string source_file = 2; + optional int32 begin = 3; + optional int32 end = 4; + } +} diff --git a/test/suite_dissection.py b/test/suite_dissection.py index c509a77c04..29fe232387 100644 --- a/test/suite_dissection.py +++ b/test/suite_dissection.py @@ -225,6 +225,26 @@ class case_dissect_protobuf(subprocesstest.SubprocessTestCase): )) self.assertTrue(self.grepOutput('tutorial.AddressBook')) + def test_protobuf_complex_syntax(self, cmd_tshark, features, dirs, capture_file): + '''Test Protobuf parsing complex syntax .proto files''' + well_know_types_dir = os.path.join(dirs.protobuf_lang_files_dir, 'well_know_types').replace('\\', '/') + complex_proto_files_dir = os.path.join(dirs.protobuf_lang_files_dir, 'complex_proto_files').replace('\\', '/') + self.assertRun((cmd_tshark, + '-r', capture_file('protobuf_udp_addressbook_with_image_ts.pcapng'), + '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(well_know_types_dir, 'FALSE'), + '-o', 'uat:protobuf_search_paths: "{}","{}"'.format(complex_proto_files_dir, 'TRUE'), + '-o', 'protobuf.preload_protos: TRUE', + '-o', 'protobuf.pbf_as_hf: TRUE', + '-Y', 'pbf.wireshark.protobuf.test.complex.syntax.TestFileParsed.last_field_for_wireshark_test' + ' && pbf.protobuf_unittest.TestFileParsed.last_field_for_wireshark_test', + )) + # the output must be empty and not contain something like: + # tshark: "pbf.xxx.TestFileParsed.last_field_for_wireshark_test" is neither a field nor a protocol name. + # or + # tshark: Protobuf: Error(s) + self.assertFalse(self.grepOutput('.last_field_for_wireshark_test')) + self.assertFalse(self.grepOutput('Protobuf: Error')) + @fixtures.mark_usefixtures('test_env') @fixtures.uses_fixtures class case_dissect_tcp(subprocesstest.SubprocessTestCase): |