GRPC: Add support for gRPC-Web

Supporting both application/grpc-web and application/grpc-web-text.
Add test case for grpc-web(-text).

close #17939

1 files changed, 168 insertions, 53 deletions

diff --git a/epan/dissectors/packet-grpc.c b/epan/dissectors/packet-grpc.c
index 93dccddc40..388fcd5da5 100644
--- a/epan/dissectors/packet-grpc.c
+++ b/epan/dissectors/packet-grpc.c
@@ -1,6 +1,6 @@
 /* packet-grpc.c
  * Routines for GRPC dissection
- * Copyright 2017, Huang Qiangxiong <qiangxiong.huang@qq.com>
+ * Copyright 2017,2022 Huang Qiangxiong <qiangxiong.huang@qq.com>
  *
  * Wireshark - Network traffic analyzer
  * By Gerald Combs <gerald@wireshark.org>
@@ -12,10 +12,12 @@
 /*
 * The information used comes from:
 * https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md
+* https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-WEB.md
 *
-* This GRPC dissector must be invoked by HTTP2 dissector.
+* This GRPC dissector must be invoked by HTTP2 or HTTP dissector.
+* The native GRPC is always over HTTP2, the GRPC-Web is over either HTTP2 or HTTP.
 *
-* The main task of grpc dissector includes:
+* The main task of GRPC dissector for native GRPC includes:
 *
 * 1. Parse grpc message header first, if header shows message is compressed,
 *    it will find grpc-encoding http2 header by invoking http2_get_header_value()
@@ -47,14 +49,15 @@
 *    Content-type level subdissector can use this information to locate
 *    the request/response message type.
 *
-*
-* TODO
-*   Support tap.
-*   Support statistics.
+* For GRPC-WEB, the ways to get information like content-type, path (request uri)
+* are different. And for GRPC-WEB-TEXT, the dissector will first decode the base64
+* payload and then dissect the data as GRPC-WEB.
 */
 
 #include "config.h"
 
+#include <epan/conversation.h>
+#include <epan/proto_data.h>
 #include <epan/packet.h>
 #include <epan/expert.h>
 #include <epan/prefs.h>
@@ -62,6 +65,7 @@
 #include <epan/proto_data.h>
 #include <epan/dissectors/packet-http2.h>
 
+#include "packet-http.h"
 #include "wsutil/pint.h"
 
 #define GRPC_MESSAGE_HEAD_LEN 5
@@ -72,6 +76,9 @@
 /* http2 for grpc */
 #define HTTP2_HEADER_GRPC_ENCODING "grpc-encoding"
 
+/* calculate the size of a bytes after decoding as base64 */
+#define BASE64_ENCODE_SIZE(len)  ((len) / 3 * 4 + ((len) % 3 == 0 ? 0 : 4))
+
 /*
 * Decompression of zlib encoded entities.
 */
@@ -83,20 +90,39 @@ static gboolean grpc_decompress_body = FALSE;
 
 /* detect json automatically */
 static gboolean grpc_detect_json_automatically = TRUE;
-/* whether embed GRPC messages under HTTP2 protocol tree items */
+/* whether embed GRPC messages under HTTP2 (or other) protocol tree items */
 static gboolean grpc_embedded_under_http2 = FALSE;
 
 void proto_register_grpc(void);
 void proto_reg_handoff_grpc(void);
 
 static int proto_grpc = -1;
+static int proto_http = -1;
 
 /* message header */
+static int hf_grpc_frame_type = -1;
 static int hf_grpc_compressed_flag = -1;
 static int hf_grpc_message_length = -1;
 /* message body */
 static int hf_grpc_message_data = -1;
 
+/* grpc protocol type */
+#define grpc_protocol_type_vals_VALUE_STRING_LIST(XXX)    \
+    XXX(GRPC_PTYPE_GRPC, 0, "GRPC")  \
+    XXX(GRPC_PTYPE_GRPC_WEB, 1, "GRPC-Web") \
+    XXX(GRPC_PTYPE_GRPC_WEB_TEXT, 2, "GRPC-Web-Text")
+
+typedef VALUE_STRING_ENUM(grpc_protocol_type_vals) grpc_protocol_type_t;
+VALUE_STRING_ARRAY(grpc_protocol_type_vals);
+
+/* grpc frame type (grpc-web extension) */
+#define grpc_frame_type_vals_VALUE_STRING_LIST(XXX)    \
+    XXX(GRPC_FRAME_TYPE_DATA, 0, "Data")  \
+    XXX(GRPC_FRAME_TYPE_TRAILER, 1, "Trailer")
+
+VALUE_STRING_ENUM(grpc_frame_type_vals);
+VALUE_STRING_ARRAY(grpc_frame_type_vals);
+
 /* compressed flag vals */
 #define grpc_compressed_flag_vals_VALUE_STRING_LIST(XXX)    \
     XXX(GRPC_NOT_COMPRESSED, 0, "Not Compressed")  \
@@ -115,6 +141,15 @@ static int ett_grpc_message = -1;
 static int ett_grpc_encoded_entity = -1;
 
 static dissector_handle_t grpc_handle;
+static dissector_handle_t data_text_lines_handle;
+
+/* the information used during dissecting a grpc message */
+typedef struct {
+    gboolean is_request; /* is request or response message */
+    const gchar* path; /* is http2 ":path" or http request_uri, format: "/" Service-Name "/" {method name} */
+    const gchar* content_type; /* is http2 or http content-type, like: application/grpc */
+    const gchar* encoding; /* is grpc-encoding header containing compressed method, for example "gzip" */
+} grpc_context_info_t;
 
 /* GRPC message type dissector table list.
 * Dissectors can register themselves in this table as grpc message data dissectors.
@@ -128,13 +163,25 @@ static dissector_handle_t grpc_handle;
 */
 static dissector_table_t grpc_message_type_subdissector_table;
 
+static grpc_protocol_type_t
+get_grpc_protocol_type(const gchar* content_type) {
+    if (content_type != NULL) {
+        if (g_str_has_prefix(content_type, "application/grpc-web-text")) {
+            return GRPC_PTYPE_GRPC_WEB_TEXT;
+        } else if (g_str_has_prefix(content_type, "application/grpc-web")) {
+            return GRPC_PTYPE_GRPC_WEB;
+        }
+    }
+    return GRPC_PTYPE_GRPC;
+}
+
 /* Try to dissect grpc message according to grpc message info or http2 content_type. */
 static void
 dissect_body_data(proto_tree *grpc_tree, packet_info *pinfo, tvbuff_t *tvb, const gint offset,
     gint length, gboolean continue_dissect,
-    const gchar* http2_path, gboolean is_request)
+    guint32 frame_type, grpc_context_info_t *grpc_ctx)
 {
-    const gchar *http2_content_type;
+    const gchar *http2_content_type = grpc_ctx->content_type;
     gchar *grpc_message_info;
     tvbuff_t *next_tvb;
     int dissected;
@@ -142,12 +189,16 @@ dissect_body_data(proto_tree *grpc_tree, packet_info *pinfo, tvbuff_t *tvb, cons
 
     proto_tree_add_bytes_format_value(grpc_tree, hf_grpc_message_data, tvb, offset, length, NULL, "%u bytes", length);
 
+    if (frame_type == GRPC_FRAME_TYPE_TRAILER) {
+        call_dissector(data_text_lines_handle, tvb_new_subset_length(tvb, offset, length), pinfo, grpc_tree);
+        return;
+    }
+
     if (!continue_dissect) {
         return; /* if uncompress failed, we don't continue dissecting. */
     }
 
-    http2_content_type = http2_get_header_value(pinfo, HTTP2_HEADER_CONTENT_TYPE, FALSE);
-    if (http2_content_type == NULL || http2_path == NULL) {
+    if (http2_content_type == NULL || grpc_ctx->path == NULL) {
         return; /* not continue if there is not enough grpc information */
     }
 
@@ -189,7 +240,7 @@ dissect_body_data(proto_tree *grpc_tree, packet_info *pinfo, tvbuff_t *tvb, cons
     *   application/grpc,/helloworld.Greeter/SayHello,request
     */
     grpc_message_info = wmem_strconcat(pinfo->pool, http2_content_type, ",",
-        http2_path, ",", (is_request ? "request" : "response"), NULL);
+        grpc_ctx->path, ",", (grpc_ctx->is_request ? "request" : "response"), NULL);
 
     parent_tree = proto_tree_get_parent_tree(grpc_tree);
 
@@ -208,11 +259,8 @@ dissect_body_data(proto_tree *grpc_tree, packet_info *pinfo, tvbuff_t *tvb, cons
 }
 
 static gboolean
-can_uncompress_body(packet_info *pinfo, const gchar **compression_method)
+can_uncompress_body(const gchar *grpc_encoding)
 {
-    const gchar *grpc_encoding = http2_get_header_value(pinfo, HTTP2_HEADER_GRPC_ENCODING, FALSE);
-    *compression_method = grpc_encoding;
-
     /* check http2 have a grpc-encoding header appropriate */
     return grpc_decompress_body
         && grpc_encoding != NULL
@@ -223,20 +271,27 @@ can_uncompress_body(packet_info *pinfo, const gchar **compression_method)
 to 5 + message_length according to grpc wire format definition. */
 static guint
 dissect_grpc_message(tvbuff_t *tvb, guint offset, guint length, packet_info *pinfo, proto_tree *grpc_tree,
-                     const gchar* http2_path, gboolean is_request)
+                     grpc_context_info_t* grpc_ctx)
 {
-    guint32 compressed_flag, message_length;
-    const gchar *compression_method;
+    guint32 frame_type, compressed_flag, message_length;
+    const gchar *compression_method = grpc_ctx->encoding;
 
     /* GRPC message format:
     Delimited-Message -> Compressed-Flag Message-Length Message
     Compressed-Flag -> 0 / 1 # encoded as 1 byte unsigned integer
     Message-Length -> {length of Message} # encoded as 4 byte unsigned integer
     Message -> *{binary octet} (may be protobuf or json)
+
+    Note: GRPC-WEB extend the MSB of Compressed-Flag as frame type (0-data, 1-trailer)
     */
+    proto_tree_add_item_ret_uint(grpc_tree, hf_grpc_frame_type, tvb, offset, 1, ENC_BIG_ENDIAN, &frame_type);
     proto_tree_add_item_ret_uint(grpc_tree, hf_grpc_compressed_flag, tvb, offset, 1, ENC_BIG_ENDIAN, &compressed_flag);
     offset += 1;
 
+    if (frame_type == GRPC_FRAME_TYPE_TRAILER) {
+        proto_item_append_text(proto_tree_get_parent(grpc_tree), " (Trailer)");
+    }
+
     proto_tree_add_item(grpc_tree, hf_grpc_message_length, tvb, offset, 4, ENC_BIG_ENDIAN);
     message_length = length - 5;  /* should be equal to tvb_get_ntohl(tvb, offset) */
     offset += 4;
@@ -247,7 +302,7 @@ dissect_grpc_message(tvbuff_t *tvb, guint offset, guint length, packet_info *pin
 
     /* uncompressed message data if compressed_flag is set */
     if (compressed_flag & GRPC_COMPRESSED) {
-        if (can_uncompress_body(pinfo, &compression_method)) {
+        if (can_uncompress_body(compression_method)) {
             proto_item *compressed_proto_item = NULL;
             tvbuff_t *uncompressed_tvb = tvb_child_uncompress(tvb, tvb, offset, message_length);
 
@@ -261,33 +316,37 @@ dissect_grpc_message(tvbuff_t *tvb, guint offset, guint length, packet_info *pin
                 guint uncompressed_length = tvb_captured_length(uncompressed_tvb);
                 add_new_data_source(pinfo, uncompressed_tvb, "Uncompressed entity body");
                 proto_item_append_text(compressed_proto_item, " -> %u bytes", uncompressed_length);
-                dissect_body_data(grpc_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, TRUE,
-                    http2_path, is_request);
+                dissect_body_data(grpc_tree, pinfo, uncompressed_tvb, 0, uncompressed_length, TRUE, frame_type, grpc_ctx);
             } else {
                 proto_tree_add_expert(compressed_entity_tree, pinfo, &ei_grpc_body_decompression_failed,
                     tvb, offset, message_length);
-                dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, FALSE, http2_path, is_request);
+                dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, FALSE, frame_type, grpc_ctx);
             }
         } else { /* compressed flag is set, but we can not uncompressed */
-            dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, FALSE, http2_path, is_request);
+            dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, FALSE, frame_type, grpc_ctx);
         }
     } else {
-        dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, TRUE, http2_path, is_request);
+        dissect_body_data(grpc_tree, pinfo, tvb, offset, message_length, TRUE, frame_type, grpc_ctx);
     }
 
     return offset + message_length;
 }
 
 static int
-dissect_grpc(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_)
+dissect_grpc_common(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, grpc_context_info_t *grpc_ctx)
 {
     proto_item *ti;
     proto_tree *grpc_tree;
     guint32 message_length;
     guint offset = 0;
-    const gchar* http2_path;
-    gboolean is_request;
     guint tvb_len = tvb_reported_length(tvb);
+    grpc_protocol_type_t proto_type;
+    const gchar* proto_name;
+
+    DISSECTOR_ASSERT_HINT(grpc_ctx && grpc_ctx->content_type && grpc_ctx->path, "The content_type and path of grpc context must be set.");
+
+    proto_type = get_grpc_protocol_type(grpc_ctx->content_type);
+    proto_name = val_to_str_const(proto_type, grpc_protocol_type_vals, "GRPC");
 
     if (!grpc_embedded_under_http2 && proto_tree_get_parent_tree(tree)) {
         tree = proto_tree_get_parent_tree(tree);
@@ -323,29 +382,73 @@ dissect_grpc(tvbuff_t *tvb, packet_info *pinfo, proto_tree *tree, void *data _U_
         }
         /* ready to add information into protocol columns and tree */
         if (offset == 0) { /* change columns only when there is at least one grpc message will be parsed */
-            col_set_str(pinfo->cinfo, COL_PROTOCOL, "GRPC");
-            col_append_str(pinfo->cinfo, COL_INFO, " (GRPC)");
+            col_set_str(pinfo->cinfo, COL_PROTOCOL, proto_name);
+            col_append_fstr(pinfo->cinfo, COL_INFO, " (%s)", proto_name);
             col_set_fence(pinfo->cinfo, COL_PROTOCOL);
         }
         ti = proto_tree_add_item(tree, proto_grpc, tvb, offset, message_length + GRPC_MESSAGE_HEAD_LEN, ENC_NA);
         grpc_tree = proto_item_add_subtree(ti, ett_grpc_message);
+        proto_item_set_text(ti, "%s Message", proto_name);
 
-        /* http2_path contains: "/" Service-Name "/" {method name} */
-        http2_path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, FALSE);
-        is_request = (http2_path != NULL);
-
-        if (http2_path == NULL) { /* this response, so we get it from http2 request stream */
-            http2_path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, TRUE);
+        if (grpc_ctx->path) {
+            proto_item_append_text(ti, ": %s, %s", grpc_ctx->path, (grpc_ctx->is_request ? "Request" : "Response"));
         }
 
-        if (http2_path) {
-            proto_item_append_text(ti, ": %s, %s", http2_path, (is_request ? "Request" : "Response"));
+        offset = dissect_grpc_message(tvb, offset, GRPC_MESSAGE_HEAD_LEN + message_length, pinfo, grpc_tree, grpc_ctx);
+    }
+
+    return tvb_captured_length(tvb);
+}
+
+static int
+dissect_grpc(tvbuff_t* tvb, packet_info* pinfo, proto_tree* tree, void* data)
+{
+    int ret;
+    http_conv_t* http_conv;
+    tvbuff_t* real_data_tvb;
+    grpc_context_info_t grpc_ctx = { 0 };
+    conversation_t* conv = find_or_create_conversation(pinfo);
+    http_message_info_t* http_msg_info = (http_message_info_t*)data;
+    gboolean is_grpc_web_text = g_str_has_prefix(pinfo->match_string, "application/grpc-web-text");
+
+    if (is_grpc_web_text) {
+        real_data_tvb = base64_tvb_to_new_tvb(tvb, 0, tvb_reported_length(tvb));
+        add_new_data_source(pinfo, real_data_tvb, "Decoded base64 body");
+    } else {
+        real_data_tvb = tvb;
+    }
+
+    if (proto_is_frame_protocol(pinfo->layers, "http2")) {
+        grpc_ctx.path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, FALSE);
+        grpc_ctx.is_request = (grpc_ctx.path != NULL);
+        if (grpc_ctx.path == NULL) {
+            /* this must be response, so we get it from http2 request stream */
+            grpc_ctx.path = http2_get_header_value(pinfo, HTTP2_HEADER_PATH, TRUE);
         }
+        grpc_ctx.content_type = http2_get_header_value(pinfo, HTTP2_HEADER_CONTENT_TYPE, FALSE);
+        grpc_ctx.encoding = http2_get_header_value(pinfo, HTTP2_HEADER_GRPC_ENCODING, FALSE);
+    }
+    else if (proto_is_frame_protocol(pinfo->layers, "http")) {
+        http_conv = (http_conv_t*)conversation_get_proto_data(conv, proto_http);
+        DISSECTOR_ASSERT_HINT(http_conv && http_msg_info, "Unexpected error: HTTP conversation or HTTP message info not available.");
+        grpc_ctx.is_request = (http_msg_info->type == HTTP_REQUEST);
+        grpc_ctx.path = http_conv->request_uri;
+        grpc_ctx.content_type = pinfo->match_string; /* only for grpc-web(-text) over http1.1 */
+    }
+    else {
+        /* unexpected protocol error */
+        DISSECTOR_ASSERT_NOT_REACHED();
+    }
 
-        offset = dissect_grpc_message(tvb, offset, GRPC_MESSAGE_HEAD_LEN + message_length, pinfo, grpc_tree, http2_path, is_request);
+    ret = dissect_grpc_common(real_data_tvb, pinfo, tree, &grpc_ctx);
+
+    if (is_grpc_web_text) {
+        /* convert reassembly the lengths of offset and remaining bytes back to the base64 lengths */
+        pinfo->desegment_offset = BASE64_ENCODE_SIZE(pinfo->desegment_offset);
+        pinfo->desegment_len = BASE64_ENCODE_SIZE(pinfo->desegment_len);
     }
 
-    return tvb_captured_length(tvb);
+    return ret;
 }
 
 void
@@ -353,20 +456,25 @@ proto_register_grpc(void)
 {
 
     static hf_register_info hf[] = {
+        { &hf_grpc_frame_type,
+          { "Frame Type", "grpc.frame_type",
+            FT_UINT8, BASE_DEC, VALS(grpc_frame_type_vals), 0x80,
+            "The frame type of this grpc message (GRPC-WEB extension)", HFILL }
+        },
         { &hf_grpc_compressed_flag,
-        { "Compressed Flag", "grpc.compressed_flag",
-        FT_UINT8, BASE_DEC, VALS(grpc_compressed_flag_vals), 0x0,
-        "Compressed-Flag value of 1 indicates that the binary octet sequence of Message is compressed", HFILL }
+          { "Compressed Flag", "grpc.compressed_flag",
+            FT_UINT8, BASE_DEC, VALS(grpc_compressed_flag_vals), 0x01,
+            "Compressed-Flag value of 1 indicates that the binary octet sequence of Message is compressed", HFILL }
         },
         { &hf_grpc_message_length,
-        { "Message Length", "grpc.message_length",
-        FT_UINT32, BASE_DEC, NULL, 0x0,
-        "The length (32 bits) of message payload (not include itself)", HFILL }
+          { "Message Length", "grpc.message_length",
+            FT_UINT32, BASE_DEC, NULL, 0x0,
+            "The length (32 bits) of message payload (not include itself)", HFILL }
         },
         { &hf_grpc_message_data,
-        { "Message Data", "grpc.message_data",
-        FT_BYTES, BASE_NONE, NULL, 0x0,
-        NULL, HFILL }
+          { "Message Data", "grpc.message_data",
+            FT_BYTES, BASE_NONE, NULL, 0x0,
+            NULL, HFILL }
         }
     };
 
@@ -409,8 +517,8 @@ proto_register_grpc(void)
         &grpc_detect_json_automatically);
 
     prefs_register_bool_preference(grpc_module, "embedded_under_http2",
-        "Embed gRPC messages under HTTP2 protocol tree items.",
-        "Embed gRPC messages under HTTP2 protocol tree items.",
+        "Embed gRPC messages under HTTP2 (or other) protocol tree items.",
+        "Embed gRPC messages under HTTP2 (or other) protocol tree items.",
         &grpc_embedded_under_http2);
 
     prefs_register_static_text_preference(grpc_module, "service_definition",
@@ -438,15 +546,22 @@ proto_reg_handoff_grpc(void)
         "application/grpc",
         "application/grpc+proto",
         "application/grpc+json",
+        "application/grpc-web",
+        "application/grpc-web+proto",
+        "application/grpc-web-text",
+        "application/grpc-web-text+proto",
         NULL /* end flag */
     };
     int i;
 
-    /* register/deregister grpc_handle to/from tables */
+    /* register native grpc handler */
     for (i = 0; content_types[i]; i++) {
         dissector_add_string("streaming_content_type", content_types[i], grpc_handle);
         dissector_add_string("media_type", content_types[i], grpc_handle);
     }
+
+    proto_http = proto_get_id_by_filter_name("http");
+    data_text_lines_handle = find_dissector_add_dependency("data-text-lines", proto_grpc);
 }
 
 /*