Kafka: include zstd compression in Kafka message batches

Change-Id: I1d06486ccf7b174ee9aa621fa3d8acb8b3673777
Reviewed-on: https://code.wireshark.org/review/34222
Petri-Dish: Anders Broman <a.broman58@gmail.com>
Tested-by: Petri Dish Buildbot
Reviewed-by: Anders Broman <a.broman58@gmail.com>

16 files changed, 190 insertions, 6 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 725c0c2fc0..01d68b8890 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1138,6 +1138,9 @@ ws_find_package(LZ4 ENABLE_LZ4 HAVE_LZ4)
 # Snappy compression
 ws_find_package(SNAPPY ENABLE_SNAPPY HAVE_SNAPPY)
 
+# zstd compression
+ws_find_package(ZSTD ENABLE_ZSTD HAVE_ZSTD)
+
 # Enhanced HTTP/2 dissection
 ws_find_package(NGHTTP2 ENABLE_NGHTTP2 HAVE_NGHTTP2)
 
@@ -1594,6 +1597,11 @@ set_package_properties(SNAPPY PROPERTIES
 	URL "https://google.github.io/snappy/"
 	PURPOSE "Snappy decompression in CQL and Kafka dissectors"
 )
+set_package_properties(ZSTD PROPERTIES
+	DESCRIPTION "A compressor/decompressor from Facebook providing better compression than Snappy at a cost of speed"
+	URL "https://facebook.github.io/zstd/"
+	PURPOSE "Zstd decompression in Kafka dissector"
+)
 set_package_properties(NGHTTP2 PROPERTIES
 	DESCRIPTION "HTTP/2 C library and tools"
 	URL "https://nghttp2.org"
@@ -1818,6 +1826,9 @@ if(WIN32)
 		list (APPEND OPTIONAL_DLLS "${LZ4_DLL_DIR}/${LZ4_DLL}")
 		list (APPEND OPTIONAL_PDBS "${LZ4_DLL_DIR}/${LZ4_PDB}")
 	endif(LZ4_FOUND)
+	if (ZSTD_FOUND)
+		list (APPEND OPTIONAL_DLLS "${ZSTD_DLL_DIR}/${ZSTD_DLL}")
+	endif(ZSTD_FOUND)
 	if (NGHTTP2_FOUND)
 		list (APPEND OPTIONAL_DLLS "${NGHTTP2_DLL_DIR}/${NGHTTP2_DLL}")
 		list (APPEND OPTIONAL_PDBS "${NGHTTP2_DLL_DIR}/${NGHTTP2_PDB}")
diff --git a/CMakeOptions.txt b/CMakeOptions.txt
index ff71677eb1..8745f2da90 100644
--- a/CMakeOptions.txt
+++ b/CMakeOptions.txt
@@ -69,6 +69,7 @@ option(ENABLE_MINIZIP    "Build with zip file compression support" ON)
 option(ENABLE_LZ4        "Build with LZ4 compression support" ON)
 option(ENABLE_BROTLI     "Build with brotli compression support" ON)
 option(ENABLE_SNAPPY     "Build with Snappy compression support" ON)
+option(ENABLE_ZSTD       "Build with Facebook zstd compression support" ON)
 option(ENABLE_NGHTTP2    "Build with HTTP/2 header decompression support" ON)
 option(ENABLE_LUA        "Build with Lua dissector support" ON)
 option(ENABLE_SMI        "Build with libsmi snmp support" ON)
diff --git a/cmake/modules/FindZSTD.cmake b/cmake/modules/FindZSTD.cmake
new file mode 100644
index 0000000000..d2d1f6207a
--- /dev/null
+++ b/cmake/modules/FindZSTD.cmake
@@ -0,0 +1,58 @@
+#
+# - Find zstd
+# Find Zstd includes and library
+#
+#  ZSTD_INCLUDE_DIRS - where to find zstd.h, etc.
+#  ZSTD_LIBRARIES    - List of libraries when using Zstd.
+#  ZSTD_FOUND        - True if Zstd found.
+#  ZSTD_DLL_DIR      - (Windows) Path to the Zstd DLL
+#  ZSTD_DLL          - (Windows) Name of the Zstd DLL
+
+include( FindWSWinLibs )
+FindWSWinLibs( "zstd-.*" "ZSTD_HINTS" )
+
+if( NOT WIN32)
+  find_package(PkgConfig)
+  pkg_search_module(ZSTD libzstd)
+endif()
+
+find_path(ZSTD_INCLUDE_DIR
+  NAMES zstd.h
+  HINTS "${ZSTD_INCLUDEDIR}" "${ZSTD_HINTS}/include"
+  /usr/include
+  /usr/local/include
+)
+
+find_library(ZSTD_LIBRARY
+  NAMES zstd
+  HINTS "${ZSTD_LIBDIR}" "${ZSTD_HINTS}/lib"
+  PATHS
+  /usr/lib
+  /usr/local/lib
+)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args( ZSTD DEFAULT_MSG ZSTD_LIBRARY ZSTD_INCLUDE_DIR )
+
+if( ZSTD_FOUND )
+  set( ZSTD_INCLUDE_DIRS ${ZSTD_INCLUDE_DIR} )
+  set( ZSTD_LIBRARIES ${ZSTD_LIBRARY} )
+  if (WIN32)
+    set ( ZSTD_DLL_DIR "${ZSTD_HINTS}/bin"
+      CACHE PATH "Path to Zstd DLL"
+    )
+    file( GLOB _zstd_dll RELATIVE "${ZSTD_DLL_DIR}"
+      "${ZSTD_DLL_DIR}/zstd*.dll"
+    )
+    set ( ZSTD_DLL ${_zstd_dll}
+      # We're storing filenames only. Should we use STRING instead?
+      CACHE FILEPATH "Zstd DLL file name"
+    )
+    mark_as_advanced( ZSTD_DLL_DIR ZSTD_DLL )
+  endif()
+else()
+  set( ZSTD_INCLUDE_DIRS )
+  set( ZSTD_LIBRARIES )
+endif()
+
+mark_as_advanced( ZSTD_LIBRARIES ZSTD_INCLUDE_DIRS )
diff --git a/cmakeconfig.h.in b/cmakeconfig.h.in
index bcf16d6597..bf1130b617 100644
--- a/cmakeconfig.h.in
+++ b/cmakeconfig.h.in
@@ -151,6 +151,9 @@
 /* Define to use snappy library */
 #cmakedefine HAVE_SNAPPY 1
 
+/* Define to use zstd library */
+#cmakedefine HAVE_ZSTD 1
+
 /* Define to 1 if you have the <linux/sockios.h> header file. */
 #cmakedefine HAVE_LINUX_SOCKIOS_H 1
 
diff --git a/debian/control b/debian/control
index df74d74a9f..4541abb0ec 100644
--- a/debian/control
+++ b/debian/control
@@ -17,7 +17,7 @@ Build-Depends:
  libmaxminddb-dev, dpkg-dev (>= 1.16.1~), libsystemd-dev | libsystemd-journal-dev,
  libnl-genl-3-dev [linux-any], libnl-route-3-dev [linux-any], asciidoctor,
  cmake (>= 3.5) | cmake3, libsbc-dev, libnghttp2-dev, libssh-gcrypt-dev,
- liblz4-dev, libsnappy-dev, libspandsp-dev, libxml2-dev, libbrotli-dev,
+ liblz4-dev, libsnappy-dev, libzstd-dev, libspandsp-dev, libxml2-dev, libbrotli-dev,
  libspeexdsp-dev
 Build-Conflicts: libsnmp4.2-dev, libsnmp-dev
 Vcs-Git: https://salsa.debian.org/debian/wireshark -b debian/master
diff --git a/epan/CMakeLists.txt b/epan/CMakeLists.txt
index 5bf71f4e2b..03f4939d2f 100644
--- a/epan/CMakeLists.txt
+++ b/epan/CMakeLists.txt
@@ -27,6 +27,7 @@ include_directories(
 		${SMI_INCLUDE_DIRS}
 		${SNAPPY_INCLUDE_DIRS}
 		${ZLIB_INCLUDE_DIRS}
+		${ZSTD_INCLUDE_DIRS}
 		${LIBXML2_INCLUDE_DIRS}
 )
 
@@ -352,6 +353,7 @@ target_link_libraries(epan
 		${WIN_PSAPI_LIBRARY}
 		${LIBXML2_LIBRARIES}
 		${ZLIB_LIBRARIES}
+		${ZSTD_LIBRARIES}
 )
 
 target_include_directories(epan
diff --git a/epan/dissectors/packet-kafka.c b/epan/dissectors/packet-kafka.c
index 6fc5b37607..d3e2671bcf 100644
--- a/epan/dissectors/packet-kafka.c
+++ b/epan/dissectors/packet-kafka.c
@@ -26,6 +26,9 @@
 #include <lz4.h>
 #include <lz4frame.h>
 #endif
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
 #include "packet-tcp.h"
 #include "packet-tls.h"
 
@@ -1522,12 +1525,51 @@ decompress_snappy(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int len
 }
 #endif /* HAVE_SNAPPY */
 
+#ifdef HAVE_ZSTD
+static int
+decompress_zstd(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, tvbuff_t **decompressed_tvb, int *decompressed_offset)
+{
+    ZSTD_inBuffer input = { tvb_memdup(wmem_packet_scope(), tvb, offset, length), length, 0 };
+    ZSTD_DStream *zds = ZSTD_createDStream();
+    size_t rc = 0;
+    tvbuff_t *composite_tvb = tvb_new_composite();
+    int ret = 0;
+
+    do {
+        ZSTD_outBuffer output = { wmem_alloc(pinfo->pool, ZSTD_DStreamOutSize()), ZSTD_DStreamOutSize(), 0 };
+        rc = ZSTD_decompressStream(zds, &output, &input);
+        // rc holds either the number of decompressed offsets or the error code.
+        // Both values are positive, one has to use ZSTD_isError to determine if the call succeeded.
+        if (ZSTD_isError(rc)) {
+            goto end;
+        }
+        tvb_composite_append(composite_tvb,
+                             tvb_new_child_real_data(tvb, (guint8*)output.dst, (guint)output.pos, (gint)output.pos));
+        // rc == 0 means there is nothing more to decompress, but there could be still something in the data
+    } while (rc > 0);
+    tvb_composite_finalize(composite_tvb);
+    *decompressed_tvb = composite_tvb;
+    *decompressed_offset = 0;
+    composite_tvb = NULL;
+    ret = 1;
+end:
+    ZSTD_freeDStream(zds);
+    if (composite_tvb != NULL) {
+        tvb_free_chain(composite_tvb);
+    }
+    if (ret == 0) {
+        col_append_str(pinfo->cinfo, COL_INFO, " [zstd decompression failed]");
+    }
+    return ret;
+}
+#else
 static int
 decompress_zstd(tvbuff_t *tvb _U_, packet_info *pinfo, int offset _U_, int length _U_, tvbuff_t **decompressed_tvb _U_, int *decompressed_offset _U_)
 {
     col_append_str(pinfo->cinfo, COL_INFO, " [zstd compression unsupported]");
     return 0;
 }
+#endif /* HAVE_ZSTD */
 
 static int
 decompress(tvbuff_t *tvb, packet_info *pinfo, int offset, int length, int codec, tvbuff_t **decompressed_tvb, int *decompressed_offset)
diff --git a/epan/epan.c b/epan/epan.c
index ad51670088..cc4327d270 100644
--- a/epan/epan.c
+++ b/epan/epan.c
@@ -799,6 +799,14 @@ epan_get_compiled_version_info(GString *str)
 	g_string_append(str, "without LZ4");
 #endif /* HAVE_LZ4 */
 
+	/* Zstandard */
+	g_string_append(str, ", ");
+#ifdef HAVE_ZSTD
+	g_string_append(str, "with Zstandard");
+#else
+	g_string_append(str, "without Zstandard");
+#endif /* HAVE_ZSTD */
+
 	/* Snappy */
 	g_string_append(str, ", ");
 #ifdef HAVE_SNAPPY
diff --git a/packaging/nsis/CMakeLists.txt b/packaging/nsis/CMakeLists.txt
index 9ccb9e1139..0e3236cfa4 100644
--- a/packaging/nsis/CMakeLists.txt
+++ b/packaging/nsis/CMakeLists.txt
@@ -141,7 +141,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
 		${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
 		${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
 		${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
-		${ZLIB_DLL} ${BROTLI_DLLS}
+		${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
 		# Needed for mmdbresolve
 		${MAXMINDDB_DLL}
 	)
diff --git a/packaging/rpm/wireshark.spec.in b/packaging/rpm/wireshark.spec.in
index e411c6f5c8..6e600186fb 100644
--- a/packaging/rpm/wireshark.spec.in
+++ b/packaging/rpm/wireshark.spec.in
@@ -20,7 +20,7 @@
 %bcond_with	sdjournal
 %bcond_with	guides
 %bcond_with	brotli
-
+%bcond_with	zstd
 
 # Set at most one of these two:
 # Note that setcap requires rpmbuild 4.7.0 or later.
@@ -161,6 +161,15 @@ Requires:	brotli
 %endif
 %endif
 
+%if %{with zstd}
+BuildRequires:	libzstd-devel
+%if 0%{?suse_version}
+Requires:	libzstd1
+%else
+Requires:	libzstd
+%endif
+%endif
+
 # Uncomment these if you want to be sure you get them...
 #BuildRequires:	krb5-devel
 #BuildRequires:	libsmi-devel
@@ -497,6 +506,9 @@ update-mime-database %{_datadir}/mime &> /dev/null || :
 %{_libdir}/pkgconfig/wireshark.pc
 
 %changelog
+* Wed Aug 15 2019 Gerald Combs
+- Add zstd
+
 * Mon Apr 22 2019 Daniel Bakai
 - Added brotli (as an option, defaulting to not required).
 
diff --git a/packaging/wix/CMakeLists.txt b/packaging/wix/CMakeLists.txt
index f261b15b26..c5bba1a17b 100644
--- a/packaging/wix/CMakeLists.txt
+++ b/packaging/wix/CMakeLists.txt
@@ -120,7 +120,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
 		${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
 		${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
 		${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
-		${ZLIB_DLL} ${BROTLI_DLLS}
+		${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
 		# Required for mmdbresolve
 		${MAXMINDDB_DLL}
 	)
@@ -148,7 +148,7 @@ foreach(_dll ${GLIB2_DLLS} ${CARES_DLL} ${GCRYPT_DLLS}
 		${GNUTLS_DLLS} ${KERBEROS_DLLS} ${LIBSSH_DLL} ${LUA_DLL}
 		${LZ4_DLL} ${NGHTTP2_DLL} ${SBC_DLL} ${SMI_DLL} ${SNAPPY_DLL}
 		${SPANDSP_DLL} ${BCG729_DLL} ${LIBXML2_DLLS} ${WINSPARKLE_DLL}
-		${ZLIB_DLL} ${BROTLI_DLLS}
+		${ZLIB_DLL} ${BROTLI_DLLS} ${ZSTD_DLL}
 		# mmdbresolve
 		${MAXMINDDB_DLL}
 	)
diff --git a/tools/debian-setup.sh b/tools/debian-setup.sh
index 589cb7e467..2f3a3579e2 100755
--- a/tools/debian-setup.sh
+++ b/tools/debian-setup.sh
@@ -76,6 +76,7 @@ ADDITIONAL_LIST="libnl-3-dev \
 	libcap-dev \
 	liblz4-dev \
 	libsnappy-dev \
+	libzstd-dev \
 	libspandsp-dev \
 	libxml2-dev \
 	libminizip-dev \
diff --git a/tools/macos-setup-brew.sh b/tools/macos-setup-brew.sh
index fe77814a47..b93052bd0c 100755
--- a/tools/macos-setup-brew.sh
+++ b/tools/macos-setup-brew.sh
@@ -13,7 +13,7 @@
 brew update
 
 #install some libs needed by Wireshark
-brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip
+brew install c-ares glib libgcrypt gnutls lua@5.1 cmake python nghttp2 snappy lz4 libxml2 ninja libmaxminddb doxygen libsmi spandsp brotli minizip zstd
 
 #install Qt5
 brew install qt5
diff --git a/tools/macos-setup.sh b/tools/macos-setup.sh
index f9343dd989..5f61c2b41e 100755
--- a/tools/macos-setup.sh
+++ b/tools/macos-setup.sh
@@ -151,6 +151,7 @@ fi
 # features present in all three versions)
 LUA_VERSION=5.2.4
 SNAPPY_VERSION=1.1.4
+ZSTD_VERSION=1.4.2
 LIBXML2_VERSION=2.9.9
 LZ4_VERSION=1.7.5
 SBC_VERSION=1.3
@@ -1241,6 +1242,41 @@ uninstall_snappy() {
     fi
 }
 
+install_zstd() {
+    if [ "$ZSTD_VERSION" -a ! -f zstd-$ZSTD_VERSION-done ] ; then
+        echo "Downloading, building, and installing zstd:"
+        [ -f zstd-$ZSTD_VERSION.tar.gz ] || curl -L -O https://github.com/facebook/zstd/releases/download/v$ZSTD_VERSION/zstd-$ZSTD_VERSION.tar.gz || exit 1
+        $no_build && echo "Skipping installation" && return
+        gzcat zstd-$ZSTD_VERSION.tar.gz | tar xf - || exit 1
+        cd zstd-$ZSTD_VERSION
+        make $MAKE_BUILD_OPTS || exit 1
+        $DO_MAKE_INSTALL || exit 1
+        cd ..
+        touch zstd-$ZSTD_VERSION-done
+    fi
+}
+
+uninstall_zstd() {
+    if [ ! -z "$installed_zstd_version" ] ; then
+        echo "Uninstalling zstd:"
+        cd zstd-$installed_zstd_version
+        $DO_MAKE_UNINSTALL || exit 1
+        make distclean || exit 1
+        cd ..
+        rm zstd-$installed_zstd_version-done
+
+        if [ "$#" -eq 1 -a "$1" = "-r" ] ; then
+            #
+            # Get rid of the previously downloaded and unpacked version.
+            #
+            rm -rf zstd-$installed_zstd_version
+            rm -rf zstd-$installed_zstd_version.tar.gz
+        fi
+
+        installed_zstd_version=""
+    fi
+}
+
 install_libxml2() {
     if [ "$LIBXML2_VERSION" -a ! -f libxml2-$LIBXML2_VERSION-done ] ; then
         echo "Downloading, building, and installing libxml2:"
@@ -2227,6 +2263,8 @@ install_all() {
 
     install_snappy
 
+    install_zstd
+
     install_libxml2
 
     install_lz4
@@ -2286,6 +2324,8 @@ uninstall_all() {
 
         uninstall_snappy
 
+        uninstall_zstd
+
         uninstall_libxml2
 
         uninstall_lz4
@@ -2459,6 +2499,7 @@ then
     installed_gnutls_version=`ls gnutls-*-done 2>/dev/null | sed 's/gnutls-\(.*\)-done/\1/'`
     installed_lua_version=`ls lua-*-done 2>/dev/null | sed 's/lua-\(.*\)-done/\1/'`
     installed_snappy_version=`ls snappy-*-done 2>/dev/null | sed 's/snappy-\(.*\)-done/\1/'`
+    installed_zstd_version=`ls zstd-*-done 2>/dev/null | sed 's/zstd-\(.*\)-done/\1/'`
     installed_libxml2_version=`ls libxml2-*-done 2>/dev/null | sed 's/libxml2-\(.*\)-done/\1/'`
     installed_lz4_version=`ls lz4-*-done 2>/dev/null | sed 's/lz4-\(.*\)-done/\1/'`
     installed_sbc_version=`ls sbc-*-done 2>/dev/null | sed 's/sbc-\(.*\)-done/\1/'`
diff --git a/tools/rpm-setup.sh b/tools/rpm-setup.sh
index 6e50427e67..a988e402f1 100755
--- a/tools/rpm-setup.sh
+++ b/tools/rpm-setup.sh
@@ -183,6 +183,8 @@ echo "nghttp2 is unavailable" >&2
 add_package ADDITIONAL_LIST snappy || add_package ADDITIONAL_LIST libsnappy1 ||
 echo "snappy is unavailable" >&2
 
+add_package ADDITIONAL_LIST libzstd-devel || echo "zstd is unavailable" >&2
+
 add_package ADDITIONAL_LIST lz4-devel || add_package ADDITIONAL_LIST liblz4-devel ||
 echo "lz4 devel is unavailable" >&2
 
diff --git a/tools/win-setup.ps1 b/tools/win-setup.ps1
index b072bea23e..c00ab6b230 100644
--- a/tools/win-setup.ps1
+++ b/tools/win-setup.ps1
@@ -94,6 +94,7 @@ $Win64Archives = @{
     "vcpkg-export-20190318-win64ws.zip" = "72c2c43594b0581de2bc86517870a561cc40df294662502536b2a6c06cace87e";
     "WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04";
     "WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd";
+    "zstd-1.4.0-win64ws.zip" = "154199227bdfdfa608972bcdcea38e20768937085e5a59a8fa06c72d07b00d6b";
 }
 
 $Win32Archives = @{
@@ -117,6 +118,7 @@ $Win32Archives = @{
     "vcpkg-export-20190318-win32ws.zip" = "5f9eb78b1ea9e6762c2a4104e0126f1f5453919dc9df66fef2b1e0be8d8c5829";
     "WinSparkle-0.5.7.zip" = "56d396ef0c4e8b0589ea74134e484376ca6459d972cd1ab1da6b9624d82e6d04";
     "WpdPack_4_1_2.zip" = "ea799cf2f26e4afb1892938070fd2b1ca37ce5cf75fec4349247df12b784edbd";
+    "zstd-1.4.0-win32ws.zip" = "9141716d4d749e67dad40d4aab6bbb3206085bf68e5acb03baf1e5667aa0b6f5";
 }
 
 # Subdirectory to extract an archive to
@@ -183,6 +185,7 @@ $CleanupItems = @(
     "zlib-1.2.5"
     "zlib-1.2.8"
     "zlib-1.2.*-ws"
+    "zstd-*-win??ws"
     "AirPcap_Devpack_4_1_0_1622"
     "GeoIP-1.*-win??ws"
     "WinSparkle-0.3-44-g2c8d9d3-win??ws"