wsutil: Improve bytes_to_str_max() API

Have these functions accept a zero max length to mean "display
the whole byte array". Change the max length parameter to receive a
number of bytes to print, not the length of the output character
string.

Adjust the macros bytes_to_str() and bytes_to_string_punct() to
produce the same output. Add more tests. Rename the functions to
bytes_to_str_maxlen() and bytes_to_str_punct_maxlen() because this is
an API break.

6 files changed, 133 insertions, 52 deletions

diff --git a/wsutil/.editorconfig b/wsutil/.editorconfig
index 6edc1e649e..7f16cdc160 100644
--- a/wsutil/.editorconfig
+++ b/wsutil/.editorconfig
@@ -101,6 +101,10 @@ indent_size = 2
 indent_style = tab
 indent_size = tab
 
+[to_str.[ch]]
+indent_style = tab
+indent_size = tab
+
 [type_util.[ch]]
 indent_size = 2
 
diff --git a/wsutil/test_wsutil.c b/wsutil/test_wsutil.c
index 35b3285811..767dab5e88 100644
--- a/wsutil/test_wsutil.c
+++ b/wsutil/test_wsutil.c
@@ -104,6 +104,60 @@ static void test_bytes_to_str_punct(void)
     g_free(str);
 }
 
+static void test_bytes_to_str_punct_maxlen(void)
+{
+    char *str;
+
+    const guint8 buf[] = { 1, 2, 3};
+
+    str = bytes_to_str_punct_maxlen(NULL, buf, sizeof(buf), ':', 4);
+    g_assert_cmpstr(str, ==, "01:02:03");
+    g_free(str);
+
+    str = bytes_to_str_punct_maxlen(NULL, buf, sizeof(buf), ':', 3);
+    g_assert_cmpstr(str, ==, "01:02:03");
+    g_free(str);
+
+    str = bytes_to_str_punct_maxlen(NULL, buf, sizeof(buf), ':', 2);
+    g_assert_cmpstr(str, ==, "01:02:" UTF8_HORIZONTAL_ELLIPSIS);
+    g_free(str);
+
+    str = bytes_to_str_punct_maxlen(NULL, buf, sizeof(buf), ':', 1);
+    g_assert_cmpstr(str, ==, "01:" UTF8_HORIZONTAL_ELLIPSIS);
+    g_free(str);
+
+    str = bytes_to_str_punct_maxlen(NULL, buf, sizeof(buf), ':', 0);
+    g_assert_cmpstr(str, ==, "01:02:03");
+    g_free(str);
+}
+
+static void test_bytes_to_str_maxlen(void)
+{
+    char *str;
+
+    const guint8 buf[] = { 1, 2, 3};
+
+    str = bytes_to_str_maxlen(NULL, buf, sizeof(buf), 4);
+    g_assert_cmpstr(str, ==, "010203");
+    g_free(str);
+
+    str = bytes_to_str_maxlen(NULL, buf, sizeof(buf), 3);
+    g_assert_cmpstr(str, ==, "010203");
+    g_free(str);
+
+    str = bytes_to_str_maxlen(NULL, buf, sizeof(buf), 2);
+    g_assert_cmpstr(str, ==, "0102" UTF8_HORIZONTAL_ELLIPSIS);
+    g_free(str);
+
+    str = bytes_to_str_maxlen(NULL, buf, sizeof(buf), 1);
+    g_assert_cmpstr(str, ==, "01" UTF8_HORIZONTAL_ELLIPSIS);
+    g_free(str);
+
+    str = bytes_to_str_maxlen(NULL, buf, sizeof(buf), 0);
+    g_assert_cmpstr(str, ==, "010203");
+    g_free(str);
+}
+
 static void test_bytes_to_string_trunc1(void)
 {
     char *str;
@@ -478,6 +532,8 @@ int main(int argc, char **argv)
     g_test_add_func("/to_str/word_to_hex", test_word_to_hex);
     g_test_add_func("/to_str/bytes_to_str", test_bytes_to_str);
     g_test_add_func("/to_str/bytes_to_str_punct", test_bytes_to_str_punct);
+    g_test_add_func("/to_str/bytes_to_str_maxlen", test_bytes_to_str_maxlen);
+    g_test_add_func("/to_str/bytes_to_str_punct_maxlen", test_bytes_to_str_punct_maxlen);
     g_test_add_func("/to_str/bytes_to_str_trunc1", test_bytes_to_string_trunc1);
     g_test_add_func("/to_str/bytes_to_str_punct_trunc1", test_bytes_to_string_punct_trunc1);
     g_test_add_func("/to_str/oct_to_str_back", test_oct_to_str_back);
diff --git a/wsutil/to_str.c b/wsutil/to_str.c
index 98ec094edc..3ed9614a41 100644
--- a/wsutil/to_str.c
+++ b/wsutil/to_str.c
@@ -216,34 +216,45 @@ bytes_to_hexstr_punct(char *out, const guint8 *ad, size_t len, char punct)
  * If punct is '\0', no punctuation is applied (and thus
  * the resulting string is (len-1) bytes shorter)
  */
-gchar *
-bytes_to_str_punct_max(wmem_allocator_t *scope, const guint8 *ad, size_t len, const char punct, size_t max)
+char *
+bytes_to_str_punct_maxlen(wmem_allocator_t *scope,
+			const guint8 *src, size_t src_size,
+			char punct, size_t max_bytes_len)
 {
-	gchar *buf;
-	size_t buflen = len;
-	gchar *buf_ptr;
+	char *buf;
+	size_t max_char_size;
+	char *buf_ptr;
 	int truncated = 0;
 
-	ws_return_str_if_zero(scope, len);
-	ws_return_str_if_null(scope, ad);
+	ws_return_str_if_null(scope, src);
+	ws_return_str_if_zero(scope, src_size);
 
 	if (!punct)
-		return bytes_to_str_max(scope, ad, len, max);
+		return bytes_to_str_maxlen(scope, src, src_size, max_bytes_len);
 
-	if (max > len * 3)
-		max = len * 3;
+	/* Sanity check */
+	if (src_size > 8192) {
+		src_size = 8192;
+		truncated = 1;
+	}
 
-	buf=(gchar *)wmem_alloc(scope, max+3+1);
-	if (buflen > max/3) {
+	if (max_bytes_len == 0 || max_bytes_len > src_size) {
+		max_bytes_len = src_size;
+	}
+	else if (max_bytes_len < src_size) {
 		truncated = 1;
-		buflen = max/3;
 	}
 
-	buf_ptr = bytes_to_hexstr_punct(buf, ad, buflen, punct); /* maximum max-1 bytes */
+	/* Include space for ellipsis and '\0'. Optional extra punct
+	 * at the end is already accounted for. */
+	max_char_size = max_bytes_len * 3 + strlen(UTF8_HORIZONTAL_ELLIPSIS) + 1;
+
+	buf = wmem_alloc(scope, max_char_size);
+	buf_ptr = bytes_to_hexstr_punct(buf, src, max_bytes_len, punct);
 
 	if (truncated) {
-		*buf_ptr++ = punct;			/* 1 byte */
-		buf_ptr    = g_stpcpy(buf_ptr, UTF8_HORIZONTAL_ELLIPSIS);	/* 3 bytes */
+		*buf_ptr++ = punct;
+		buf_ptr = g_stpcpy(buf_ptr, UTF8_HORIZONTAL_ELLIPSIS);
 	}
 
 	*buf_ptr = '\0';
@@ -251,31 +262,41 @@ bytes_to_str_punct_max(wmem_allocator_t *scope, const guint8 *ad, size_t len, co
 }
 
 char *
-bytes_to_str_max(wmem_allocator_t *scope, const guint8 *bd, size_t bd_len, size_t max)
+bytes_to_str_maxlen(wmem_allocator_t *scope,
+			const guint8 *src, size_t src_size,
+			size_t max_bytes_len)
 {
-	gchar *cur;
-	gchar *cur_ptr;
+	char *buf;
+	size_t max_char_size;
+	char *buf_ptr;
 	int truncated = 0;
 
-	ws_return_str_if_zero(scope, bd_len);
-	ws_return_str_if_null(scope, bd);
+	ws_return_str_if_null(scope, src);
+	ws_return_str_if_zero(scope, src_size);
 
-	if (max > bd_len * 2)
-		max = bd_len * 2;
+	/* Sanity check */
+	if (src_size > 8192) {
+		src_size = 8192;
+		truncated = 1;
+	}
 
-	cur=(gchar *)wmem_alloc(scope, max+3+1);
-	if (bd_len > max/2) {
+	if (max_bytes_len == 0 || max_bytes_len > src_size) {
+		max_bytes_len = src_size;
+	}
+	else if (max_bytes_len < src_size) {
 		truncated = 1;
-		bd_len = max/2;
 	}
 
-	cur_ptr = bytes_to_hexstr(cur, bd, bd_len);	/* max MAX_BYTE_STR_LEN bytes */
+	max_char_size = max_bytes_len * 2 + strlen(UTF8_HORIZONTAL_ELLIPSIS) + 1;
+
+	buf = wmem_alloc(scope, max_char_size);
+	buf_ptr = bytes_to_hexstr(buf, src, max_bytes_len);
 
 	if (truncated)
-		cur_ptr = g_stpcpy(cur_ptr, UTF8_HORIZONTAL_ELLIPSIS);	/* 3 bytes */
+		buf_ptr = g_stpcpy(buf_ptr, UTF8_HORIZONTAL_ELLIPSIS);
 
-	*cur_ptr = '\0';				/* 1 byte */
-	return cur;
+	*buf_ptr = '\0';
+	return buf;
 }
 
 /*
diff --git a/wsutil/to_str.h b/wsutil/to_str.h
index a8d6f829fb..ae3308c2b3 100644
--- a/wsutil/to_str.h
+++ b/wsutil/to_str.h
@@ -11,9 +11,8 @@
 #ifndef __WSUTIL_TO_STR_H__
 #define __WSUTIL_TO_STR_H__
 
-#include <glib.h>
+#include <wireshark.h>
 
-#include <ws_symbol_export.h>
 #include <wsutil/wmem/wmem.h>
 #include <wsutil/inet_ipv6.h>
 
@@ -135,36 +134,37 @@ WS_DLL_PUBLIC char *bytes_to_hexstr(char *out, const guint8 *ad, size_t len);
  */
 WS_DLL_PUBLIC char *bytes_to_hexstr_punct(char *out, const guint8 *ad, size_t len, char punct);
 
-/* Max string length for displaying byte string.  */
-#define	MAX_BYTE_STR_LEN	72
-
 /** Turn an array of bytes into a string showing the bytes in hex,
  *  separated by a punctuation character.
  *
  * @param scope memory allocation scheme used
- * @param ad A pointer to the byte array
- * @param len The length of the byte array
+ * @param buf A pointer to the byte array
+ * @param buf_size The length of the byte array
  * @param punct The punctuation character
- * @param max Maximum string length, plus ellipsis if present
+ * @param max_bytes_len Maximum number of bytes to represent, zero for no limit.
  * @return A pointer to the formatted string
- *
- * @see bytes_to_str()
  */
-WS_DLL_PUBLIC gchar *bytes_to_str_punct_max(wmem_allocator_t *scope, const guint8 *ad, size_t len, const char punct, size_t max);
+WS_DLL_PUBLIC char *bytes_to_str_punct_maxlen(wmem_allocator_t *scope,
+				const guint8 *buf, size_t buf_size,
+				char punct, size_t max_bytes_len);
 
-#define bytes_to_str_punct(scope, ad, len, punct) bytes_to_str_punct_max(scope, ad, len, punct, MAX_BYTE_STR_LEN)
+#define bytes_to_str_punct(scope, buf, buf_size, punct) \
+    bytes_to_str_punct_maxlen(scope, buf, buf_size, punct, 24)
 
 /** Turn an array of bytes into a string showing the bytes in hex.
  *
  * @param scope memory allocation scheme used
- * @param bd A pointer to the byte array
- * @param bd_len The length of the byte array
- * @param max Maximum string length, plus ellipsis if present
+ * @param buf A pointer to the byte array
+ * @param buf_size The length of the byte array
+ * @param max_bytes_len Maximum number of bytes to represent, zero for no limit.
  * @return A pointer to the formatted string
  */
-WS_DLL_PUBLIC char *bytes_to_str_max(wmem_allocator_t *scope, const guint8 *bd, size_t bd_len, size_t max);
+WS_DLL_PUBLIC char *bytes_to_str_maxlen(wmem_allocator_t *scope,
+				const guint8 *buf, size_t buf_size,
+				size_t max_bytes_len);
 
-#define bytes_to_str(scope, bd, len) bytes_to_str_max(scope, bd, len, MAX_BYTE_STR_LEN)
+#define bytes_to_str(scope, buf, buf_size) \
+    bytes_to_str_maxlen(scope, buf, buf_size, 36)
 
 /**
  * oct_to_str_back()
diff --git a/wsutil/wslog.c b/wsutil/wslog.c
index f9bfa3919d..e8570371eb 100644
--- a/wsutil/wslog.c
+++ b/wsutil/wslog.c
@@ -921,13 +921,13 @@ void ws_log_write_always_full(const char *domain, enum ws_log_level level,
 
 void ws_log_buffer_full(const char *domain, enum ws_log_level level,
                     const char *file, int line, const char *func,
-                    const guint8 *ptr, size_t size,  size_t max_len,
+                    const guint8 *ptr, size_t size,  size_t max_bytes_len,
                     const char *msg)
 {
     if (!ws_log_msg_is_active(domain, level))
         return;
 
-    char *bufstr = bytes_to_str_max(NULL, ptr, size, max_len);
+    char *bufstr = bytes_to_str_maxlen(NULL, ptr, size, max_bytes_len);
 
     if (G_UNLIKELY(msg == NULL))
         ws_log_write_always_full(domain, level, file, line, func,
diff --git a/wsutil/wslog.h b/wsutil/wslog.h
index 5a074c256b..fda58339f6 100644
--- a/wsutil/wslog.h
+++ b/wsutil/wslog.h
@@ -303,14 +303,14 @@ void ws_logv_full(const char *domain, enum ws_log_level level,
 WS_DLL_PUBLIC
 void ws_log_buffer_full(const char *domain, enum ws_log_level level,
                     const char *file, int line, const char *func,
-                    const guint8 *buffer, size_t size, size_t max_len,
-                    const char *msg);
+                    const guint8 *buffer, size_t size,
+                    size_t max_bytes_len, const char *msg);
 
 
 #define _LOG_BUFFER(buf, size) \
     ws_log_buffer_full(_LOG_DOMAIN, LOG_LEVEL_DEBUG, \
                         __FILE__, __LINE__, __func__, \
-                        buf, size, 72, #buf)
+                        buf, size, 36, #buf)
 
 #ifdef WS_DISABLE_DEBUG
 #define ws_log_buffer(buf, size) \