Clean up color handling.

For Windows:

Don't use ws_isatty() - which is a wrapper around _isatty() - to
determine whether the output is to a Windows console or not; it returns
a non-zero value for *any* character device, not just a console.
Instead, use a console API; if it succeeds, it's a console, otherwise it
isn't.

If we're writing to a console, and the
ENABLE_VIRTUAL_TERMINAL_PROCESSING flag is set, or it isn't set but we
can set it, assume the console supports the escape sequences that
request 24-bit color, and use them.

For UN*X:

We can isatty() to determine if the output is to a terminal, as it
doesn't check for character special files, it specifically checks for
terminals (which, in practice, means "device that supports one of the
ioctls to get terminal modes" in most if not all cases; that covers
serial lines, pseudo-ttys, and perhaps some other devices).

Only use the 24-bit color escape sequences if the COLORTERM environment
variable is set to "truecolor" or "24bit".

Bug: 15659
Change-Id: I673667b86bd6b2ab48c06e00ed16b537d6723453
Reviewed-on: https://code.wireshark.org/review/32689
Petri-Dish: Guy Harris <guy@alum.mit.edu>
Tested-by: Petri Dish Buildbot
Reviewed-by: Guy Harris <guy@alum.mit.edu>

1 files changed, 276 insertions, 93 deletions

diff --git a/epan/print_stream.c b/epan/print_stream.c
index 71037e20f3..296ef0a19b 100644
--- a/epan/print_stream.c
+++ b/epan/print_stream.c
@@ -17,6 +17,13 @@
 
 #ifdef _WIN32
 #include <windows.h>
+
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
+#endif /* ENABLE_VIRTUAL_TERMINAL_PROCESSING */
+#else
+#include <stdlib.h> /* for getenv() */
+#include <unistd.h> /* for isatty() */
 #endif
 
 #include <glib.h>
@@ -30,77 +37,41 @@
 #define TERM_SGR_RESET "\x1B[0m"  /* SGR - reset */
 #define TERM_CSI_EL    "\x1B[K"   /* EL - Erase in Line (to end of line) */
 
+typedef enum {
+    COLOR_NONE,
+#ifdef _WIN32
+    COLOR_CONSOLE,
+#endif
+    COLOR_24BIT_ESCAPE
+} color_type_t;
+
 typedef struct {
-    gboolean    to_file;
-    FILE       *fh;
-    gboolean    isatty;
-    const char *to_codeset;
+    gboolean     to_file;
+    FILE        *fh;
+    gboolean     isatty;
+    const char  *to_codeset;
+    color_type_t color_type;
 #ifdef _WIN32
-    WORD        csb_attrs;
+    WORD         csb_attrs;
+    DWORD        console_mode;
 #endif
 } output_text;
 
+#ifdef _WIN32
+/*
+ * The classic Windows Console offers 1-bit color, so you can't set
+ * the red, green, or blue intensities, you can only set
+ * "{foreground, background} contains {red, green, blue}". So
+ * include red, green or blue if the numeric intensity is high
+ * enough.
+ */
 static void
-print_color_escape(FILE *fh, const color_t *fg, const color_t *bg)
+set_color_console(FILE *fh, const color_t *fg, const color_t *bg)
 {
-#ifdef _WIN32
     /* default to white foreground, black background */
     WORD win_fg_color = FOREGROUND_RED|FOREGROUND_BLUE|FOREGROUND_GREEN;
     WORD win_bg_color = 0;
 
-    /* The classic Windows Console offers 1-bit color, so you can't set
-     * the red, green, or blue intensities, you can only set
-     * "{foreground, background} contains {red, green, blue}". So
-     * include red, green or blue if the numeric intensity is high
-     * enough.
-     *
-     * The console in Windows 10 version 1511 (TH2), build 10586, and later
-     * supports SGR escape sequences:
-     *
-     *  http://www.nivot.org/blog/post/2016/02/04/Windows-10-TH2-(v1511)-Console-Host-Enhancements
-     *
-     * but only supports 16 colors.  The "undocumented" 0x04 bit to which
-     * they refer is documented in the current version of the SetConsoleMode()
-     * documentation:
-     *
-     *  https://docs.microsoft.com/en-us/windows/console/setconsolemode
-     *
-     * as ENABLE_VIRTUAL_TERMINAL_PROCESSING, saying
-     *
-     *  When writing with WriteFile or WriteConsole, characters are parsed
-     *  for VT100 and similar control character sequences that control cursor
-     *  movement, color/font mode, and other operations that can also be
-     *  performed via the existing Console APIs. For more information, see
-     *  Console Virtual Terminal Sequences.
-     *
-     * Console Virtual Terminal Sequences:
-     *
-     *  https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
-     *
-     * documents all the escape sequences the Console supports.  It currently
-     * seems to indicate that the ODA versions with 24-bit color are supported
-     * but select the closest color from the 16-color palette.
-     *
-     * The console in Windows 10 builds 14931 (a preview version of Windows 10
-     * version 1703) and later supports SGR RGB sequences:
-     *
-     *	https://devblogs.microsoft.com/commandline/24-bit-color-in-the-windows-console/
-     *
-     * That page says:
-     *
-     *   Thanks to our ability to run Linux apps and scripts using our new
-     *   Bash on Ubuntu on Windows environment atop the Windows Subsystem
-     *   for Linux (WSL), we can use some Linux scripts and tools to
-     *   demonstrate the Console's new 24-bit color support:
-     *
-     * which suggests that, with that version, whatever escape sequences
-     * work on UN*Xes also work on Windows, so maybe they support full
-     * 24-bit color with the ODA sequences.
-     *
-     * We might want to print those instead depending on the version of
-     * Windows or just remove the SetConsoleTextAttribute calls and only
-     * print SGR sequences if they are supported.
-     */
     if (fg) {
         if (((fg->red >> 8) & 0xff) >= 0x80)
         {
@@ -156,10 +127,16 @@ print_color_escape(FILE *fh, const color_t *fg, const color_t *bg)
     }
 
     SetConsoleTextAttribute((HANDLE)_get_osfhandle(_fileno(fh)), win_fg_color|win_bg_color);
-#else
+}
+#endif
+
+/*
+ * Use the SGR escape sequences to specify a 24-bit color.
+ */
+static void
+set_color_24bit_escape(FILE *fh, const color_t *fg, const color_t *bg)
+{
     /*
-     * UN*X.
-     *
      * Use the "select character foreground colour" and "select character
      * background colour" options to the Select Graphic Rendition control
      * sequence; those are reserved in ECMA-48, and are specified in ISO
@@ -173,19 +150,20 @@ print_color_escape(FILE *fh, const color_t *fg, const color_t *bg)
      *
      * For more than you ever wanted to know about all of this, see
      *
+     *    https://github.com/termstandard/colors
+     *
+     * and
+     *
      *    https://gist.github.com/XVilka/8346728
      *
-     * including the discussion following it.
+     * including the discussion following it, and
      *
-     * XXX - this isn't always treated correctly; macOS Terminal currently
-     * doesn't handle this correctly - it gives weird colors.  Sadly, as
-     * per various other discussions mentioned in the discussion cited above,
-     * there's nothing in terminfo to indicate the presence of 24-bit color
-     * support, so there's no good way to decide whether to use this or not.
+     *    https://en.wikipedia.org/wiki/ANSI_escape_code#Colors
      *
-     * XXX - fall back on 8-color or 256-color support if we can somehow
-     * determine that 24-bit color support isn't available but 8-color or
-     * 256-color support is?
+     * They are also supported by versions of the Windows Console that
+     * allow setting the ENABLE_VIRTUAL_TERMINAL_PROCESSING mode; that
+     * mode tells the console to interpret escape sequences written
+     * to it.
      */
     if (fg) {
         fprintf(fh, "\x1B[38;2;%u;%u;%um",
@@ -200,18 +178,25 @@ print_color_escape(FILE *fh, const color_t *fg, const color_t *bg)
                 (bg->green >> 8) & 0xff,
                 (bg->blue  >> 8) & 0xff);
     }
-#endif
 }
 
+#ifdef _WIN32
 static void
-print_color_eol(print_stream_t *self)
+do_color_eol_console(print_stream_t *self)
 {
     output_text *output = (output_text *)self->data;
     FILE *fh = output->fh;
-#ifdef _WIN32
+
     SetConsoleTextAttribute((HANDLE)_get_osfhandle(_fileno(fh)), output->csb_attrs);
     fprintf(fh, "\n");
-#else // UN*X
+}
+#endif
+
+static void
+do_color_eol_24bit_escape(print_stream_t *self)
+{
+    output_text *output = (output_text *)self->data;
+    FILE *fh = output->fh;
 
     /*
      * Emit CSI EL to extend current background color all the way to EOL,
@@ -220,7 +205,6 @@ print_color_eol(print_stream_t *self)
      * works.
      */
     fprintf(fh, "%s\n%s", TERM_CSI_EL, TERM_SGR_RESET);
-#endif
 }
 
 static FILE *
@@ -312,9 +296,23 @@ print_line_color_text(print_stream_t *self, int indent, const char *line, const
         memset(spaces, ' ', sizeof(spaces));
 
     if (emit_color) {
-        print_color_escape(output->fh, fg, bg);
-        if (ferror(output->fh))
-            return FALSE;
+        switch (output->color_type) {
+
+        case COLOR_NONE:
+            break;
+
+#ifdef _WIN32
+        case COLOR_CONSOLE:
+            set_color_console(output->fh, fg, bg);
+            break;
+#endif
+
+        case COLOR_24BIT_ESCAPE:
+            set_color_24bit_escape(output->fh, fg, bg);
+            if (ferror(output->fh))
+                return FALSE;
+            break;
+        }
     }
 
     /* Prepare the tabs for printing, depending on tree level */
@@ -357,9 +355,24 @@ print_line_color_text(print_stream_t *self, int indent, const char *line, const
         }
 
 
-        if (emit_color)
-            print_color_eol(self);
-        else
+        if (emit_color) {
+            switch (output->color_type) {
+
+            case COLOR_NONE:
+                putc('\n', output->fh);
+                break;
+
+#ifdef _WIN32
+            case COLOR_CONSOLE:
+                do_color_eol_console(self);
+                break;
+#endif
+
+            case COLOR_24BIT_ESCAPE:
+                do_color_eol_24bit_escape(self);
+                break;
+            }
+        } else
             putc('\n', output->fh);
     }
 
@@ -387,6 +400,12 @@ destroy_text(print_stream_t *self)
     output_text *output = (output_text *)self->data;
     gboolean     ret;
 
+#ifdef _WIN32
+    /* Restore the console mode before we changed it. */
+    if (output->color_type == COLOR_24BIT_ESCAPE)
+        SetConsoleMode((HANDLE)_get_osfhandle(_fileno(output->fh)), output->console_mode);
+#endif
+
     ret = close_print_dest(output->to_file, output->fh);
     g_free(output);
     g_free(self);
@@ -412,13 +431,26 @@ print_stream_text_alloc(gboolean to_file, FILE *fh)
     output          = (output_text *)g_malloc(sizeof *output);
     output->to_file = to_file;
     output->fh      = fh;
-    output->isatty  = ws_isatty(ws_fileno(fh));
 
-    if (output->isatty) {
 #ifdef _WIN32
-        CONSOLE_SCREEN_BUFFER_INFO csb_info;
+    /*
+     * On Windows, "_isatty()", which is what ws_isatty() wraps,
+     * "determines whether fd is associated with a character device
+     * (a terminal, console, printer, or serial port)".
+     *
+     * We specifically want to know if it's assciated with a *console*,
+     * as, if it is, we'll be using console-specific APIs.
+     */
+    CONSOLE_SCREEN_BUFFER_INFO csb_info;
+    DWORD console_mode;
 
-        GetConsoleScreenBufferInfo((HANDLE)_get_osfhandle(_fileno(fh)), &csb_info);
+    if (GetConsoleScreenBufferInfo((HANDLE)_get_osfhandle(_fileno(fh)),
+        &csb_info)) {
+        /*
+         * The console-specific API GetConsoleScreenBufferInfo() succeeded,
+         * so we'll assume this is a console.
+         */
+        output->isatty = TRUE;
         output->csb_attrs = csb_info.wAttributes;
 
         /*
@@ -427,7 +459,124 @@ print_stream_text_alloc(gboolean to_file, FILE *fh)
          * of Unicode on Windows, which is little-endian UTF-16.
          */
         output->to_codeset = "UTF-16LE";
+
+        /*
+         * As indicated above, the classic Windows Console only offers
+         * 1-bit color, set through special console APIs.
+         *
+         * The console in Windows 10 version 1511 (TH2), build 10586, and
+         * later supports SGR escape sequences:
+         *
+         *  http://www.nivot.org/blog/post/2016/02/04/Windows-10-TH2-(v1511)-Console-Host-Enhancements
+         *
+         * but only supports 16 colors.  The "undocumented" 0x04 bit to
+         * which they refer is documented in the current version of the
+         * SetConsoleMode() documentation:
+         *
+         *   https://docs.microsoft.com/en-us/windows/console/setconsolemode
+         *
+         * as ENABLE_VIRTUAL_TERMINAL_PROCESSING, saying
+         *
+         *   When writing with WriteFile or WriteConsole, characters are
+         *   parsed for VT100 and similar control character sequences that
+         *   control cursor movement, color/font mode, and other operations
+         *   that can also be performed via the existing Console APIs. For
+         *   more information, see Console Virtual Terminal Sequences.
+         *
+         * Console Virtual Terminal Sequences:
+         *
+         *   https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences
+         *
+         * documents all the escape sequences the Console supports.  It
+         * currently seems to indicate that the ODA versions with 24-bit
+         * color are supported but select the closest color from the
+         * 16-color palette.
+         *
+         * The console in Windows 10 builds 14931 (a preview version of
+         * Windows 10 version 1703) and later supports SGR RGB sequences:
+         *
+         *   https://devblogs.microsoft.com/commandline/24-bit-color-in-the-windows-console/
+         *
+         * That page says:
+         *
+         *   Thanks to our ability to run Linux apps and scripts using our
+         *   new Bash on Ubuntu on Windows environment atop the Windows
+         *   Subsystem for Linux (WSL), we can use some Linux scripts and
+         *   tools to demonstrate the Console's new 24-bit color support:
+         *
+         * which suggests that, with that version, whatever escape sequences
+         * work on UN*Xes also work on Windows, so maybe they support full
+         * 24-bit color with the ODA sequences.
+         *
+         * So, if ENABLE_VIRTUAL_TERMINAL_PROCESSING is already set on
+         * the console, or if it isn't but we can set it, we use the SGR
+         * sequences to set colors, otherwise, we just use the
+         * SetConsoleTextAttribute calls.
+         */
+        GetConsoleMode((HANDLE)_get_osfhandle(_fileno(fh)), &output->console_mode);
+        if (output->console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) {
+            /*
+             * It's already enabled; assume that means we can use the
+             * 24-bit color escape sequences (although the console might
+             * not support full 24-bit color, and would map the 24-bit
+             * color to the closest color in a smaller palette).
+             */
+            output->color_type = COLOR_24BIT_ESCAPE;
+        } else {
+            /*
+             * See if we can enable it.
+             */
+            console_mode = output->console_mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+            if (!SetConsoleMode((HANDLE)_get_osfhandle(_fileno(fh)), console_mode)) {
+                /*
+                 * We can't - use console-mode color.
+                 *
+                 * It's not documented which error is returned if
+                 * you try to set a mode bit that's not supported,
+                 * but, at least on Windows 7, ERROR_INVALID_PARAMETER
+                 * is returned if you try to set
+                 * ENABLE_VIRTUAL_TERMINAL_PROCESSING.
+                 *
+                 * We could check for that error and report other
+                 * errors as failures.
+                 */
+                output->color_type = COLOR_CONSOLE;
+            } else {
+                /* We can - use 24-bit color */
+                output->color_type = COLOR_24BIT_ESCAPE;
+            }
+        }
+    } else {
+        /*
+         * GetConsoleScreenBufferInfo() failed; it's not documented
+         * whether a particular error means "not a console", so we'll
+         * just assume this means it's not a console.
+         *
+         * The error we see on Windows 7 is ERROR_INVALID_HANDLE, but
+         * "invalid" is vague enough that I'm not sure we should
+         * treat that as meaning "not a console and everything else
+         * as being an error that we should report.
+         */
+        output->isatty = FALSE;
+
+        /*
+         * This is not used if we're not on a console, as we're not doing
+         * coloring.
+         */
+        output->csb_attrs = 0;
+    }
 #else
+    /*
+     * On UN*X, isatty() tests "whether fildes, an open file descriptor,
+     * is associated with a terminal device", to quote the Single UNIX
+     * Specification, and the documentation for UN*Xes that haven't
+     * been tested against the SUS validation suite say similar things.
+     * It does *not* just test whether it's associated with a character
+     * device that may or may not be a terminal device, so it's what we
+     * want on UN*X.
+     */
+    output->isatty = isatty(ws_fileno(fh));
+    if (output->isatty) {
         const gchar *charset;
         gboolean is_utf8;
 
@@ -447,16 +596,50 @@ print_stream_text_alloc(gboolean to_file, FILE *fh)
              */
             output->to_codeset = NULL;
         }
-#endif
-    } else {
+
         /*
-         * Not used if we're not on a console; we're not doing
-         * coloring or mapping from UTF-8 to a local character set.
+         * Not all UN*X terminal emulators support the 24-bit color SGR
+         * sequences (for example, macOS Terminal currently doesn't).
+         *
+         * As per
+         *
+         *    https://github.com/termstandard/colors
+         *
+         * terminfo currently doesn't have a flag to indicate 24-bit
+         * color support - a future release will - so we can't use
+         * that to determine if the terminal emulator (or terminal)
+         * supports it.
+         *
+         * That page notes that some terminal emulators set the
+         * COLORTERM environment variable either to "truecolor"
+         * or "24bit" if 24-bit color is supported; we use that
+         * test for now.
+         *
+         * XXX - if there are terminal emulators that use the 24-bit
+         * color escape sequences but don't set COLORTERM, add code
+         * here to look at other environment variables to try to
+         * recognize them.
+         *
+         * XXX - fall back on 8-color or 256-color support if we can
+         * somehow determine that 24-bit color support isn't available
+         * but 8-color or 256-color support is?
          */
-#ifdef _WIN32
-        output->csb_attrs = 0;
+        char *colorterm = getenv("COLORTERM");
+        if (colorterm != NULL &&
+            (strcmp(colorterm, "truecolor") == 0 || strcmp(colorterm, "24bit") == 0))
+            output->color_type = COLOR_24BIT_ESCAPE;
+        else
+            output->color_type = COLOR_NONE;
+    }
 #endif
+    if (!output->isatty) {
+        /*
+         * OK, this was determined *not* to be a terminal, so we won't
+         * be doing coloring or mapping from UTF-8 to a local character
+         * set.
+         */
         output->to_codeset = NULL;
+        output->color_type = COLOR_NONE;
     }
 
     stream          = (print_stream_t *)g_malloc(sizeof (print_stream_t));