androiddump: Fix and simplify tcpdump capture

1. Use "exec:" ADB command to get raw (non-PTY) tcpdump output
This is also supported on Android devices before Android 7, and is a
much easier approach than testing the new "shell,raw:" command and
falling back if unsupported. This basically undoes commit 5ebc3277.

2. Pass "-U" to tcpdump to prevent on-target buffering
Before using the "shell,raw" approach in commit 5ebc3277, I tried the
"exec:" command already, but experienced extreme buffering of the
tcpdump output, which is unacceptable for live trace viewing.
Turns out, the buffering is determined "automatically" by libpcap:
- When running in a PTY, output is flushed fast for viewing
- When _not_ in a PTY, output is not flushed and thus heavily buffered.
The "exec" command obviously doesn't use a PTY.
Fortunately, tcpdump has a "-U" option to flush the output after each
catpured packet, which is exactly what we need.

3. Ignore tcpdump stderr output
Enabling "-U" caused androiddump to fail, because it happened that the
tcpdump stderr logs were mixed with the stdout data. (We were probably
lucky this didn't happen without -U as well).
To fix this, we just ignore stderr completely by adding "2>/dev/null" to
the tcpdump command.

4. Get linktype from pcap global header
The stderr logs were previously parsed to get the textual linktype.
This is now replaced by a simpler & less fragile approach: tcpdump
prints the global pcap header, which contains precicesly the linktype
info we need.

5. Parse pcap global header magic correctly for timestamps & endianness
The previous code only supported the "classic" pcap header magic and
might also been incorrect on big-endian host machines.
Now, endian handling is simplified and we can detect the "nanosecond
timestamp" magic values as well.
This fixes the problem that extcap_dumper_dump expects *nano*second
timestamps, but the previous code supplied *micro*seconds if on-target
tcpdump outputs microseconds.

6. The parsing simplifications above allowed the main loop for tcpdump
capture to be simplified considerably.

Change-Id: Id66791e700a8943b86128f044f080bee60a9fa79
Reviewed-on: https://code.wireshark.org/review/25713
Petri-Dish: Michael Mann <mmann78@netscape.net>
Petri-Dish: Anders Broman <a.broman58@gmail.com>
Tested-by: Petri Dish Buildbot
Reviewed-by: Anders Broman <a.broman58@gmail.com>

1 files changed, 89 insertions, 192 deletions

diff --git a/extcap/androiddump.c b/extcap/androiddump.c
index 7f82ed26cb..15d9d9c2a6 100644
--- a/extcap/androiddump.c
+++ b/extcap/androiddump.c
@@ -172,7 +172,6 @@ enum exit_code {
     EXIT_CODE_INVALID_SOCKET_9,
     EXIT_CODE_INVALID_SOCKET_10,
     EXIT_CODE_INVALID_SOCKET_11,
-    EXIT_CODE_INVALID_SOCKET_12,
     EXIT_CODE_GENERIC = -1
 };
 
@@ -224,6 +223,17 @@ typedef struct _own_pcap_bluetooth_h4_header {
     uint32_t direction;
 } own_pcap_bluetooth_h4_header;
 
+typedef struct pcap_hdr_s {
+        guint32 magic_number;   /* magic number */
+        guint16 version_major;  /* major version number */
+        guint16 version_minor;  /* minor version number */
+        gint32  thiszone;       /* GMT to local correction */
+        guint32 sigfigs;        /* accuracy of timestamps */
+        guint32 snaplen;        /* max length of captured packets, in octets */
+        guint32 network;        /* data link type */
+} pcap_hdr_t;
+
+
 typedef struct pcaprec_hdr_s {
         guint32 ts_sec;         /* timestamp seconds */
         guint32 ts_usec;        /* timestamp microseconds */
@@ -2269,36 +2279,6 @@ static int capture_android_logcat(char *interface, char *fifo,
     return EXIT_CODE_SUCCESS;
 }
 
-/* Translate tcpdump data link type strings to EXTCAP_ENCAP_ types
- * For info about available data link types see:
- * http://www.tcpdump.org/linktypes.html
- */
-static int linktype_to_extcap_encap(const char* linktype)
-{
-    struct dlt_encap {
-        int extcap_encap;
-        const char* const dlt;
-    };
-    const struct dlt_encap lookup[] = {
-        { EXTCAP_ENCAP_LINUX_SLL, "LINUX_SLL" },
-        { EXTCAP_ENCAP_ETHERNET, "EN10MB" },
-        { EXTCAP_ENCAP_IEEE802_11_RADIO, "IEEE802_11_RADIO" },
-        { EXTCAP_ENCAP_NETLINK, "NETLINK" },
-        { -1, NULL }
-    };
-    int i;
-    int ret = EXTCAP_ENCAP_ETHERNET;
-
-    if (!linktype)
-        return ret;
-    for (i = 0; lookup[i].dlt; i++) {
-        if (!strcmp(lookup[i].dlt, linktype)) {
-            ret = lookup[i].extcap_encap;
-        }
-    }
-    return ret;
-}
-
 
 /*----------------------------------------------------------------------------*/
 /* Android Wifi Tcpdump                                                       */
@@ -2307,30 +2287,25 @@ static int linktype_to_extcap_encap(const char* linktype)
 /*----------------------------------------------------------------------------*/
 static int capture_android_tcpdump(char *interface, char *fifo,
         const char *adb_server_ip, unsigned short *adb_server_tcp_port) {
-    static const char                       *const adb_shell_tcpdump_format = "shell,raw:tcpdump -n -s 0 -u -i %s -w -";
-    static const char                       *const adb_shell_legacy_tcpdump_format = "shell:tcpdump -n -s 0 -u -i %s -w -";
+    static const char                       *const adb_shell_tcpdump_format = "exec:tcpdump -U -n -s 0 -u -i %s -w - 2>/dev/null";
     static const char                       *const regex_interface = INTERFACE_ANDROID_TCPDUMP "-(?<iface>.*?)-(?<serial>.*)";
-    static const char                       *const regex_linktype = "tcpdump: listening on .*?, link-type (?<linktype>.*?) ";
     struct extcap_dumper                     extcap_dumper;
     static char                              data[PACKET_LENGTH];
     gssize                                   length;
     gssize                                   used_buffer_length =  0;
-    gssize                                   filter_buffer_length = 0;
     gssize                                   frame_length=0;
     socket_handle_t                          sock;
     gint                                     result;
     char                                    *iface = NULL;
     char                                    *serial_number = NULL;
-    static char                              filter_buffer[PACKET_LENGTH];
-    gint                                     device_endiness = G_LITTLE_ENDIAN;
-    gboolean                                 global_header_skipped=FALSE;
+    gboolean                                 nanosecond_timestamps;
+    gboolean                                 swap_byte_order;
+    pcap_hdr_t                              *global_header;
     pcaprec_hdr_t                            p_header;
-    char                                    *linktype = NULL;
     GRegex                                  *regex = NULL;
     GError                                  *err = NULL;
     GMatchInfo                              *match = NULL;
     char                                     tcpdump_cmd[80];
-    gboolean                                 pty_mode = FALSE;
 
     regex = g_regex_new(regex_interface, (GRegexCompileFlags)0, (GRegexMatchFlags)0, &err);
     if (!regex) {
@@ -2352,51 +2327,24 @@ static int capture_android_tcpdump(char *interface, char *fifo,
 
     /* First check for the device if it is connected or not */
     sock = adb_connect_transport(adb_server_ip, adb_server_tcp_port, serial_number);
+    g_free(serial_number);
     if (sock == INVALID_SOCKET) {
         g_free(iface);
-        g_free(serial_number);
         return EXIT_CODE_INVALID_SOCKET_11;
     }
 
-    /* Try the new raw (non-PTY) shell protocol first */
     g_snprintf(tcpdump_cmd, sizeof(tcpdump_cmd), adb_shell_tcpdump_format, iface);
+    g_free(iface);
     result = adb_send(sock, tcpdump_cmd);
     if (result) {
-        g_debug("Target does not support raw shell protocol");
-        closesocket(sock);
-
-        /* Fall back to the old PTY shell */
-        sock = adb_connect_transport(adb_server_ip, adb_server_tcp_port, serial_number);
-        if (sock == INVALID_SOCKET) {
-            g_free(iface);
-            g_free(serial_number);
-            return EXIT_CODE_INVALID_SOCKET_12;
-        }
-
-        pty_mode = TRUE;
-        g_snprintf(tcpdump_cmd, sizeof(tcpdump_cmd), adb_shell_legacy_tcpdump_format, iface);
-        result = adb_send(sock, tcpdump_cmd);
-    }
-    if (result) {
         g_warning("Error while setting adb transport");
         closesocket(sock);
         return EXIT_CODE_GENERIC;
     }
 
-    g_free(iface);
-    g_free(serial_number);
-
-    regex = g_regex_new(regex_linktype, (GRegexCompileFlags)0, (GRegexMatchFlags)0, &err);
-    if (!regex) {
-        g_warning("Failed to compile regex for tcpdump data link type matching");
-        closesocket(sock);
-        return EXIT_CODE_GENERIC;
-    }
-
-    while (endless_loop) {
-        char *i_position;
+    while (used_buffer_length < PCAP_GLOBAL_HEADER_LENGTH) {
         errno = 0;
-        length = recv(sock, data + used_buffer_length, (int)(PACKET_LENGTH - used_buffer_length), 0);
+        length = recv(sock, data + used_buffer_length, (int)(PCAP_GLOBAL_HEADER_LENGTH - used_buffer_length), 0);
         if (errno == EAGAIN
 #if EWOULDBLOCK != EAGAIN
             || errno == EWOULDBLOCK
@@ -2407,155 +2355,104 @@ static int capture_android_tcpdump(char *interface, char *fifo,
         else if (errno != 0) {
             g_warning("ERROR capture: %s", strerror(errno));
             closesocket(sock);
-            g_regex_unref(regex);
             return EXIT_CODE_GENERIC;
         }
 
         if (length <= 0) {
             g_warning("Broken socket connection.");
             closesocket(sock);
-            g_regex_unref(regex);
             return EXIT_CODE_GENERIC;
         }
 
         used_buffer_length += length;
+    }
 
-        /*
-         * Checking for the starting for the pcap global header using the magic number
-         */
-        if (used_buffer_length > 4) {
-            guint * magic_number;
-            magic_number= (guint *)data;
-            if (*magic_number == 0xd4c3b2a1 || *magic_number == 0xa1b2c3d4) {
-                if (data[0] == (char)0xd4){
-                    device_endiness = G_LITTLE_ENDIAN;
-                }
-                else {
-                    device_endiness = G_BIG_ENDIAN;
-                }
-                break;
-            }
-        }
-
-        g_regex_match(regex, data, (GRegexMatchFlags)0, &match);
-        if (g_match_info_matches(match)) {
-            g_free(linktype);
-            linktype = g_match_info_fetch_named(match, "linktype");
-        }
-        g_match_info_free(match);
-        i_position = (char *) memchr(data, '\n', used_buffer_length);
-        if (i_position && i_position < data + used_buffer_length) {
-            memmove(data, i_position + 1 , used_buffer_length - (i_position + 1 - data));
-            used_buffer_length = used_buffer_length - (gssize) (i_position + 1 - data);
-        }
+    global_header = (pcap_hdr_t*) data;
+    switch (global_header->magic_number) {
+    case 0xa1b2c3d4:
+        swap_byte_order = FALSE;
+        nanosecond_timestamps = FALSE;
+        break;
+    case 0xd4c3b2a1:
+        swap_byte_order = TRUE;
+        nanosecond_timestamps = FALSE;
+        break;
+    case 0xa1b23c4d:
+        swap_byte_order = FALSE;
+        nanosecond_timestamps = TRUE;
+        break;
+    case 0x4d3cb2a1:
+        swap_byte_order = TRUE;
+        nanosecond_timestamps = TRUE;
+        break;
+    default:
+        g_warning("Received incorrect magic");
+        closesocket(sock);
+        return EXIT_CODE_GENERIC;
     }
-    g_regex_unref(regex);
-    extcap_dumper = extcap_dumper_open(fifo, linktype_to_extcap_encap(linktype));
-    g_free(linktype);
 
-    filter_buffer_length=0;
+    extcap_dumper = extcap_dumper_open(fifo, (int) data[20]);
+
+    used_buffer_length = 0;
     while (endless_loop) {
-        gssize i = 0,read_offset,j=0;
-
-        /*
-         * Before Android 7 adb runs tcpdump in a shell/pseudoterminal and the PTY layer on the target converts
-         * all \n to \r\n. In that case we need to undo this by changing all \r\n (0x0d0a) back to \n (0x0a).
-         */
-        for (i = 0; i < (used_buffer_length - 1); i++) {
-            if (pty_mode && data[i] == 0x0d && data[i + 1] == 0x0a) {
-                i++;
-            }
-            filter_buffer[filter_buffer_length++] = data[i];
-        }
+        gssize offset = 0;
 
-        /* Put the last characters in the start if it is still left in buffer.*/
-        for (j=0; i < used_buffer_length; i++,j++) {
-            data[j] = data[i];
+        errno = 0;
+        length = recv(sock, data + used_buffer_length, (int)(PACKET_LENGTH - used_buffer_length), 0);
+        if (errno == EAGAIN
+#if EWOULDBLOCK != EAGAIN
+            || errno == EWOULDBLOCK
+#endif
+            ) {
+            continue;
         }
-        used_buffer_length = j;
-        if (global_header_skipped==FALSE && filter_buffer_length >= PCAP_GLOBAL_HEADER_LENGTH) {
-            /*Skip the Global pcap header*/
-            filter_buffer_length -= PCAP_GLOBAL_HEADER_LENGTH;
-
-            /*Move the remaining content from start*/
-            memmove(filter_buffer , filter_buffer + PCAP_GLOBAL_HEADER_LENGTH , filter_buffer_length);
-            global_header_skipped = TRUE;
+        else if (errno != 0) {
+            g_warning("ERROR capture: %s", strerror(errno));
+            closesocket(sock);
+            return EXIT_CODE_GENERIC;
         }
-        else if (global_header_skipped && filter_buffer_length > PCAP_RECORD_HEADER_LENGTH) {
-            read_offset=0;
-            while (filter_buffer_length > PCAP_RECORD_HEADER_LENGTH) {
-                gchar *packet;
-                packet = filter_buffer + read_offset;
-                 /*
-                 * This fills the pcap header info based upon the endianess of the machine and android device.
-                 * If the endianess are different, pcap header bytes received from the android device are swapped
-                 * to be read properly by the machine else pcap header bytes are taken as it is.
-                 */
-                if (device_endiness == G_BYTE_ORDER) {
-                    p_header = *((pcaprec_hdr_t*)packet);
-                }
-                else {
-                    p_header.ts_sec = GUINT32_SWAP_LE_BE(*((guint32*)packet));
-                    p_header.ts_usec = GUINT32_SWAP_LE_BE(*(guint32*)(packet +4));
-                    p_header.incl_len = GUINT32_SWAP_LE_BE(*(guint32*)(packet +8));
-                    p_header.orig_len = GUINT32_SWAP_LE_BE(*(guint32*)(packet +12));
-                }
 
-                if ((gssize)(p_header.incl_len + PCAP_RECORD_HEADER_LENGTH) <= filter_buffer_length) {
+        if (length <= 0) {
+            g_warning("Broken socket connection.");
+            closesocket(sock);
+            return EXIT_CODE_GENERIC;
+        }
 
-                    /*
-                     * It was observed that some times tcpdump reports the length of packet as '0' and that leads to the
-                     * ( Warn Error "Less data was read than was expected" while reading )
-                     * So to avoid this error we are checking for length of packet before passing it to dumper.
-                     */
-                    if (p_header.incl_len > 0) {
-                        endless_loop = extcap_dumper_dump(extcap_dumper, fifo, filter_buffer + read_offset+ PCAP_RECORD_HEADER_LENGTH,
-                        p_header.incl_len , p_header.orig_len , p_header.ts_sec , p_header.ts_usec);
-                    }
-                    frame_length = p_header.incl_len + PCAP_RECORD_HEADER_LENGTH;
+        used_buffer_length += length;
 
-                    /*update the offset value for the next packet*/
-                    filter_buffer_length -= frame_length;
-                    read_offset += frame_length;
-                }
-                else {
-                    /*The complete packet has not yet received*/
-                    break;
-                }
+        while ((used_buffer_length - offset) > PCAP_RECORD_HEADER_LENGTH) {
+            p_header = *((pcaprec_hdr_t*) (data + offset));
+            if (swap_byte_order) {
+                p_header.ts_sec   = GUINT32_SWAP_LE_BE(p_header.ts_sec);
+                p_header.ts_usec  = GUINT32_SWAP_LE_BE(p_header.ts_usec);
+                p_header.incl_len = GUINT32_SWAP_LE_BE(p_header.incl_len);
+                p_header.orig_len = GUINT32_SWAP_LE_BE(p_header.orig_len);
             }
-            if (read_offset!=0) {
-                /*Move the rest of the filter  data to the beginning of the filter_buffer */
-                memmove(filter_buffer, filter_buffer + read_offset , filter_buffer_length);
+            if (!nanosecond_timestamps) {
+                p_header.ts_usec = p_header.ts_usec * 1000;
             }
-        }
 
-        /*Get the data from the tcpdump process running in the android device*/
-        while (endless_loop) {
-            errno = 0;
-            length = recv(sock, data + used_buffer_length, (int)(PACKET_LENGTH -(used_buffer_length + filter_buffer_length)), 0);
-            if (errno == EAGAIN
-#if EWOULDBLOCK != EAGAIN
-                || errno == EWOULDBLOCK
-#endif
-                ) {
-                continue;
-            }
-            else if (errno != 0) {
-                g_warning("ERROR capture: %s", strerror(errno));
-                closesocket(sock);
-                return EXIT_CODE_GENERIC;
+            frame_length = p_header.incl_len + PCAP_RECORD_HEADER_LENGTH;
+            if ((used_buffer_length - offset) < frame_length) {
+                break; /* wait for complete packet */
             }
 
-            if (length <= 0) {
-                g_warning("Broken socket connection.");
-                closesocket(sock);
-                return EXIT_CODE_GENERIC;
+            /* It was observed that some times tcpdump reports the length of packet as '0' and that leads to the
+             * ( Warn Error "Less data was read than was expected" while reading )
+             * So to avoid this error we are checking for length of packet before passing it to dumper.
+             */
+            if (p_header.incl_len > 0) {
+                endless_loop = extcap_dumper_dump(extcap_dumper, fifo, data + offset + PCAP_RECORD_HEADER_LENGTH,
+                    p_header.incl_len, p_header.orig_len, p_header.ts_sec, p_header.ts_usec);
             }
 
-            if ((used_buffer_length += length) > 1) {
-                break;
-            }
+            offset += frame_length;
+        }
+
+        if (offset < used_buffer_length) {
+            memmove(data, data + offset, used_buffer_length - offset);
         }
+        used_buffer_length -= offset;
     }
 
     closesocket(sock);