wslua: fix memleaks related to Proto and Pref

Proto objects were only freed while reloading Lua plugins, be sure to
release these on program exit too. Fix missing deallocation of heur_list
(matches per-protocol cleanup in proto_cleanup_base).

Be sure to keep a reference to the "Pref" object after registering it to
a Proto, otherwise it could be garbage-collected early, resulting in
memleaks (because the preference was still in use).

Fixes a lot of memory leaks reported by ASAN for tests, ten tests were
affected by Proto_new leaks, four were affected by the new_pref leaks.

Change-Id: Ica52718849a33eda614775f533dc0fcefec9cc74
Reviewed-on: https://code.wireshark.org/review/31746
Petri-Dish: Peter Wu <peter@lekensteyn.nl>
Tested-by: Petri Dish Buildbot
Reviewed-by: Peter Wu <peter@lekensteyn.nl>

4 files changed, 49 insertions, 46 deletions

diff --git a/epan/wslua/init_wslua.c b/epan/wslua/init_wslua.c
index 302aa82571..eded6eead4 100644
--- a/epan/wslua/init_wslua.c
+++ b/epan/wslua/init_wslua.c
@@ -1014,6 +1014,10 @@ void wslua_init(register_cb cb, gpointer client_data) {
     first_time = FALSE;
 }
 
+void wslua_early_cleanup(void) {
+    wslua_deregister_protocols(L);
+}
+
 void wslua_reload_plugins (register_cb cb, gpointer client_data) {
     const funnel_ops_t* ops = funnel_get_funnel_ops();
 
diff --git a/epan/wslua/wslua.h b/epan/wslua/wslua.h
index 92c7a9318a..25c5182a8c 100644
--- a/epan/wslua/wslua.h
+++ b/epan/wslua/wslua.h
@@ -165,6 +165,7 @@ typedef struct _wslua_pref_t {
 
     struct _wslua_pref_t* next;
     struct _wslua_proto_t* proto;
+    int ref;            /* Reference to enable Proto to deregister prefs. */
 } wslua_pref_t;
 
 typedef struct _wslua_proto_t {
@@ -787,6 +788,7 @@ extern void clear_outstanding_FieldInfo(void);
 extern void wslua_print_stack(char* s, lua_State* L);
 
 extern void wslua_init(register_cb cb, gpointer client_data);
+extern void wslua_early_cleanup(void);
 extern void wslua_cleanup(void);
 
 extern tap_extractor_t wslua_get_tap_extractor(const gchar* name);
diff --git a/epan/wslua/wslua_pref.c b/epan/wslua/wslua_pref.c
index 4ec4d1076f..036a4bf787 100644
--- a/epan/wslua/wslua_pref.c
+++ b/epan/wslua/wslua_pref.c
@@ -91,6 +91,7 @@ static int new_pref(lua_State* L, pref_type_t type) {
     pref->label = g_strdup(label);
     pref->desc = g_strdup(descr);
     pref->type = type;
+    pref->ref = LUA_NOREF;
 
     switch(type) {
         case PREF_BOOL: {
@@ -242,53 +243,42 @@ static range_t* get_range(lua_State *L, int idx_r, int idx_m)
 static int Pref__gc(lua_State* L) {
     Pref pref = toPref(L,1);
 
-    /*
-     * Only free never-registered and registered-and-then-deregistered
-     * Prefs; those have a null name pointer.
-     *
-     * If this has never been registered, it obviously has not been
-     * deregistered, so, if it's a string preference, we need to
-     * free the initial value in pref->info.default_s.  We don't
-     * need to free the current value, as that's the same string
-     * as the initial value.
-     *
-     * If this has been registred and deregistered, and the current
-     * value was allocated, it was freed when it was deregistered,
-     * so we don't need to free it.  If it's a string preference,
-     * the initial value was freed and the pointer to it set to
-     * NULL, so we can still call g_free() on it, as that won't
-     * do anything.
-     */
-    if (! pref->name) {
-        g_free(pref->label);
-        g_free(pref->desc);
-        switch (pref->type) {
-            case PREF_STRING:
-                /*
-                 * Free the initial string value; if it's not NULL, that
-                 * means this is a never-registered preference, so the
-                 * initial value hasn't been freed.
-                 */
-                g_free(pref->info.default_s);
-                break;
-            case PREF_ENUM: {
-                /*
-                 * Free the enum values allocated in get_enum().
-                 */
-                const enum_val_t *enum_valp = pref->info.enum_info.enumvals;
-                while (enum_valp->name) {
-                    g_free((char *)enum_valp->name);
-                    g_free((char *)enum_valp->description);
-                    enum_valp++;
-                }
-                g_free ((enum_val_t *)pref->info.enum_info.enumvals);
-                break;
+    if (pref->ref != LUA_NOREF) {
+        // Did the user try to call __gc explicitly while it was registered to a
+        // protocol? Forbid that!
+        luaL_error(L, "Direct call to __gc is forbidden");
+        return 0;
+    }
+
+    g_free(pref->name);
+    g_free(pref->label);
+    g_free(pref->desc);
+    switch (pref->type) {
+        case PREF_STRING:
+            /*
+             * Free the initial string value; if it's not NULL, that
+             * means this is a never-registered preference, so the
+             * initial value hasn't been freed.
+             */
+            g_free(pref->info.default_s);
+            break;
+        case PREF_ENUM: {
+            /*
+             * Free the enum values allocated in get_enum().
+             */
+            const enum_val_t *enum_valp = pref->info.enum_info.enumvals;
+            while (enum_valp->name) {
+                g_free((char *)enum_valp->name);
+                g_free((char *)enum_valp->description);
+                enum_valp++;
             }
-            default:
-                break;
+            g_free((enum_val_t *)pref->info.enum_info.enumvals);
+            break;
         }
-        g_free(pref);
+        default:
+            break;
     }
+    g_free(pref);
 
     return 0;
 }
@@ -364,6 +354,11 @@ WSLUA_METAMETHOD Prefs__newindex(lua_State* L) {
         }
 
         if ( ! p->next) {
+            // Keep a reference to the Pref to ensure it remains valid
+            // until the protocol is deregistered.
+            lua_pushvalue(L, WSLUA_ARG_Prefs__newindex_PREF);
+            pref->ref = luaL_ref(L, LUA_REGISTRYINDEX);
+
             p->next = pref;
             pref->name = g_strdup(name);
 
diff --git a/epan/wslua/wslua_proto.c b/epan/wslua/wslua_proto.c
index 3583b6d459..d57afebc9b 100644
--- a/epan/wslua/wslua_proto.c
+++ b/epan/wslua/wslua_proto.c
@@ -618,9 +618,11 @@ int wslua_deregister_protocols(lua_State* L) {
         if (proto->prefs_module) {
             Pref pref;
             prefs_deregister_protocol(proto->hfid);
+            /* Preferences are unregistered, now free its memory via Pref__gc */
             for (pref = proto->prefs.next; pref; pref = pref->next) {
-                g_free(pref->name);
-                pref->name = NULL; /* Deregister Pref, freed in Pref__gc */
+                int pref_ref = pref->ref;
+                pref->ref = LUA_NOREF;
+                luaL_unref(L, LUA_REGISTRYINDEX, pref_ref);
             }
         }
         if (proto->expert_module) {