Use libvlr in libmsc (large refactoring)

Original libvlr code is by Harald Welte <laforge@gnumonks.org>,
polished and tweaked by Neels Hofmeyr <nhofmeyr@sysmocom.de>.

This is a long series of development collapsed in one patch. The original
history may still be available as branch neels/vlr_orig.

TODO: This commit may be split in several smaller changes before merging to
master.

SMS:

The SQL based lookup of SMS for attached subscribers no longer works since the
SQL database no longer has the subscriber data. Replace with a round-robin on
the SMS recipient MSISDNs paired with a VLR subscriber RAM lookup whether the
subscriber is currently attached.

If there are many SMS for not-attached subscribers in the SMS database, this
will become inefficient: a DB hit returns a pending SMS, the RAM lookup will
reveal that the subscriber is not attached, after which the DB is hit for the
next SMS. It would become more efficient e.g. by having an MSISDN based hash
list for the VLR subscribers and by marking non-attached SMS recipients in the
SMS database so that they can be excluded with the SQL query already.

There is a sanity limit to do at most 100 db hits per attempt to find a pending
SMS. So if there are more than 100 stored SMS waiting for their recipients to
actually attach to the MSC, it may take more than one SMS queue trigger to
deliver SMS for subscribers that are actually attached.

This is not very beautiful, but is merely intended to carry us over to a time
when we have a proper separate SMSC entity.

Introduce gsm_subscriber_connection ref-counting in libmsc.

Related: OS#1592

Change-Id: I702ba504ce2de93507312c28eca8d11f09f4ee8b

4 files changed, 358 insertions, 0 deletions

diff --git a/openbsc/tests/sms_queue/Makefile.am b/openbsc/tests/sms_queue/Makefile.am
new file mode 100644
index 000000000..b2266ebb8
--- /dev/null
+++ b/openbsc/tests/sms_queue/Makefile.am
@@ -0,0 +1,45 @@
+AM_CPPFLAGS = \
+	$(all_includes) \
+	-I$(top_srcdir)/include \
+	$(NULL)
+
+AM_CFLAGS = \
+	-Wall \
+	-ggdb3 \
+	$(LIBOSMOCORE_CFLAGS) \
+	$(LIBOSMOGSM_CFLAGS) \
+	$(NULL)
+
+EXTRA_DIST = \
+	sms_queue_test.ok \
+	sms_queue_test.err \
+	$(NULL)
+
+noinst_PROGRAMS = \
+	sms_queue_test \
+	$(NULL)
+
+sms_queue_test_SOURCES = \
+	sms_queue_test.c \
+	$(NULL)
+
+sms_queue_test_LDADD = \
+	$(top_builddir)/src/libmsc/libmsc.a \
+	$(top_builddir)/src/libvlr/libvlr.a \
+	$(top_builddir)/src/libbsc/libbsc.a \
+	$(top_builddir)/src/libtrau/libtrau.a \
+	$(top_builddir)/src/libcommon/libcommon.a \
+	$(top_builddir)/src/libcommon-cs/libcommon-cs.a \
+	$(LIBSMPP34_LIBS) \
+	$(LIBOSMOCORE_LIBS) \
+	$(LIBOSMOGSM_LIBS) \
+	$(LIBCRYPTO_LIBS) \
+	$(LIBOSMOVTY_LIBS) \
+	$(LIBOSMOABIS_LIBS) \
+	-ldbi \
+	-lrt \
+	$(NULL)
+
+sms_queue_test_LDFLAGS = \
+	-Wl,--wrap=db_sms_get_next_unsent_rr_msisdn \
+	$(NULL)
diff --git a/openbsc/tests/sms_queue/sms_queue_test.c b/openbsc/tests/sms_queue/sms_queue_test.c
new file mode 100644
index 000000000..af25b0645
--- /dev/null
+++ b/openbsc/tests/sms_queue/sms_queue_test.c
@@ -0,0 +1,215 @@
+/* Test Osmocom SMS queue */
+
+/*
+ * (C) 2017 by sysmocom s.f.m.c. GmbH
+ * All Rights Reserved
+ *
+ * Author: Neels Hofmeyr <nhofmeyr@sysmocom.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <osmocom/core/application.h>
+
+#include <openbsc/debug.h>
+#include <openbsc/vlr.h>
+
+static void *talloc_ctx = NULL;
+
+struct gsm_sms *smsq_take_next_sms(struct gsm_network *net,
+				   char *last_msisdn,
+				   size_t last_msisdn_buflen);
+
+static void _test_take_next_sms_print(int i,
+				      struct gsm_sms *sms,
+				      const char *last_msisdn)
+{
+	printf("#%d: ", i);
+	if (sms)
+		printf("sending SMS to %s", sms->text);
+	else
+		printf("no SMS to send");
+	printf(" (last_msisdn='%s')\n", last_msisdn? last_msisdn : "NULL");
+}
+
+static struct gsm_sms fake_sms = { 0 };
+
+struct {
+	const char *msisdn;
+	int nr_of_sms;
+	int failed_attempts;
+	bool vsub_attached;
+} fake_sms_db[] = {
+	{
+		.msisdn = "1111",
+		.nr_of_sms = 0,
+		.vsub_attached = true,
+	},
+	{
+		.msisdn = "2222",
+		.nr_of_sms = 2,
+		.failed_attempts = 2,
+		.vsub_attached = true,
+	},
+	{
+		.msisdn = "3333",
+		.nr_of_sms = 2,
+		.failed_attempts = 3,
+		.vsub_attached = true,
+	},
+	{
+		.msisdn = "4444",
+		.nr_of_sms = 0,
+		.vsub_attached = true,
+	},
+	{
+		.msisdn = "5555",
+		.nr_of_sms = 2,
+		.failed_attempts = 5,
+		.vsub_attached = false,
+	},
+};
+
+/* override, requires '-Wl,--wrap=db_sms_get_next_unsent_rr_msisdn' */
+struct gsm_sms *__real_db_sms_get_next_unsent_rr_msisdn(struct gsm_network *net,
+							const char *last_msisdn,
+							unsigned int max_failed);
+struct gsm_sms *__wrap_db_sms_get_next_unsent_rr_msisdn(struct gsm_network *net,
+							const char *last_msisdn,
+							unsigned int max_failed)
+{
+	static struct vlr_subscr arbitrary_vsub = { .lu_complete = true };
+	int i;
+	printf("     hitting database: looking for MSISDN > '%s', failed_attempts <= %d\n",
+	       last_msisdn, max_failed);
+
+	for (i = 0; i < ARRAY_SIZE(fake_sms_db); i++) {
+		if (!fake_sms_db[i].nr_of_sms)
+			continue;
+		if (strcmp(fake_sms_db[i].msisdn, last_msisdn) <= 0)
+			continue;
+		if (fake_sms_db[i].failed_attempts > max_failed)
+			continue;
+		osmo_strlcpy(fake_sms.dst.addr, fake_sms_db[i].msisdn,
+			     sizeof(fake_sms.dst.addr));
+		fake_sms.receiver = fake_sms_db[i].vsub_attached? &arbitrary_vsub : NULL;
+		osmo_strlcpy(fake_sms.text, fake_sms_db[i].msisdn, sizeof(fake_sms.text));
+		if (fake_sms_db[i].vsub_attached)
+			fake_sms_db[i].nr_of_sms --;
+		return &fake_sms;
+	}
+	return NULL;
+}
+
+void show_fake_sms_db()
+{
+	int i;
+	for (i = 0; i < ARRAY_SIZE(fake_sms_db); i++) {
+		printf("  %s%s has %u SMS pending, %u failed attempts\n",
+		       fake_sms_db[i].msisdn,
+		       fake_sms_db[i].vsub_attached ? "" : " (NOT attached)",
+		       fake_sms_db[i].nr_of_sms,
+		       fake_sms_db[i].failed_attempts);
+	}
+	printf("-->\n");
+}
+
+static void test_next_sms()
+{
+	int i;
+	char last_msisdn[GSM_EXTENSION_LENGTH+1] = "";
+
+	printf("Testing smsq_take_next_sms()\n");
+
+	printf("\n- vsub 2, 3 and 5 each have 2 SMS pending, but 5 is not attached\n");
+	last_msisdn[0] = '\0';
+	show_fake_sms_db();
+	for (i = 0; i < 7; i++) {
+		struct gsm_sms *sms = smsq_take_next_sms(NULL, last_msisdn, sizeof(last_msisdn));
+		_test_take_next_sms_print(i, sms, last_msisdn);
+		OSMO_ASSERT(i >= 4 || sms);
+	}
+
+	printf("\n- SMS are pending at various nr failed attempts (cutoff at >= 10)\n");
+	last_msisdn[0] = '\0';
+	for (i = 0; i < ARRAY_SIZE(fake_sms_db); i++) {
+		fake_sms_db[i].vsub_attached = true;
+		fake_sms_db[i].nr_of_sms = 1 + i;
+		fake_sms_db[i].failed_attempts = i*5;
+
+	}
+	show_fake_sms_db();
+	for (i = 0; i < 7; i++) {
+		struct gsm_sms *sms = smsq_take_next_sms(NULL, last_msisdn, sizeof(last_msisdn));
+		_test_take_next_sms_print(i, sms, last_msisdn);
+		OSMO_ASSERT(i >= 2 || sms);
+	}
+
+	printf("\n- iterate the SMS DB at most once\n");
+	osmo_strlcpy(last_msisdn, "2345", sizeof(last_msisdn));
+	for (i = 0; i < ARRAY_SIZE(fake_sms_db); i++) {
+		fake_sms_db[i].vsub_attached = false;
+		fake_sms_db[i].nr_of_sms = 1;
+		fake_sms_db[i].failed_attempts = 0;
+	}
+	show_fake_sms_db();
+	for (i = 0; i < 3; i++) {
+		struct gsm_sms *sms = smsq_take_next_sms(NULL, last_msisdn, sizeof(last_msisdn));
+		_test_take_next_sms_print(i, sms, last_msisdn);
+		OSMO_ASSERT(!sms);
+	}
+
+	printf("\n- there are no SMS in the DB\n");
+	last_msisdn[0] = '\0';
+	for (i = 0; i < ARRAY_SIZE(fake_sms_db); i++) {
+		fake_sms_db[i].vsub_attached = true;
+		fake_sms_db[i].nr_of_sms = 0;
+		fake_sms_db[i].failed_attempts = 0;
+	}
+	show_fake_sms_db();
+	for (i = 0; i < 3; i++) {
+		struct gsm_sms *sms = smsq_take_next_sms(NULL, last_msisdn, sizeof(last_msisdn));
+		_test_take_next_sms_print(i, sms, last_msisdn);
+		OSMO_ASSERT(!sms);
+	}
+}
+
+
+static struct log_info_cat sms_queue_test_categories[] = {
+};
+
+static struct log_info info = {
+	.cat = sms_queue_test_categories,
+	.num_cat = ARRAY_SIZE(sms_queue_test_categories),
+};
+
+int main(int argc, char **argv)
+{
+	talloc_ctx = talloc_named_const(NULL, 1, "sms_queue_test");
+	msgb_talloc_ctx_init(talloc_ctx, 0);
+	osmo_init_logging(&info);
+
+	OSMO_ASSERT(osmo_stderr_target);
+	log_set_use_color(osmo_stderr_target, 0);
+	log_set_print_timestamp(osmo_stderr_target, 0);
+	log_set_print_filename(osmo_stderr_target, 0);
+	log_set_print_category(osmo_stderr_target, 1);
+	log_parse_category_mask(osmo_stderr_target, "DLOAP,1");
+
+	test_next_sms();
+	printf("Done\n");
+
+	return 0;
+}
diff --git a/openbsc/tests/sms_queue/sms_queue_test.err b/openbsc/tests/sms_queue/sms_queue_test.err
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/openbsc/tests/sms_queue/sms_queue_test.err
diff --git a/openbsc/tests/sms_queue/sms_queue_test.ok b/openbsc/tests/sms_queue/sms_queue_test.ok
new file mode 100644
index 000000000..146400d21
--- /dev/null
+++ b/openbsc/tests/sms_queue/sms_queue_test.ok
@@ -0,0 +1,98 @@
+Testing smsq_take_next_sms()
+
+- vsub 2, 3 and 5 each have 2 SMS pending, but 5 is not attached
+  1111 has 0 SMS pending, 0 failed attempts
+  2222 has 2 SMS pending, 2 failed attempts
+  3333 has 2 SMS pending, 3 failed attempts
+  4444 has 0 SMS pending, 0 failed attempts
+  5555 (NOT attached) has 2 SMS pending, 5 failed attempts
+-->
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#0: sending SMS to 2222 (last_msisdn='2222')
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+#1: sending SMS to 3333 (last_msisdn='3333')
+     hitting database: looking for MSISDN > '3333', failed_attempts <= 9
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#2: sending SMS to 2222 (last_msisdn='2222')
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+#3: sending SMS to 3333 (last_msisdn='3333')
+     hitting database: looking for MSISDN > '3333', failed_attempts <= 9
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#4: no SMS to send (last_msisdn='5555')
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#5: no SMS to send (last_msisdn='5555')
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#6: no SMS to send (last_msisdn='5555')
+
+- SMS are pending at various nr failed attempts (cutoff at >= 10)
+  1111 has 1 SMS pending, 0 failed attempts
+  2222 has 2 SMS pending, 5 failed attempts
+  3333 has 3 SMS pending, 10 failed attempts
+  4444 has 4 SMS pending, 15 failed attempts
+  5555 has 5 SMS pending, 20 failed attempts
+-->
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#0: sending SMS to 1111 (last_msisdn='1111')
+     hitting database: looking for MSISDN > '1111', failed_attempts <= 9
+#1: sending SMS to 2222 (last_msisdn='2222')
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#2: sending SMS to 2222 (last_msisdn='2222')
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#3: no SMS to send (last_msisdn='')
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#4: no SMS to send (last_msisdn='')
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#5: no SMS to send (last_msisdn='')
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#6: no SMS to send (last_msisdn='')
+
+- iterate the SMS DB at most once
+  1111 (NOT attached) has 1 SMS pending, 0 failed attempts
+  2222 (NOT attached) has 1 SMS pending, 0 failed attempts
+  3333 (NOT attached) has 1 SMS pending, 0 failed attempts
+  4444 (NOT attached) has 1 SMS pending, 0 failed attempts
+  5555 (NOT attached) has 1 SMS pending, 0 failed attempts
+-->
+     hitting database: looking for MSISDN > '2345', failed_attempts <= 9
+     hitting database: looking for MSISDN > '3333', failed_attempts <= 9
+     hitting database: looking for MSISDN > '4444', failed_attempts <= 9
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+     hitting database: looking for MSISDN > '1111', failed_attempts <= 9
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+#0: no SMS to send (last_msisdn='3333')
+     hitting database: looking for MSISDN > '3333', failed_attempts <= 9
+     hitting database: looking for MSISDN > '4444', failed_attempts <= 9
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+     hitting database: looking for MSISDN > '1111', failed_attempts <= 9
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+#1: no SMS to send (last_msisdn='3333')
+     hitting database: looking for MSISDN > '3333', failed_attempts <= 9
+     hitting database: looking for MSISDN > '4444', failed_attempts <= 9
+     hitting database: looking for MSISDN > '5555', failed_attempts <= 9
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+     hitting database: looking for MSISDN > '1111', failed_attempts <= 9
+     hitting database: looking for MSISDN > '2222', failed_attempts <= 9
+#2: no SMS to send (last_msisdn='3333')
+
+- there are no SMS in the DB
+  1111 has 0 SMS pending, 0 failed attempts
+  2222 has 0 SMS pending, 0 failed attempts
+  3333 has 0 SMS pending, 0 failed attempts
+  4444 has 0 SMS pending, 0 failed attempts
+  5555 has 0 SMS pending, 0 failed attempts
+-->
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#0: no SMS to send (last_msisdn='')
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#1: no SMS to send (last_msisdn='')
+     hitting database: looking for MSISDN > '', failed_attempts <= 9
+#2: no SMS to send (last_msisdn='')
+Done