New SMS v4 database, proper handling of the sender address on the SMPP interface

5 files changed, 212 insertions, 77 deletions

diff --git a/openbsc/include/openbsc/gsm_data.h b/openbsc/include/openbsc/gsm_data.h
index 1b4720fe0..e6a94ad78 100644
--- a/openbsc/include/openbsc/gsm_data.h
+++ b/openbsc/include/openbsc/gsm_data.h
@@ -304,7 +304,6 @@ struct gsm_sms_addr {
 
 struct gsm_sms {
 	unsigned long long id;
-	struct gsm_subscriber *sender;
 	struct gsm_subscriber *receiver;
 	struct gsm_sms_addr src, dst;
 	enum gsm_sms_source_id source;
diff --git a/openbsc/src/libbsc/gsm_subscriber_base.c b/openbsc/src/libbsc/gsm_subscriber_base.c
index 5755687a1..7a640a73a 100644
--- a/openbsc/src/libbsc/gsm_subscriber_base.c
+++ b/openbsc/src/libbsc/gsm_subscriber_base.c
@@ -85,7 +85,7 @@ struct gsm_subscriber *subscr_put(struct gsm_subscriber *subscr)
 	subscr->use_count--;
 	DEBUGP(DREF, "subscr %s usage decreased usage to: %d\n",
 			subscr->extension, subscr->use_count);
-	if (subscr->use_count <= 0 && !subscr->net->keep_subscr)
+	if (subscr->use_count <= 0 && (!subscr->net || !subscr->net->keep_subscr))
 		subscr_free(subscr);
 	return NULL;
 }
diff --git a/openbsc/src/libmsc/db.c b/openbsc/src/libmsc/db.c
index bbd4e90b8..edda46ef1 100644
--- a/openbsc/src/libmsc/db.c
+++ b/openbsc/src/libmsc/db.c
@@ -41,8 +41,35 @@
 static char *db_basename = NULL;
 static char *db_dirname = NULL;
 static dbi_conn conn;
-
-#define SCHEMA_REVISION "3"
+static struct gsm_sms *sms_from_result(struct gsm_network *net, dbi_result result);
+
+#define SCHEMA_REVISION "4"
+
+#define SMS_TABLE_CREATE_STMT \
+	"CREATE TABLE IF NOT EXISTS SMS (" \
+		/* metadata, not part of sms */ \
+		"id INTEGER PRIMARY KEY AUTOINCREMENT, " \
+		"created TIMESTAMP NOT NULL, " \
+		"sent TIMESTAMP, " \
+		"deliver_attempts INTEGER NOT NULL DEFAULT 0, " \
+		/* data directly copied/derived from SMS */ \
+		"valid_until TIMESTAMP, " \
+		"reply_path_req INTEGER NOT NULL, " \
+		"status_rep_req INTEGER NOT NULL, " \
+		"protocol_id INTEGER NOT NULL, " \
+		"data_coding_scheme INTEGER NOT NULL, " \
+		"ud_hdr_ind INTEGER NOT NULL, " \
+		"src_addr TEXT NOT NULL, " \
+		"src_ton INTEGER NOT NULL, " \
+		"src_npi INTEGER NOT NULL, " \
+		"dest_addr TEXT NOT NULL, " \
+		"dest_ton INTEGER NOT NULL, " \
+		"dest_npi INTEGER NOT NULL, " \
+		"user_data BLOB, "	/* TP-UD */ \
+		/* additional data, interpreted from SMS */ \
+		"header BLOB, "		/* UD Header */ \
+		"text TEXT "		/* decoded UD after UDH */ \
+		")"
 
 static char *create_stmts[] = {
 	"CREATE TABLE IF NOT EXISTS Meta ("
@@ -90,27 +117,7 @@ static char *create_stmts[] = {
 		"equipment_id NUMERIC NOT NULL, "
 		"UNIQUE (subscriber_id, equipment_id) "
 		")",
-	"CREATE TABLE IF NOT EXISTS SMS ("
-		/* metadata, not part of sms */
-		"id INTEGER PRIMARY KEY AUTOINCREMENT, "
-		"created TIMESTAMP NOT NULL, "
-		"sent TIMESTAMP, "
-		"sender_id INTEGER NOT NULL, "
-		"receiver_id INTEGER NOT NULL, "
-		"deliver_attempts INTEGER NOT NULL DEFAULT 0, "
-		/* data directly copied/derived from SMS */
-		"valid_until TIMESTAMP, "
-		"reply_path_req INTEGER NOT NULL, "
-		"status_rep_req INTEGER NOT NULL, "
-		"protocol_id INTEGER NOT NULL, "
-		"data_coding_scheme INTEGER NOT NULL, "
-		"ud_hdr_ind INTEGER NOT NULL, "
-		"dest_addr TEXT, "
-		"user_data BLOB, "	/* TP-UD */
-		/* additional data, interpreted from SMS */
-		"header BLOB, "		/* UD Header */
-		"text TEXT "		/* decoded UD after UDH */
-		")",
+	SMS_TABLE_CREATE_STMT,
 	"CREATE TABLE IF NOT EXISTS VLR ("
 		"id INTEGER PRIMARY KEY AUTOINCREMENT, "
 		"created TIMESTAMP NOT NULL, "
@@ -171,7 +178,7 @@ static int update_db_revision_2(void)
 				"TIMESTAMP DEFAULT NULL");
 	if (!result) {
 		LOGP(DDB, LOGL_ERROR,
-		     "Failed to alter table Subscriber (upgrade vom rev 2).\n");
+		     "Failed to alter table Subscriber (upgrade from rev 2).\n");
 		return -EINVAL;
 	}
 	dbi_result_free(result);
@@ -182,7 +189,138 @@ static int update_db_revision_2(void)
 				"WHERE key = 'revision'");
 	if (!result) {
 		LOGP(DDB, LOGL_ERROR,
-		     "Failed set new revision (upgrade vom rev 2).\n");
+		     "Failed to update DB schema revision  (upgrade from rev 2).\n");
+		return -EINVAL;
+	}
+	dbi_result_free(result);
+
+	return 0;
+}
+
+
+static struct gsm_sms *sms_from_result_v3(dbi_result result)
+{
+	struct gsm_sms *sms = sms_alloc();
+	long long unsigned int sender_id;
+	struct gsm_subscriber *sender;
+	const char *text, *daddr;
+	const unsigned char *user_data;
+	char buf[32];
+
+	if (!sms)
+		return NULL;
+
+	sms->id = dbi_result_get_ulonglong(result, "id");
+
+	sender_id = dbi_result_get_ulonglong(result, "sender_id");
+	sprintf(buf, "%llu", sender_id);
+	sender = db_get_subscriber(NULL, GSM_SUBSCRIBER_ID, buf);
+	strncpy(sms->src.addr, sender->extension, sizeof(sms->src.addr)-1);
+	subscr_put(sender);
+
+	sms->receiver = NULL;
+
+	/* FIXME: validity */
+	/* FIXME: those should all be get_uchar, but sqlite3 is braindead */
+	sms->reply_path_req = dbi_result_get_uint(result, "reply_path_req");
+	sms->status_rep_req = dbi_result_get_uint(result, "status_rep_req");
+	sms->ud_hdr_ind = dbi_result_get_uint(result, "ud_hdr_ind");
+	sms->protocol_id = dbi_result_get_uint(result, "protocol_id");
+	sms->data_coding_scheme = dbi_result_get_uint(result,
+						  "data_coding_scheme");
+	/* sms->msg_ref is temporary and not stored in DB */
+
+	daddr = dbi_result_get_string(result, "dest_addr");
+	if (daddr) {
+		strncpy(sms->dst.addr, daddr, sizeof(sms->dst.addr));
+		sms->dst.addr[sizeof(sms->dst.addr)-1] = '\0';
+	}
+
+	sms->user_data_len = dbi_result_get_field_length(result, "user_data");
+	user_data = dbi_result_get_binary(result, "user_data");
+	if (sms->user_data_len > sizeof(sms->user_data))
+		sms->user_data_len = (uint8_t) sizeof(sms->user_data);
+	memcpy(sms->user_data, user_data, sms->user_data_len);
+
+	text = dbi_result_get_string(result, "text");
+	if (text) {
+		strncpy(sms->text, text, sizeof(sms->text));
+		sms->text[sizeof(sms->text)-1] = '\0';
+	}
+	return sms;
+}
+
+static int update_db_revision_3(void)
+{
+	dbi_result result;
+	struct gsm_sms *sms;
+
+	/* Rename old SMS table to be able create a new one */
+	result = dbi_conn_query(conn,
+				"ALTER TABLE SMS "
+				"RENAME TO SMS_3");
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed to rename the old SMS table (upgrade from rev 3).\n");
+		return -EINVAL;
+	}
+	dbi_result_free(result);
+
+	/* Create new SMS table with all the bells and whistles! */
+	result = dbi_conn_query(conn,
+				SMS_TABLE_CREATE_STMT);
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed to create a new SMS table (upgrade from rev 3).\n");
+		return -EINVAL;
+	}
+	dbi_result_free(result);
+
+	/* Cycle through old messages and convert them to the new format */
+	result = dbi_conn_queryf(conn,
+				"SELECT * FROM SMS_3");
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed fetch messages from the old SMS table (upgrade from rev 3).\n");
+		return -EINVAL;
+	}
+	while (dbi_result_next_row(result)) {
+		sms = sms_from_result_v3(result);
+		if (db_sms_store(sms) != 0) {
+			LOGP(DDB, LOGL_ERROR, "Failed to store message to the new SMS table(upgrade from rev 3).\n");
+		}
+		sms_free(sms);
+	}
+	dbi_result_free(result);
+
+	/* Mark SMS_3 table for removal */
+	result = dbi_conn_query(conn,
+				"DROP TABLE SMS_3");
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed to drop the old SMS table (upgrade from rev 3).\n");
+		return -EINVAL;
+	}
+	dbi_result_free(result);
+
+	/* Shrink DB file size by actually wiping out SMS_3 table data */
+	result = dbi_conn_query(conn,
+				"VACUUM");
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed VACUUM after droping the old SMS table (upgrade from rev 3).\n");
+		return -EINVAL;
+	}
+	dbi_result_free(result);
+
+	/* We're done. Bump DB Meta revision to 4 */
+	result = dbi_conn_query(conn,
+				"UPDATE Meta "
+				"SET value = '4' "
+				"WHERE key = 'revision'");
+	if (!result) {
+		LOGP(DDB, LOGL_ERROR,
+		     "Failed to update DB schema revision (upgrade from rev 3).\n");
 		return -EINVAL;
 	}
 	dbi_result_free(result);
@@ -215,6 +353,12 @@ static int check_db_revision(void)
 			dbi_result_free(result);
 			return -EINVAL;
 		}
+	} else if (!strcmp(rev_s, "3")) {
+			if (update_db_revision_3()) {
+				LOGP(DDB, LOGL_FATAL, "Failed to update database from schema revision '%s'.\n", rev_s);
+				dbi_result_free(result);
+				return -EINVAL;
+			}
 	} else if (!strcmp(rev_s, SCHEMA_REVISION)) {
 		/* everything is fine */
 	} else {
@@ -1097,7 +1241,7 @@ int db_subscriber_assoc_imei(struct gsm_subscriber *subscriber, char imei[GSM_IM
 int db_sms_store(struct gsm_sms *sms)
 {
 	dbi_result result;
-	char *q_text, *q_daddr;
+	char *q_text, *q_daddr, *q_saddr;
 	unsigned char *q_udata;
 	char *validity_timestamp = "2222-2-2";
 
@@ -1105,25 +1249,35 @@ int db_sms_store(struct gsm_sms *sms)
 
 	dbi_conn_quote_string_copy(conn, (char *)sms->text, &q_text);
 	dbi_conn_quote_string_copy(conn, (char *)sms->dst.addr, &q_daddr);
+	dbi_conn_quote_string_copy(conn, (char *)sms->src.addr, &q_saddr);
 	dbi_conn_quote_binary_copy(conn, sms->user_data, sms->user_data_len,
 				   &q_udata);
+
 	/* FIXME: correct validity period */
 	result = dbi_conn_queryf(conn,
 		"INSERT INTO SMS "
-		"(created, sender_id, receiver_id, valid_until, "
+		"(created, valid_until, "
 		 "reply_path_req, status_rep_req, protocol_id, "
-		 "data_coding_scheme, ud_hdr_ind, dest_addr, "
-		 "user_data, text) VALUES "
-		"(datetime('now'), %llu, %llu, %u, "
-		 "%u, %u, %u, %u, %u, %s, %s, %s)",
-		sms->sender->id,
-		sms->receiver ? sms->receiver->id : 0, validity_timestamp,
+		 "data_coding_scheme, ud_hdr_ind, "
+		 "user_data, text, "
+		 "dest_addr, dest_ton, dest_npi, "
+		 "src_addr, src_ton, src_npi) VALUES "
+		"(datetime('now'), %u, "
+		"%u, %u, %u, "
+		"%u, %u, "
+		"%s, %s, "
+		"%s, %u, %u, "
+		"%s, %u, %u)",
+		validity_timestamp,
 		sms->reply_path_req, sms->status_rep_req, sms->protocol_id,
 		sms->data_coding_scheme, sms->ud_hdr_ind,
-		q_daddr, q_udata, q_text);
+		q_udata, q_text,
+		q_daddr, sms->dst.ton, sms->dst.npi,
+		q_saddr, sms->src.ton, sms->src.npi);
 	free(q_text);
-	free(q_daddr);
 	free(q_udata);
+	free(q_daddr);
+	free(q_saddr);
 
 	if (!result)
 		return -EIO;
@@ -1135,8 +1289,7 @@ int db_sms_store(struct gsm_sms *sms)
 static struct gsm_sms *sms_from_result(struct gsm_network *net, dbi_result result)
 {
 	struct gsm_sms *sms = sms_alloc();
-	long long unsigned int sender_id, receiver_id;
-	const char *text, *daddr;
+	const char *text, *daddr, *saddr;
 	const unsigned char *user_data;
 
 	if (!sms)
@@ -1144,28 +1297,6 @@ static struct gsm_sms *sms_from_result(struct gsm_network *net, dbi_result resul
 
 	sms->id = dbi_result_get_ulonglong(result, "id");
 
-	sender_id = dbi_result_get_ulonglong(result, "sender_id");
-	sms->sender = subscr_get_by_id(net, sender_id);
-	if (!sms->sender) {
-		LOGP(DLSMS, LOGL_ERROR,
-			"Failed to find sender(%llu) for id(%llu)\n",
-			sender_id, sms->id);
-		sms_free(sms);
-		return NULL;
-	}
-
-	strncpy(sms->src.addr, sms->sender->extension, sizeof(sms->src.addr)-1);
-
-	receiver_id = dbi_result_get_ulonglong(result, "receiver_id");
-	sms->receiver = subscr_get_by_id(net, receiver_id);
-	if (!sms->receiver) {
-		LOGP(DLSMS, LOGL_ERROR,
-			"Failed to find receiver(%llu) for id(%llu)\n",
-			receiver_id, sms->id);
-		sms_free(sms);
-		return NULL;
-	}
-
 	/* FIXME: validity */
 	/* FIXME: those should all be get_uchar, but sqlite3 is braindead */
 	sms->reply_path_req = dbi_result_get_uint(result, "reply_path_req");
@@ -1176,11 +1307,22 @@ static struct gsm_sms *sms_from_result(struct gsm_network *net, dbi_result resul
 						  "data_coding_scheme");
 	/* sms->msg_ref is temporary and not stored in DB */
 
+	sms->dst.npi = dbi_result_get_uint(result, "dest_npi");
+	sms->dst.ton = dbi_result_get_uint(result, "dest_ton");
 	daddr = dbi_result_get_string(result, "dest_addr");
 	if (daddr) {
 		strncpy(sms->dst.addr, daddr, sizeof(sms->dst.addr));
 		sms->dst.addr[sizeof(sms->dst.addr)-1] = '\0';
 	}
+	sms->receiver = subscr_get_by_extension(net, sms->dst.addr);
+
+	sms->src.npi = dbi_result_get_uint(result, "src_npi");
+	sms->src.ton = dbi_result_get_uint(result, "src_ton");
+	saddr = dbi_result_get_string(result, "src_addr");
+	if (saddr) {
+		strncpy(sms->src.addr, saddr, sizeof(sms->src.addr));
+		sms->src.addr[sizeof(sms->src.addr)-1] = '\0';
+	}
 
 	sms->user_data_len = dbi_result_get_field_length(result, "user_data");
 	user_data = dbi_result_get_binary(result, "user_data");
@@ -1227,7 +1369,7 @@ struct gsm_sms *db_sms_get_unsent(struct gsm_network *net, unsigned long long mi
 	result = dbi_conn_queryf(conn,
 		"SELECT SMS.* "
 			"FROM SMS JOIN Subscriber ON "
-				"SMS.receiver_id = Subscriber.id "
+				"SMS.dest_addr = Subscriber.extension "
 			"WHERE SMS.id >= %llu AND SMS.sent IS NULL "
 				"AND Subscriber.lac > 0 "
 			"ORDER BY SMS.id LIMIT 1",
@@ -1257,10 +1399,10 @@ struct gsm_sms *db_sms_get_unsent_by_subscr(struct gsm_network *net,
 	result = dbi_conn_queryf(conn,
 		"SELECT SMS.* "
 			"FROM SMS JOIN Subscriber ON "
-				"SMS.receiver_id = Subscriber.id "
-			"WHERE SMS.receiver_id >= %llu AND SMS.sent IS NULL "
+				"SMS.dest_addr = Subscriber.extension "
+			"WHERE Subscriber.id >= %llu AND SMS.sent IS NULL "
 				"AND Subscriber.lac > 0 AND SMS.deliver_attempts < %u "
-			"ORDER BY SMS.receiver_id, SMS.id LIMIT 1",
+			"ORDER BY Subscriber.id, SMS.id LIMIT 1",
 		min_subscr_id, failed);
 	if (!result)
 		return NULL;
@@ -1286,8 +1428,8 @@ struct gsm_sms *db_sms_get_unsent_for_subscr(struct gsm_subscriber *subscr)
 	result = dbi_conn_queryf(conn,
 		"SELECT SMS.* "
 			"FROM SMS JOIN Subscriber ON "
-				"SMS.receiver_id = Subscriber.id "
-			"WHERE SMS.receiver_id = %llu AND SMS.sent IS NULL "
+				"SMS.dest_addr = Subscriber.extension "
+			"WHERE Subscriber.id = %llu AND SMS.sent IS NULL "
 				"AND Subscriber.lac > 0 "
 			"ORDER BY SMS.id LIMIT 1",
 		subscr->id);
diff --git a/openbsc/src/libmsc/gsm_04_11.c b/openbsc/src/libmsc/gsm_04_11.c
index 71a79bd10..eea86f34e 100644
--- a/openbsc/src/libmsc/gsm_04_11.c
+++ b/openbsc/src/libmsc/gsm_04_11.c
@@ -73,8 +73,6 @@ struct gsm_sms *sms_alloc(void)
 void sms_free(struct gsm_sms *sms)
 {
 	/* drop references to subscriber structure */
-	if (sms->sender)
-		subscr_put(sms->sender);
 	if (sms->receiver)
 		subscr_put(sms->receiver);
 #ifdef BUILD_SMPP
@@ -97,8 +95,7 @@ struct gsm_sms *sms_from_text(struct gsm_subscriber *receiver,
 	sms->receiver = subscr_get(receiver);
 	strncpy(sms->text, text, sizeof(sms->text)-1);
 
-	sms->sender = subscr_get(sender);
-	strncpy(sms->src.addr, sms->sender->extension, sizeof(sms->src.addr)-1);
+	strncpy(sms->src.addr, sender->extension, sizeof(sms->src.addr)-1);
 	sms->reply_path_req = 0;
 	sms->status_rep_req = 0;
 	sms->ud_hdr_ind = 0;
@@ -378,12 +375,10 @@ static int gsm340_rx_tpdu(struct gsm_subscriber_connection *conn, struct msgb *m
 		}
 	}
 
-	gsms->sender = subscr_get(conn->subscr);
-
 	LOGP(DLSMS, LOGL_INFO, "RX SMS: Sender: %s, MTI: 0x%02x, VPF: 0x%02x, "
 	     "MR: 0x%02x PID: 0x%02x, DCS: 0x%02x, DA: %s, "
 	     "UserDataLength: 0x%02x, UserData: \"%s\"\n",
-	     subscr_name(gsms->sender), sms_mti, sms_vpf, gsms->msg_ref,
+	     subscr_name(conn->subscr), sms_mti, sms_vpf, gsms->msg_ref,
 	     gsms->protocol_id, gsms->data_coding_scheme, gsms->dst.addr,
 	     gsms->user_data_len,
 			sms_alphabet == DCS_7BIT_DEFAULT ? gsms->text :
diff --git a/openbsc/src/libmsc/smpp_openbsc.c b/openbsc/src/libmsc/smpp_openbsc.c
index ab558ce9d..ec541c271 100644
--- a/openbsc/src/libmsc/smpp_openbsc.c
+++ b/openbsc/src/libmsc/smpp_openbsc.c
@@ -133,7 +133,6 @@ static int submit_to_sms(struct gsm_sms **psms, struct gsm_network *net,
 	strncpy(sms->dst.addr, dest->extension, sizeof(sms->dst.addr)-1);
 
 	/* fill in the source address */
-	sms->sender = subscr_get_by_id(net, 1);
 	sms->src.ton = submit->source_addr_ton;
 	sms->src.npi = submit->source_addr_npi;
 	strncpy(sms->src.addr, (char *)submit->source_addr, sizeof(sms->src.addr)-1);
@@ -475,13 +474,13 @@ static int deliver_to_esme(struct osmo_esme *esme, struct gsm_sms *sms,
 		deliver.source_addr_npi = NPI_Land_Mobile_E212;
 		snprintf((char *)deliver.source_addr,
 			sizeof(deliver.source_addr), "%s",
-			sms->sender->imsi);
+			conn->subscr->imsi);
 	} else {
 		deliver.source_addr_ton = TON_Network_Specific;
 		deliver.source_addr_npi = NPI_ISDN_E163_E164;
 		snprintf((char *)deliver.source_addr,
 			 sizeof(deliver.source_addr), "%s",
-			 sms->sender->extension);
+			 conn->subscr->extension);
 	}
 
 	deliver.dest_addr_ton	= sms->dst.ton;