osmux: add osmux circuit ID management and resolve NAT problems

This patch includes several osmux fixes that are interdependent:

1) This adds Osmux circuit ID, this is allocated from the bsc-nat. This
   announces the circuit ID in the CRCX MGCP message. This aims to resolve
   the lack of uniqueness due to the use of endp->ci, which is local to
   the bsc. This ID is notified via X-Osmux: NUM where NUM is the osmux
   circuit ID.

2) The dummy load routines are now used to setup osmux both in bsc and
   bsc-nat to resolve source port NAT issues as suggested by Holger. The
   source port that is used from the bsc is not known until the first
   voice message is sent to the bsc-nat, therefore enabling osmux from
   the MGCP plane breaks when a different source port is used.

3) Add refcnt to struct osmux_handle, several endpoints can be using the
   same input RTP osmux handle to perform the batching. Remove it from the
   osmux handle list once nobody is using it anymore to clean it up.

4) Add a simple Osmux state-machine with three states. The initial state
   is disabled, then if the bsc-nat requests Osmux, both sides enters
   activating. The final enabled state is reached once the bsc-nat sees
   the dummy load message that tells what source port is used by the bsc.

5) The osmux input handle (which transforms RTP messages to one Osmux batch)
   is now permanently attached to the endpoint when Osmux is set up from the
   dummy load path, so we skip a lookup for each message. This simplifies
   osmux_xfrm_to_osmux().

After this patch, the workflow to setup Osmux is the following:

                    bsc                   bsc-nat
                     |                       |
                     |<------ CRCX ----------|
                     |      X-Osmux: 3       | (where 3 is the Osmux circuit ID
                     |                       |  that the bsc-nat has allocated)
                     |------- resp --------->|
                     |      X-Osmux: 3       | (the bsc confirm that it can
                     |                       |  use Osmux).
                     .                       .
                     |                       |
         setup osmux |----- dummy load ----->| setup osmux
                     |      Osmux CID: 3     |

In two steps:

1st) Allocate the Osmux Circuit ID (CID): The bsc-nat allocates an unique
     Osmux CID that is notified to the bsc through the 'X-Osmux:' extension.
     The bsc-nat annotates this circuit ID in the endpoint object. The bsc
     replies back with the 'X-Osmux:' to confirm that it agrees to use Osmux.
     If the bsc doesn't want to use Osmux, it doesn't include the extension
     so the bsc-nat knows that it has to use to RTP.

2nd) The dummy load is used to convey the Osmux CID. This needs to happen
     at this stage since the bsc-nat needs to know what source port the bsc
     uses to get this working since the bsc may use a different source
     port due to NAT. Unfortunately, this can't be done from the MGCP signal
     plane since the real source port is not known that the bsc uses is not
     known.

This patch also reverts the MDCX handling until it is clear that we need
this special handling for this case.

1 files changed, 145 insertions, 77 deletions

diff --git a/openbsc/src/libmgcp/osmux.c b/openbsc/src/libmgcp/osmux.c
index 1370c9f21..0a58a2edb 100644
--- a/openbsc/src/libmgcp/osmux.c
+++ b/openbsc/src/libmgcp/osmux.c
@@ -27,7 +27,6 @@
 
 static struct osmo_fd osmux_fd;
 
-/* TODO: expire old handles.. */
 static LLIST_HEAD(osmux_handle_list);
 
 struct osmux_handle {
@@ -35,6 +34,7 @@ struct osmux_handle {
 	struct osmux_in_handle *in;
 	struct in_addr rem_addr;
 	int rem_port;
+	int refcnt;
 };
 
 static void *osmux;
@@ -70,6 +70,7 @@ osmux_handle_find_get(struct in_addr *addr, int rem_port)
 			LOGP(DMGCP, LOGL_DEBUG, "using existing OSMUX handle "
 						"for addr=%s:%d\n",
 				inet_ntoa(*addr), ntohs(rem_port));
+			h->refcnt++;
 			return h;
 		}
 	}
@@ -77,6 +78,27 @@ osmux_handle_find_get(struct in_addr *addr, int rem_port)
 	return NULL;
 }
 
+static void osmux_handle_put(struct osmux_in_handle *in)
+{
+	struct osmux_handle *h;
+
+	/* Lookup for existing OSMUX handle for this destination address. */
+	llist_for_each_entry(h, &osmux_handle_list, head) {
+		if (h->in == in) {
+			if (--h->refcnt == 0) {
+				LOGP(DMGCP, LOGL_DEBUG,
+				     "Releasing unused osmux handle for %s:%d\n",
+				     inet_ntoa(h->rem_addr),
+				     ntohs(h->rem_port));
+				llist_del(&h->head);
+				talloc_free(h);
+			}
+			return;
+		}
+	}
+	LOGP(DMGCP, LOGL_ERROR, "cannot find Osmux input handle %p\n", in);
+}
+
 static struct osmux_handle *
 osmux_handle_alloc(struct mgcp_config *cfg, struct in_addr *addr, int rem_port)
 {
@@ -87,6 +109,7 @@ osmux_handle_alloc(struct mgcp_config *cfg, struct in_addr *addr, int rem_port)
 		return NULL;
 	h->rem_addr = *addr;
 	h->rem_port = rem_port;
+	h->refcnt++;
 
 	h->in = talloc_zero(h, struct osmux_in_handle);
 	if (!h->in) {
@@ -126,10 +149,8 @@ osmux_handle_lookup(struct mgcp_config *cfg, struct in_addr *addr, int rem_port)
 
 int osmux_xfrm_to_osmux(int type, char *buf, int rc, struct mgcp_endpoint *endp)
 {
-	int ret, port;
+	int ret;
 	struct msgb *msg;
-	struct in_addr *addr;
-	struct osmux_in_handle *in;
 
 	msg = msgb_alloc(4096, "RTP");
 	if (!msg)
@@ -138,43 +159,12 @@ int osmux_xfrm_to_osmux(int type, char *buf, int rc, struct mgcp_endpoint *endp)
 	memcpy(msg->data, buf, rc);
 	msgb_put(msg, rc);
 
-	switch(type) {
-	case MGCP_DEST_NET:
-		addr = &endp->net_end.addr;
-		port = htons(OSMUX_PORT);
-		break;
-	case MGCP_DEST_BTS:
-		addr = &endp->bts_end.addr;
-		port = endp->bts_end.rtp_port;
-		break;
-	default:
-		/* Should not ever happen */
-		LOGP(DMGCP, LOGL_ERROR, "Bad type %d. Fix your code.\n", type);
-		msgb_free(msg);
-		return 0;
-	}
+	LOGP(DMGCP, LOGL_DEBUG, "Osmux uses CID %u from endpoint=%d (active=%d)\n",
+		endp->osmux.cid, ENDPOINT_NUMBER(endp), endp->allocated);
 
-	if (port == 0) {
-		LOGP(DMGCP, LOGL_ERROR, "0x%x remote end not known yet.\n",
-			ENDPOINT_NUMBER(endp));
-		msgb_free(msg);
-		return 0;
-	}
-
-	/* Lookup for osmux input handle that munches this RTP frame */
-	in = osmux_handle_lookup(endp->cfg, addr, port);
-	if (!in) {
-		LOGP(DMGCP, LOGL_ERROR, "No osmux handle, aborting\n");
-		msgb_free(msg);
-		return 0;
-	}
-
-	LOGP(DMGCP, LOGL_DEBUG, "Osmux uses cid=%u from endpoint=%d (active=%d)\n",
-		endp->ci, ENDPOINT_NUMBER(endp), endp->allocated);
-
-	while ((ret = osmux_xfrm_input(in, msg, endp->ci)) > 0) {
+	while ((ret = osmux_xfrm_input(endp->osmux.in, msg, endp->osmux.cid)) > 0) {
 		/* batch full, build and deliver it */
-		osmux_xfrm_input_deliver(in);
+		osmux_xfrm_input_deliver(endp->osmux.in);
 	}
 	return 0;
 }
@@ -208,7 +198,7 @@ endpoint_lookup(struct mgcp_config *cfg, int cid,
 			return NULL;
 		}
 
-		if ((tmp->ci & 0xFF) == cid && this->s_addr == from_addr->s_addr)
+		if (tmp->osmux.cid == cid && this->s_addr == from_addr->s_addr)
 			return tmp;
 	}
 
@@ -304,34 +294,51 @@ out:
 	return 0;
 }
 
-/*
- * Try to figure out where it came from and enter the rtp_port
- */
-static int osmux_handle_dummy(struct mgcp_config *cfg,
-			struct sockaddr_in *addr, struct msgb *msg)
+/* This is called from the bsc-nat */
+static int osmux_handle_dummy(struct mgcp_config *cfg, struct sockaddr_in *addr,
+			      struct msgb *msg)
 {
 	struct mgcp_endpoint *endp;
-	uint32_t ci;
+	uint8_t osmux_cid;
+
+	if (msg->len < 1 + sizeof(osmux_cid)) {
+		LOGP(DMGCP, LOGL_ERROR,
+		     "Discarding truncated Osmux dummy load\n");
+		goto out;
+	}
 
-	if (msg->len < 1 + sizeof(ci))
+	LOGP(DMGCP, LOGL_DEBUG, "Received Osmux dummy load from %s\n",
+	     inet_ntoa(addr->sin_addr));
+
+	if (!cfg->osmux) {
+		LOGP(DMGCP, LOGL_ERROR,
+		     "bsc wants to use Osmux but bsc-nat did not request it\n");
 		goto out;
+	}
 
-	/* extract the CI from the dummy message */
-	memcpy(&ci, &msg->data[1], sizeof(ci));
-	ci = ntohl(ci);
+	/* extract the osmux CID from the dummy message */
+	memcpy(&osmux_cid, &msg->data[1], sizeof(osmux_cid));
 
-	endp = endpoint_lookup(cfg, ci & 0xff, &addr->sin_addr, MGCP_DEST_BTS);
+	endp = endpoint_lookup(cfg, osmux_cid, &addr->sin_addr, MGCP_DEST_BTS);
 	if (!endp) {
-		LOGP(DMGCP, LOGL_ERROR, "Can not find CI=%d\n", ci & 0xff);
+		LOGP(DMGCP, LOGL_ERROR,
+		     "Cannot find endpoint for Osmux CID %d\n", osmux_cid);
 		goto out;
 	}
 
-	if (endp->bts_end.rtp_port == 0) {
-		endp->bts_end.rtp_port = addr->sin_port;
-		LOGP(DMGCP, LOGL_NOTICE, "0x%x found BTS on endpoint %s:%d\n",
-			ENDPOINT_NUMBER(endp),
-			inet_ntoa(addr->sin_addr), htons(addr->sin_port));
+	if (endp->osmux.state == OSMUX_STATE_ENABLED)
+		goto out;
+
+	if (osmux_enable_endpoint(endp, OSMUX_ROLE_BSC_NAT,
+				  &addr->sin_addr, addr->sin_port) < 0 ){
+		LOGP(DMGCP, LOGL_ERROR,
+		     "Could not update osmux in endpoint %d\n",
+		     ENDPOINT_NUMBER(endp));
 	}
+
+	LOGP(DMGCP, LOGL_INFO, "Enabling osmux in endpoint %d for %s:%u\n",
+	     ENDPOINT_NUMBER(endp), inet_ntoa(addr->sin_addr),
+	     ntohs(addr->sin_port));
 out:
 	msgb_free(msg);
 	return 0;
@@ -373,13 +380,6 @@ int osmux_read_from_bsc_cb(struct osmo_fd *ofd, unsigned int what)
 			goto out;
 		}
 
-		if (endp->bts_end.rtp_port == 0) {
-			endp->bts_end.rtp_port = addr.sin_port;
-			LOGP(DMGCP, LOGL_NOTICE, "0x%x found BTS on endpoint %s:%d\n",
-				ENDPOINT_NUMBER(endp),
-				inet_ntoa(addr.sin_addr), htons(addr.sin_port));
-		}
-
 		LOGP(DMGCP, LOGL_DEBUG,
 		     "sending extracted RTP from OSMUX to MSC via endpoint=%u "
 		     "(allocated=%d)\n", ENDPOINT_NUMBER(endp), endp->allocated);
@@ -426,23 +426,36 @@ int osmux_init(int role, struct mgcp_config *cfg)
 	return 0;
 }
 
-int osmux_enable_endpoint(struct mgcp_endpoint *endp, int role)
+int osmux_enable_endpoint(struct mgcp_endpoint *endp, int role,
+			  struct in_addr *addr, uint16_t port)
 {
 	/* If osmux is enabled, initialize the output handler. This handler is
 	 * used to reconstruct the RTP flow from osmux. The RTP SSRC is
-	 * allocated based on the circuit ID (endp->ci), which is unique in the
-	 * local scope to the BSC/BSC-NAT. We use it to divide the RTP SSRC
-	 * space (2^32) by the 256 possible circuit IDs, then randomly select
-	 * one value from that window. Thus, we have no chance to have
+	 * allocated based on the circuit ID (endp->osmux.cid), which is unique
+	 * in the local scope to the BSC/BSC-NAT. We use it to divide the RTP
+	 * SSRC space (2^32) by the 256 possible circuit IDs, then randomly
+	 * select one value from that window. Thus, we have no chance to have
 	 * overlapping RTP SSRC traveling to the BTSes behind the BSC,
 	 * similarly, for flows traveling to the MSC.
 	 */
 	static const uint32_t rtp_ssrc_winlen = UINT32_MAX / 256;
 
+	if (endp->osmux.state == OSMUX_STATE_DISABLED) {
+		LOGP(DMGCP, LOGL_ERROR, "Endpoint %u didn't request Osmux\n",
+		     ENDPOINT_NUMBER(endp));
+		return -1;
+	}
+
 	osmux_xfrm_output_init(&endp->osmux.out,
-			       (endp->ci * rtp_ssrc_winlen) +
+			       (endp->osmux.cid * rtp_ssrc_winlen) +
 			       (random() % rtp_ssrc_winlen));
 
+	endp->osmux.in = osmux_handle_lookup(endp->cfg, addr, port);
+	if (!endp->osmux.in) {
+		LOGP(DMGCP, LOGL_ERROR, "Cannot allocate input osmux handle\n");
+		return -1;
+	}
+
 	switch (endp->cfg->role) {
 		case MGCP_BSC_NAT:
 			endp->type = MGCP_OSMUX_BSC_NAT;
@@ -451,27 +464,82 @@ int osmux_enable_endpoint(struct mgcp_endpoint *endp, int role)
 			endp->type = MGCP_OSMUX_BSC;
 			break;
 	}
-	endp->osmux.enable = 1;
+	endp->osmux.state = OSMUX_STATE_ENABLED;
 
 	return 0;
 }
 
+void osmux_disable_endpoint(struct mgcp_endpoint *endp)
+{
+	LOGP(DMGCP, LOGL_INFO, "Releasing endpoint %u using Osmux CID %u\n",
+	     ENDPOINT_NUMBER(endp), endp->osmux.cid);
+	endp->osmux.state = OSMUX_STATE_DISABLED;
+	endp->osmux.cid = -1;
+	osmux_handle_put(endp->osmux.in);
+}
+
 /* We don't need to send the dummy load for osmux so often as another endpoint
  * may have already punched the hole in the firewall. This approach is simple
  * though.
  */
 int osmux_send_dummy(struct mgcp_endpoint *endp)
 {
-	uint32_t ci_be;
-	char buf[1 + sizeof(uint32_t)];
+	char buf[1 + sizeof(uint8_t)];
+	struct in_addr addr_unset = {};
 
-	ci_be = htonl(endp->ci);
 	buf[0] = MGCP_DUMMY_LOAD;
-	memcpy(&buf[1], &ci_be, sizeof(ci_be));
+	memcpy(&buf[1], &endp->osmux.cid, sizeof(endp->osmux.cid));
+
+	/* Wait until we have the connection information from MDCX */
+	if (memcmp(&endp->net_end.addr, &addr_unset, sizeof(addr_unset)) == 0)
+		return 0;
 
-	LOGP(DMGCP, LOGL_DEBUG, "sending OSMUX dummy load to %s\n",
-		inet_ntoa(endp->net_end.addr));
+	if (endp->osmux.state == OSMUX_STATE_ACTIVATING) {
+		if (osmux_enable_endpoint(endp, OSMUX_ROLE_BSC,
+					  &endp->net_end.addr,
+					  htons(OSMUX_PORT)) < 0) {
+			LOGP(DMGCP, LOGL_ERROR,
+			     "Could not activate osmux in endpoint %d\n",
+			     ENDPOINT_NUMBER(endp));
+		}
+		LOGP(DMGCP, LOGL_ERROR,
+		     "Osmux CID %u for %s:%u is now enabled\n",
+		     endp->osmux.cid, inet_ntoa(endp->net_end.addr),
+		     OSMUX_PORT);
+	}
+	LOGP(DMGCP, LOGL_DEBUG,
+	     "sending OSMUX dummy load to %s CID %u\n",
+	     inet_ntoa(endp->net_end.addr), endp->osmux.cid);
 
 	return mgcp_udp_send(osmux_fd.fd, &endp->net_end.addr,
 			     htons(OSMUX_PORT), buf, sizeof(buf));
 }
+
+/* bsc-nat allocates/releases the Osmux circuit ID */
+static uint8_t osmux_cid_bitmap[16];
+
+int osmux_get_cid(void)
+{
+	int i, j;
+
+	for (i = 0; i < sizeof(osmux_cid_bitmap) / 8; i++) {
+		for (j = 0; j < 8; j++) {
+			if (osmux_cid_bitmap[i] & (1 << j))
+				continue;
+
+			osmux_cid_bitmap[i] |= (1 << j);
+			LOGP(DMGCP, LOGL_DEBUG,
+			     "Allocating Osmux CID %u from pool\n", (i * 8) + j);
+			return (i * 8) + j;
+		}
+	}
+
+	LOGP(DMGCP, LOGL_ERROR, "All Osmux circuits are in use!\n");
+	return -1;
+}
+
+void osmux_put_cid(uint8_t osmux_cid)
+{
+	LOGP(DMGCP, LOGL_DEBUG, "Osmux CID %u is back to the pool\n", osmux_cid);
+	osmux_cid_bitmap[osmux_cid / 8] &= ~(1 << (osmux_cid % 8));
+}