Add some debugging code that ensures that when we do deadlock avoidance, we

don't lose the information about how a lock was originally acquired.


git-svn-id: http://svn.digium.com/svn/asterisk/branches/1.4@118953 f38db490-d61c-443f-a65b-d21fe96a405b

8 files changed, 86 insertions, 79 deletions

diff --git a/channels/chan_agent.c b/channels/chan_agent.c
index 4149f1c7f..51633bece 100644
--- a/channels/chan_agent.c
+++ b/channels/chan_agent.c
@@ -1588,9 +1588,7 @@ static int agent_logoff(const char *agent, int soft)
 					ast_mutex_lock(&p->lock);
 
 					while (p->owner && ast_channel_trylock(p->owner)) {
-						ast_mutex_unlock(&p->lock);
-						usleep(1);
-						ast_mutex_lock(&p->lock);
+						DEADLOCK_AVOIDANCE(&p->lock);
 					}
 					if (p->owner) {
 						ast_softhangup(p->owner, AST_SOFTHANGUP_EXPLICIT);
@@ -1598,9 +1596,7 @@ static int agent_logoff(const char *agent, int soft)
 					}
 
 					while (p->chan && ast_channel_trylock(p->chan)) {
-						ast_mutex_unlock(&p->lock);
-						usleep(1);
-						ast_mutex_lock(&p->lock);
+						DEADLOCK_AVOIDANCE(&p->lock);
 					}
 					if (p->chan) {
 						ast_softhangup(p->chan, AST_SOFTHANGUP_EXPLICIT);
diff --git a/channels/chan_alsa.c b/channels/chan_alsa.c
index d5817493f..c45740ef1 100644
--- a/channels/chan_alsa.c
+++ b/channels/chan_alsa.c
@@ -506,9 +506,7 @@ static int alsa_text(struct ast_channel *c, const char *text)
 static void grab_owner(void)
 {
 	while (alsa.owner && ast_mutex_trylock(&alsa.owner->lock)) {
-		ast_mutex_unlock(&alsalock);
-		usleep(1);
-		ast_mutex_lock(&alsalock);
+		DEADLOCK_AVOIDANCE(&alsalock);
 	}
 }
 
diff --git a/channels/chan_iax2.c b/channels/chan_iax2.c
index 439177771..a68f336e2 100644
--- a/channels/chan_iax2.c
+++ b/channels/chan_iax2.c
@@ -1278,9 +1278,7 @@ retry:
 	if (owner) {
 		if (ast_mutex_trylock(&owner->lock)) {
 			ast_log(LOG_NOTICE, "Avoiding IAX destroy deadlock\n");
-			ast_mutex_unlock(&iaxsl[callno]);
-			usleep(1);
-			ast_mutex_lock(&iaxsl[callno]);
+			DEADLOCK_AVOIDANCE(&iaxsl[callno]);
 			goto retry;
 		}
 	}
@@ -1652,9 +1650,7 @@ static int iax2_queue_frame(int callno, struct ast_frame *f)
 		if (iaxs[callno] && iaxs[callno]->owner) {
 			if (ast_mutex_trylock(&iaxs[callno]->owner->lock)) {
 				/* Avoid deadlock by pausing and trying again */
-				ast_mutex_unlock(&iaxsl[callno]);
-				usleep(1);
-				ast_mutex_lock(&iaxsl[callno]);
+				DEADLOCK_AVOIDANCE(&iaxsl[callno]);
 			} else {
 				ast_queue_frame(iaxs[callno]->owner, f);
 				ast_mutex_unlock(&iaxs[callno]->owner->lock);
@@ -1685,9 +1681,7 @@ static int iax2_queue_hangup(int callno)
 		if (iaxs[callno] && iaxs[callno]->owner) {
 			if (ast_mutex_trylock(&iaxs[callno]->owner->lock)) {
 				/* Avoid deadlock by pausing and trying again */
-				ast_mutex_unlock(&iaxsl[callno]);
-				usleep(1);
-				ast_mutex_lock(&iaxsl[callno]);
+				DEADLOCK_AVOIDANCE(&iaxsl[callno]);
 			} else {
 				ast_queue_hangup(iaxs[callno]->owner);
 				ast_mutex_unlock(&iaxs[callno]->owner->lock);
@@ -1719,9 +1713,7 @@ static int iax2_queue_control_data(int callno,
 		if (iaxs[callno] && iaxs[callno]->owner) {
 			if (ast_mutex_trylock(&iaxs[callno]->owner->lock)) {
 				/* Avoid deadlock by pausing and trying again */
-				ast_mutex_unlock(&iaxsl[callno]);
-				usleep(1);
-				ast_mutex_lock(&iaxsl[callno]);
+				DEADLOCK_AVOIDANCE(&iaxsl[callno]);
 			} else {
 				ast_queue_control_data(iaxs[callno]->owner, control, data, datalen);
 				ast_mutex_unlock(&iaxs[callno]->owner->lock);
@@ -3451,9 +3443,7 @@ static void lock_both(unsigned short callno0, unsigned short callno1)
 {
 	ast_mutex_lock(&iaxsl[callno0]);
 	while (ast_mutex_trylock(&iaxsl[callno1])) {
-		ast_mutex_unlock(&iaxsl[callno0]);
-		usleep(10);
-		ast_mutex_lock(&iaxsl[callno0]);
+		DEADLOCK_AVOIDANCE(&iaxsl[callno0]);
 	}
 }
 
@@ -3637,9 +3627,7 @@ static int iax2_indicate(struct ast_channel *c, int condition, const void *data,
 		/* We don't know the remote side's call number, yet.  :( */
 		int count = 10;
 		while (count-- && pvt && !pvt->peercallno) {
-			ast_mutex_unlock(&iaxsl[callno]);
-			usleep(1);
-			ast_mutex_lock(&iaxsl[callno]);
+			DEADLOCK_AVOIDANCE(&iaxsl[callno]);
 			pvt = iaxs[callno];
 		}
 		if (!pvt->peercallno) {
@@ -7384,9 +7372,7 @@ static int socket_process(struct iax2_thread *thread)
 						int orignative;
 retryowner:
 						if (ast_mutex_trylock(&iaxs[fr->callno]->owner->lock)) {
-							ast_mutex_unlock(&iaxsl[fr->callno]);
-							usleep(1);
-							ast_mutex_lock(&iaxsl[fr->callno]);
+							DEADLOCK_AVOIDANCE(&iaxsl[fr->callno]);
 							if (iaxs[fr->callno] && iaxs[fr->callno]->owner) goto retryowner;
 						}
 						if (iaxs[fr->callno]) {
@@ -7822,9 +7808,7 @@ retryowner:
 							ast_verbose(VERBOSE_PREFIX_3 "Format for call is %s\n", ast_getformatname(iaxs[fr->callno]->owner->nativeformats));
 retryowner2:
 						if (ast_mutex_trylock(&iaxs[fr->callno]->owner->lock)) {
-							ast_mutex_unlock(&iaxsl[fr->callno]);
-							usleep(1);
-							ast_mutex_lock(&iaxsl[fr->callno]);
+							DEADLOCK_AVOIDANCE(&iaxsl[fr->callno]);
 							if (iaxs[fr->callno] && iaxs[fr->callno]->owner) goto retryowner2;
 						}
 						
diff --git a/channels/chan_mgcp.c b/channels/chan_mgcp.c
index bd57e54f3..5bc1afd99 100644
--- a/channels/chan_mgcp.c
+++ b/channels/chan_mgcp.c
@@ -574,9 +574,7 @@ static void mgcp_queue_frame(struct mgcp_subchannel *sub, struct ast_frame *f)
 				ast_mutex_unlock(&sub->owner->lock);
 				break;
 			} else {
-				ast_mutex_unlock(&sub->lock);
-				usleep(1);
-				ast_mutex_lock(&sub->lock);
+				DEADLOCK_AVOIDANCE(&sub->lock);
 			}
 		} else
 			break;
@@ -592,9 +590,7 @@ static void mgcp_queue_hangup(struct mgcp_subchannel *sub)
 				ast_mutex_unlock(&sub->owner->lock);
 				break;
 			} else {
-				ast_mutex_unlock(&sub->lock);
-				usleep(1);
-				ast_mutex_lock(&sub->lock);
+				DEADLOCK_AVOIDANCE(&sub->lock);
 			}
 		} else
 			break;
diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index f270d1f12..b6b5c7930 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -1960,9 +1960,7 @@ static int retrans_pkt(const void *data)
 
 	if (ast_test_flag(pkt, FLAG_FATAL)) {
 		while(pkt->owner->owner && ast_channel_trylock(pkt->owner->owner)) {
-			ast_mutex_unlock(&pkt->owner->lock);	/* SIP_PVT, not channel */
-			usleep(1);
-			ast_mutex_lock(&pkt->owner->lock);
+			DEADLOCK_AVOIDANCE(&pkt->owner->lock);	/* SIP_PVT, not channel */
 		}
 
 		if (pkt->owner->owner && !pkt->owner->owner->hangupcause) 
@@ -2184,9 +2182,7 @@ static void __sip_ack(struct sip_pvt *p, int seqno, int resp, int sipmethod)
 			 * of the retransid to -1 is ensured since in both cases p's lock is held.
 			 */
 			while (cur->retransid > -1 && ast_sched_del(sched, cur->retransid)) {
-				ast_mutex_unlock(&p->lock);
-				usleep(1);
-				ast_mutex_lock(&p->lock);
+				DEADLOCK_AVOIDANCE(&p->lock);
 			}
 			free(cur);
 			break;
@@ -9015,9 +9011,7 @@ static struct sip_pvt *get_sip_pvt_byid_locked(const char *callid, const char *t
 
 			/* deadlock avoidance... */
 			while (sip_pvt_ptr->owner && ast_channel_trylock(sip_pvt_ptr->owner)) {
-				ast_mutex_unlock(&sip_pvt_ptr->lock);
-				usleep(1);
-				ast_mutex_lock(&sip_pvt_ptr->lock);
+				DEADLOCK_AVOIDANCE(&sip_pvt_ptr->lock);
 			}
 			break;
 		}
@@ -13248,9 +13242,7 @@ static int sip_park(struct ast_channel *chan1, struct ast_channel *chan2, struct
 	 * that hold the channel lock and want the pvt lock.  */
 	while (ast_channel_trylock(chan2)) {
 		struct sip_pvt *pvt = chan2->tech_pvt;
-		ast_mutex_unlock(&pvt->lock);
-		usleep(1);
-		ast_mutex_lock(&pvt->lock);
+		DEADLOCK_AVOIDANCE(&pvt->lock);
 	}
 	ast_channel_masquerade(transferer, chan2);
 	ast_channel_unlock(chan2);
@@ -15733,9 +15725,7 @@ restartsearch:
 						/* Needs a hangup */
 						if (ast_rtp_get_rtptimeout(sip->rtp)) {
 							while (sip->owner && ast_channel_trylock(sip->owner)) {
-								ast_mutex_unlock(&sip->lock);
-								usleep(1);
-								ast_mutex_lock(&sip->lock);
+								DEADLOCK_AVOIDANCE(&sip->lock);
 							}
 							if (sip->owner) {
 								ast_log(LOG_NOTICE,
diff --git a/channels/chan_zap.c b/channels/chan_zap.c
index c75670e23..96a2838ea 100644
--- a/channels/chan_zap.c
+++ b/channels/chan_zap.c
@@ -722,10 +722,7 @@ static inline int pri_grab(struct zt_pvt *pvt, struct zt_pri *pri)
 	do {
 		res = ast_mutex_trylock(&pri->lock);
 		if (res) {
-			ast_mutex_unlock(&pvt->lock);
-			/* Release the lock and try again */
-			usleep(1);
-			ast_mutex_lock(&pvt->lock);
+			DEADLOCK_AVOIDANCE(&pvt->lock);
 		}
 	} while (res);
 	/* Then break the poll */
@@ -792,9 +789,7 @@ static void wakeup_sub(struct zt_pvt *p, int a, void *pri)
 	for (;;) {
 		if (p->subs[a].owner) {
 			if (ast_mutex_trylock(&p->subs[a].owner->lock)) {
-				ast_mutex_unlock(&p->lock);
-				usleep(1);
-				ast_mutex_lock(&p->lock);
+				DEADLOCK_AVOIDANCE(&p->lock);
 			} else {
 				ast_queue_frame(p->subs[a].owner, &ast_null_frame);
 				ast_mutex_unlock(&p->subs[a].owner->lock);
@@ -823,9 +818,7 @@ static void zap_queue_frame(struct zt_pvt *p, struct ast_frame *f, void *pri)
 	for (;;) {
 		if (p->owner) {
 			if (ast_mutex_trylock(&p->owner->lock)) {
-				ast_mutex_unlock(&p->lock);
-				usleep(1);
-				ast_mutex_lock(&p->lock);
+				DEADLOCK_AVOIDANCE(&p->lock);
 			} else {
 				ast_queue_frame(p->owner, f);
 				ast_mutex_unlock(&p->owner->lock);
@@ -3038,9 +3031,7 @@ static void zt_unlink(struct zt_pvt *slave, struct zt_pvt *master, int needlock)
 		ast_mutex_lock(&master->lock);
 		if (slave) {
 			while (ast_mutex_trylock(&slave->lock)) {
-				ast_mutex_unlock(&master->lock);
-				usleep(1);
-				ast_mutex_lock(&master->lock);
+				DEADLOCK_AVOIDANCE(&master->lock);
 			}
 		}
 	}
@@ -3177,9 +3168,7 @@ static enum ast_bridge_result zt_bridge(struct ast_channel *c0, struct ast_chann
 
 	ast_mutex_lock(&c0->lock);
 	while (ast_mutex_trylock(&c1->lock)) {
-		ast_mutex_unlock(&c0->lock);
-		usleep(1);
-		ast_mutex_lock(&c0->lock);
+		DEADLOCK_AVOIDANCE(&c0->lock);
 	}
 
 	p0 = c0->tech_pvt;
@@ -3349,9 +3338,7 @@ static enum ast_bridge_result zt_bridge(struct ast_channel *c0, struct ast_chann
 		   and then balking if anything is wrong */
 		ast_mutex_lock(&c0->lock);
 		while (ast_mutex_trylock(&c1->lock)) {
-			ast_mutex_unlock(&c0->lock);
-			usleep(1);
-			ast_mutex_lock(&c0->lock);
+			DEADLOCK_AVOIDANCE(&c0->lock);
 		}
 
 		p0 = c0->tech_pvt;
@@ -3910,12 +3897,10 @@ static struct ast_frame *zt_handle_event(struct ast_channel *ast)
 						while (p->subs[SUB_THREEWAY].owner && ast_mutex_trylock(&p->subs[SUB_THREEWAY].owner->lock)) {
 							/* Yuck, didn't get the lock on the 3-way, gotta release everything and re-grab! */
 							ast_mutex_unlock(&p->lock);
-							ast_mutex_unlock(&ast->lock);
-							usleep(1);
+							DEADLOCK_AVOIDANCE(&ast->lock);
 							/* We can grab ast and p in that order, without worry.  We should make sure
 							   nothing seriously bad has happened though like some sort of bizarre double
 							   masquerade! */
-							ast_mutex_lock(&ast->lock);
 							ast_mutex_lock(&p->lock);
 							if (p->owner != ast) {
 								ast_log(LOG_WARNING, "This isn't good...\n");
@@ -8301,9 +8286,7 @@ static int pri_hangup_all(struct zt_pvt *p, struct zt_pri *pri)
 		for (x = 0; x < 3; x++) {
 			while (p->subs[x].owner && ast_mutex_trylock(&p->subs[x].owner->lock)) {
 				redo++;
-				ast_mutex_unlock(&p->lock);
-				usleep(1);
-				ast_mutex_lock(&p->lock);
+				DEADLOCK_AVOIDANCE(&p->lock);
 			}
 			if (p->subs[x].owner) {
 				ast_queue_hangup(p->subs[x].owner);
diff --git a/include/asterisk/lock.h b/include/asterisk/lock.h
index 2570ee7bb..b3141ffda 100644
--- a/include/asterisk/lock.h
+++ b/include/asterisk/lock.h
@@ -179,6 +179,38 @@ void ast_remove_lock_info(void *lock_addr);
 #define ast_remove_lock_info(ignore)
 #endif
 
+/*!
+ * \brief retrieve lock info for the specified mutex
+ *
+ * this gets called during deadlock avoidance, so that the information may
+ * be preserved as to what location originally acquired the lock.
+ */
+#if !defined(LOW_MEMORY)
+int ast_find_lock_info(void *lock_addr, const char **filename, int *lineno, const char **func, const char **mutex_name);
+#else
+#define ast_find_lock_info(a,b,c,d,e) -1
+#endif
+
+/*!
+ * \brief Unlock a lock briefly
+ *
+ * used during deadlock avoidance, to preserve the original location where
+ * a lock was originally acquired.
+ */
+#define DEADLOCK_AVOIDANCE(lock) \
+	do { \
+		const char *__filename, *__func, *__mutex_name; \
+		int __lineno; \
+		int __res = ast_find_lock_info(lock, &__filename, &__lineno, &__func, &__mutex_name); \
+		ast_mutex_unlock(lock); \
+		usleep(1); \
+		if (__res < 0) { /* Shouldn't ever happen, but just in case... */ \
+			ast_mutex_lock(lock); \
+		} else { \
+			__ast_pthread_mutex_lock(__filename, __lineno, __func, __mutex_name, lock); \
+		} \
+	} while (0)
+
 static void __attribute__((constructor)) init_empty_mutex(void)
 {
 	memset(&empty_mutex, 0, sizeof(empty_mutex));
diff --git a/main/utils.c b/main/utils.c
index 5190f5f64..7b6e64812 100644
--- a/main/utils.c
+++ b/main/utils.c
@@ -662,6 +662,34 @@ void ast_mark_lock_failed(void *lock_addr)
 	pthread_mutex_unlock(&lock_info->lock);
 }
 
+int ast_find_lock_info(void *lock_addr, const char **filename, int *lineno, const char **func, const char **mutex_name)
+{
+	struct thr_lock_info *lock_info;
+	int i = 0;
+
+	if (!(lock_info = ast_threadstorage_get(&thread_lock_info, sizeof(*lock_info))))
+		return -1;
+
+	pthread_mutex_lock(&lock_info->lock);
+
+	for (i = lock_info->num_locks - 1; i >= 0; i--) {
+		if (lock_info->locks[i].lock_addr == lock_addr)
+			break;
+	}
+
+	if (i == -1) {
+		/* Lock not found :( */
+		pthread_mutex_unlock(&lock_info->lock);
+		return -1;
+	}
+
+	*filename = lock_info->locks[i].file;
+	*lineno = lock_info->locks[i].line_num;
+	*func = lock_info->locks[i].func;
+	*mutex_name = lock_info->locks[i].lock_name;
+	return 0;
+}
+
 void ast_remove_lock_info(void *lock_addr)
 {
 	struct thr_lock_info *lock_info;