A new feature thanks to the fine folks at Switchvox!

If a deadlock is detected, then the typical lock information will be
printed along with a backtrace of the stack for the offending threads.
Use of this requires compiling with DETECT_DEADLOCKS and having glibc
installed.

Furthermore, issuing the "core show locks" CLI command will print the
normal lock information as well as a backtraces for each lock. This
requires that DEBUG_THREADS is enabled and that glibc is installed.

All the backtrace features may be disabled by running the configure
script with --without-execinfo as an argument



git-svn-id: http://svn.digium.com/svn/asterisk/trunk@118173 f38db490-d61c-443f-a65b-d21fe96a405b

1 files changed, 368 insertions, 164 deletions

diff --git a/include/asterisk/lock.h b/include/asterisk/lock.h
index cbc2926b3..d78a83437 100644
--- a/include/asterisk/lock.h
+++ b/include/asterisk/lock.h
@@ -51,6 +51,8 @@
 #include <pthread.h>
 #include <sys/param.h>
 
+#include <execinfo.h>
+
 #include "asterisk/logger.h"
 
 /* internal macro to profile mutexes. Only computes the delay on
@@ -108,9 +110,15 @@
 
 #include <errno.h>
 
+#ifdef HAVE_BKTR
+#define AST_MUTEX_INIT_VALUE { PTHREAD_MUTEX_INIT_VALUE, 1, { NULL }, { 0 }, 0, { NULL }, { 0 }, {{{ 0 }}}, PTHREAD_MUTEX_INIT_VALUE }
+#define AST_MUTEX_INIT_VALUE_NOTRACKING \
+                             { PTHREAD_MUTEX_INIT_VALUE, 0, { NULL }, { 0 }, 0, { NULL }, { 0 }, {{{ 0 }}}, PTHREAD_MUTEX_INIT_VALUE }
+#else
 #define AST_MUTEX_INIT_VALUE { PTHREAD_MUTEX_INIT_VALUE, 1, { NULL }, { 0 }, 0, { NULL }, { 0 }, PTHREAD_MUTEX_INIT_VALUE }
 #define AST_MUTEX_INIT_VALUE_NOTRACKING \
                              { PTHREAD_MUTEX_INIT_VALUE, 0, { NULL }, { 0 }, 0, { NULL }, { 0 }, PTHREAD_MUTEX_INIT_VALUE }
+#endif
 
 #define AST_MAX_REENTRANCY 10
 
@@ -125,6 +133,9 @@ struct ast_mutex_info {
 	int reentrancy;
 	const char *func[AST_MAX_REENTRANCY];
 	pthread_t thread[AST_MAX_REENTRANCY];
+#ifdef HAVE_BKTR
+	struct ast_bt backtrace[AST_MAX_REENTRANCY];
+#endif
 	pthread_mutex_t reentr_mutex;
 };
 
@@ -149,12 +160,22 @@ enum ast_lock_type {
  * on the lock.  ast_mark_lock_acquired() will mark it as held by this thread.
  */
 #if !defined(LOW_MEMORY)
+#ifdef HAVE_BKTR
+void ast_store_lock_info(enum ast_lock_type type, const char *filename,
+	int line_num, const char *func, const char *lock_name, void *lock_addr, struct ast_bt *bt);
+#else
 void ast_store_lock_info(enum ast_lock_type type, const char *filename,
 	int line_num, const char *func, const char *lock_name, void *lock_addr);
+#endif /* HAVE_BKTR */
+
 #else
-#define ast_store_lock_info(I,DONT,CARE,ABOUT,THE,PARAMETERS)
-#endif
 
+#ifdef HAVE_BKTR
+#define ast_store_lock_info(I,DONT,CARE,ABOUT,THE,PARAMETERS,BUD)
+#else
+#define ast_store_lock_info(I,DONT,CARE,ABOUT,THE,PARAMETERS)
+#endif /* HAVE_BKTR */
+#endif /* !defined(LOW_MEMORY) */
 
 /*!
  * \brief Mark the last lock as acquired
@@ -181,12 +202,34 @@ void ast_mark_lock_failed(void *lock_addr);
  * be removed from the current thread's lock info struct.
  */
 #if !defined(LOW_MEMORY)
+#ifdef HAVE_BKTR
+void ast_remove_lock_info(void *lock_addr, struct ast_bt *bt);
+#else
 void ast_remove_lock_info(void *lock_addr);
+#endif /* HAVE_BKTR */
+#else
+#ifdef HAVE_BKTR
+#define ast_remove_lock_info(ignore,me)
 #else
 #define ast_remove_lock_info(ignore)
-#endif
+#endif /* HAVE_BKTR */
+#endif /* !defined(LOW_MEMORY) */
+
+#ifdef HAVE_BKTR
+static inline void __dump_backtrace(struct ast_bt *bt, int canlog)
+{
+	char **strings;
+
+	size_t i;
 
+	strings = backtrace_symbols(bt->addresses, bt->num_frames);
 
+	for (i = 0; i < bt->num_frames; i++)
+		__ast_mutex_logger("%s\n", strings[i]);
+
+	free(strings);
+}
+#endif
 
 /*!
  * \brief log info for the current lock with ast_log().
@@ -223,6 +266,9 @@ static inline void ast_reentrancy_init(ast_mutex_t *p_ast_mutex)
 		p_ast_mutex->lineno[i] = 0;
 		p_ast_mutex->func[i] = NULL;
 		p_ast_mutex->thread[i] = 0;
+#ifdef HAVE_BKTR
+		memset(&p_ast_mutex->backtrace[i], 0, sizeof(p_ast_mutex->backtrace[i]));
+#endif
 	}
 
 	p_ast_mutex->reentrancy = 0;
@@ -308,6 +354,9 @@ static inline int __ast_pthread_mutex_destroy(const char *filename, int lineno,
 		ast_reentrancy_lock(t);
 		__ast_mutex_logger("%s line %d (%s): Error: '%s' was locked here.\n",
 			    t->file[t->reentrancy-1], t->lineno[t->reentrancy-1], t->func[t->reentrancy-1], mutex_name);
+#ifdef HAVE_BKTR
+		__dump_backtrace(&t->backtrace[t->reentrancy-1], canlog);
+#endif
 		ast_reentrancy_unlock(t);
 		break;
 	}
@@ -325,6 +374,9 @@ static inline int __ast_pthread_mutex_destroy(const char *filename, int lineno,
 	t->func[0] = func;
 	t->reentrancy = 0;
 	t->thread[0] = 0;
+#ifdef HAVE_BKTR
+	memset(&t->backtrace[0], 0, sizeof(t->backtrace[0]));
+#endif
 	ast_reentrancy_unlock(t);
 	delete_reentrancy_cs(t);
 
@@ -336,6 +388,9 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 {
 	int res;
 	int canlog = strcmp(filename, "logger.c") & t->track;
+#ifdef HAVE_BKTR
+	struct ast_bt *bt = NULL;
+#endif
 
 #if defined(AST_MUTEX_INIT_W_CONSTRUCTORS)
 	if ((t->mutex) == ((pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER)) {
@@ -352,8 +407,17 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
 
-	if (t->track)
+	if (t->track) {
+#ifdef HAVE_BKTR
+		ast_reentrancy_lock(t);
+		ast_bt_get_addresses(&t->backtrace[t->reentrancy]);
+		bt = &t->backtrace[t->reentrancy];
+		ast_reentrancy_unlock(t);
+		ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex, bt);
+#else
 		ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex);
+#endif
+	}
 
 #ifdef DETECT_DEADLOCKS
 	{
@@ -373,9 +437,15 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 					__ast_mutex_logger("%s line %d (%s): Deadlock? waited %d sec for mutex '%s'?\n",
 							   filename, lineno, func, (int) wait_time, mutex_name);
 					ast_reentrancy_lock(t);
+#ifdef HAVE_BKTR
+					__dump_backtrace(&t->backtrace[t->reentrancy], canlog);
+#endif
 					__ast_mutex_logger("%s line %d (%s): '%s' was locked here.\n",
 							   t->file[t->reentrancy-1], t->lineno[t->reentrancy-1],
 							   t->func[t->reentrancy-1], mutex_name);
+#ifdef HAVE_BKTR
+					__dump_backtrace(&t->backtrace[t->reentrancy-1], canlog);
+#endif
 					ast_reentrancy_unlock(t);
 					reported_wait = wait_time;
 				}
@@ -409,8 +479,20 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 		if (t->track)
 			ast_mark_lock_acquired(&t->mutex);
 	} else {
+#ifdef HAVE_BKTR
+		if (t->reentrancy) {
+			ast_reentrancy_lock(t);
+			bt = &t->backtrace[t->reentrancy-1];
+			ast_reentrancy_unlock(t);
+		} else {
+			bt = NULL;
+		}
+		if (t->track)
+			ast_remove_lock_info(&t->mutex, bt);
+#else
 		if (t->track)
 			ast_remove_lock_info(&t->mutex);
+#endif
 		__ast_mutex_logger("%s line %d (%s): Error obtaining mutex: %s\n",
 				   filename, lineno, func, strerror(res));
 		DO_THREAD_CRASH;
@@ -424,6 +506,9 @@ static inline int __ast_pthread_mutex_trylock(const char *filename, int lineno,
 {
 	int res;
 	int canlog = strcmp(filename, "logger.c") & t->track;
+#ifdef HAVE_BKTR
+	struct ast_bt *bt = NULL;
+#endif
 
 #if defined(AST_MUTEX_INIT_W_CONSTRUCTORS)
 	if ((t->mutex) == ((pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER)) {
@@ -440,8 +525,17 @@ static inline int __ast_pthread_mutex_trylock(const char *filename, int lineno,
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
 
-	if (t->track)
+	if (t->track) {
+#ifdef HAVE_BKTR
+		ast_reentrancy_lock(t);
+		ast_bt_get_addresses(&t->backtrace[t->reentrancy]);
+		bt = &t->backtrace[t->reentrancy];
+		ast_reentrancy_unlock(t);
+		ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex, bt);
+#else
 		ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex);
+#endif
+	}
 
 	if (!(res = pthread_mutex_trylock(&t->mutex))) {
 		ast_reentrancy_lock(t);
@@ -470,6 +564,9 @@ static inline int __ast_pthread_mutex_unlock(const char *filename, int lineno, c
 {
 	int res;
 	int canlog = strcmp(filename, "logger.c") & t->track;
+#ifdef HAVE_BKTR
+	struct ast_bt *bt = NULL;
+#endif
 
 #ifdef AST_MUTEX_INIT_W_CONSTRUCTORS
 	if ((t->mutex) == ((pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER)) {
@@ -490,6 +587,9 @@ static inline int __ast_pthread_mutex_unlock(const char *filename, int lineno, c
 				   filename, lineno, func, mutex_name);
 		__ast_mutex_logger("%s line %d (%s): '%s' was locked here.\n",
 				   t->file[t->reentrancy-1], t->lineno[t->reentrancy-1], t->func[t->reentrancy-1], mutex_name);
+#ifdef HAVE_BKTR
+		__dump_backtrace(&t->backtrace[t->reentrancy-1], canlog);
+#endif
 		DO_THREAD_CRASH;
 	}
 
@@ -505,11 +605,21 @@ static inline int __ast_pthread_mutex_unlock(const char *filename, int lineno, c
 		t->func[t->reentrancy] = NULL;
 		t->thread[t->reentrancy] = 0;
 	}
+
+#ifdef HAVE_BKTR
+	if (t->reentrancy) {
+		bt = &t->backtrace[t->reentrancy - 1];
+	}
+#endif
 	ast_reentrancy_unlock(t);
 
-	if (t->track)
+	if (t->track) {
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(&t->mutex, bt);
+#else
 		ast_remove_lock_info(&t->mutex);
-
+#endif
+	}
 	if ((res = pthread_mutex_unlock(&t->mutex))) {
 		__ast_mutex_logger("%s line %d (%s): Error releasing mutex: %s\n", 
 				   filename, lineno, func, strerror(res));
@@ -549,6 +659,9 @@ static inline int __ast_cond_wait(const char *filename, int lineno, const char *
 {
 	int res;
 	int canlog = strcmp(filename, "logger.c") & t->track;
+#ifdef HAVE_BKTR
+	struct ast_bt *bt = NULL;
+#endif
 
 #ifdef AST_MUTEX_INIT_W_CONSTRUCTORS
 	if ((t->mutex) == ((pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER)) {
@@ -569,6 +682,9 @@ static inline int __ast_cond_wait(const char *filename, int lineno, const char *
 				   filename, lineno, func, mutex_name);
 		__ast_mutex_logger("%s line %d (%s): '%s' was locked here.\n",
 				   t->file[t->reentrancy-1], t->lineno[t->reentrancy-1], t->func[t->reentrancy-1], mutex_name);
+#ifdef HAVE_BKTR
+		__dump_backtrace(&t->backtrace[t->reentrancy-1], canlog);
+#endif
 		DO_THREAD_CRASH;
 	}
 
@@ -584,10 +700,21 @@ static inline int __ast_cond_wait(const char *filename, int lineno, const char *
 		t->func[t->reentrancy] = NULL;
 		t->thread[t->reentrancy] = 0;
 	}
+
+#ifdef HAVE_BKTR
+	if (t->reentrancy) {
+		bt = &t->backtrace[t->reentrancy - 1];
+	}
+#endif
 	ast_reentrancy_unlock(t);
 
-	if (t->track)
+	if (t->track) {
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(&t->mutex, bt);
+#else
 		ast_remove_lock_info(&t->mutex);
+#endif
+	}
 
 	if ((res = pthread_cond_wait(cond, &t->mutex))) {
 		__ast_mutex_logger("%s line %d (%s): Error waiting on condition mutex '%s'\n", 
@@ -600,6 +727,10 @@ static inline int __ast_cond_wait(const char *filename, int lineno, const char *
 			t->lineno[t->reentrancy] = lineno;
 			t->func[t->reentrancy] = func;
 			t->thread[t->reentrancy] = pthread_self();
+#ifdef HAVE_BKTR
+			ast_bt_get_addresses(&t->backtrace[t->reentrancy]);
+			bt = &t->backtrace[t->reentrancy];
+#endif
 			t->reentrancy++;
 		} else {
 			__ast_mutex_logger("%s line %d (%s): '%s' really deep reentrancy!\n",
@@ -607,8 +738,13 @@ static inline int __ast_cond_wait(const char *filename, int lineno, const char *
 		}
 		ast_reentrancy_unlock(t);
 
-		if (t->track)
+		if (t->track) {
+#ifdef HAVE_BKTR
+			ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex, bt);
+#else
 			ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex);
+#endif
+		}
 	}
 
 	return res;
@@ -620,6 +756,9 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 {
 	int res;
 	int canlog = strcmp(filename, "logger.c") & t->track;
+#ifdef HAVE_BKTR
+	struct ast_bt *bt = NULL;
+#endif
 
 #ifdef AST_MUTEX_INIT_W_CONSTRUCTORS
 	if ((t->mutex) == ((pthread_mutex_t) PTHREAD_MUTEX_INITIALIZER)) {
@@ -640,6 +779,9 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 				   filename, lineno, func, mutex_name);
 		__ast_mutex_logger("%s line %d (%s): '%s' was locked here.\n",
 				   t->file[t->reentrancy-1], t->lineno[t->reentrancy-1], t->func[t->reentrancy-1], mutex_name);
+#ifdef HAVE_BKTR
+		__dump_backtrace(&t->backtrace[t->reentrancy-1], canlog);
+#endif
 		DO_THREAD_CRASH;
 	}
 
@@ -655,10 +797,19 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 		t->func[t->reentrancy] = NULL;
 		t->thread[t->reentrancy] = 0;
 	}
+#ifdef HAVE_BKTR
+	if (t->reentrancy) {
+		bt = &t->backtrace[t->reentrancy - 1];
+	}
+#endif
 	ast_reentrancy_unlock(t);
 
 	if (t->track)
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(&t->mutex, bt);
+#else
 		ast_remove_lock_info(&t->mutex);
+#endif
 
 	if ((res = pthread_cond_timedwait(cond, &t->mutex, abstime)) && (res != ETIMEDOUT)) {
 		__ast_mutex_logger("%s line %d (%s): Error waiting on condition mutex '%s'\n", 
@@ -671,6 +822,10 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 			t->lineno[t->reentrancy] = lineno;
 			t->func[t->reentrancy] = func;
 			t->thread[t->reentrancy] = pthread_self();
+#ifdef HAVE_BKTR
+			ast_bt_get_addresses(&t->backtrace[t->reentrancy]);
+			bt = &t->backtrace[t->reentrancy];
+#endif
 			t->reentrancy++;
 		} else {
 			__ast_mutex_logger("%s line %d (%s): '%s' really deep reentrancy!\n",
@@ -678,8 +833,13 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 		}
 		ast_reentrancy_unlock(t);
 
-		if (t->track)
+		if (t->track) {
+#ifdef HAVE_BKTR
+			ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex, bt);
+#else
 			ast_store_lock_info(AST_MUTEX, filename, lineno, func, mutex_name, &t->mutex);
+#endif
+		}
 	}
 
 	return res;
@@ -696,151 +856,14 @@ static inline int __ast_cond_timedwait(const char *filename, int lineno, const c
 #define ast_cond_wait(cond, mutex)		__ast_cond_wait(__FILE__, __LINE__, __PRETTY_FUNCTION__, #cond, #mutex, cond, mutex)
 #define ast_cond_timedwait(cond, mutex, time)	__ast_cond_timedwait(__FILE__, __LINE__, __PRETTY_FUNCTION__, #cond, #mutex, cond, mutex, time)
 
-#else /* !DEBUG_THREADS */
-
-
-typedef pthread_mutex_t ast_mutex_t;
-
-#define AST_MUTEX_INIT_VALUE			((ast_mutex_t) PTHREAD_MUTEX_INIT_VALUE)
-#define AST_MUTEX_INIT_VALUE_NOTRACKING		((ast_mutex_t) PTHREAD_MUTEX_INIT_VALUE)
-
-#define ast_mutex_init_notracking(m)		ast_mutex_init(m)
-
-static inline int ast_mutex_init(ast_mutex_t *pmutex)
-{
-	int res;
-	pthread_mutexattr_t attr;
-
-	pthread_mutexattr_init(&attr);
-	pthread_mutexattr_settype(&attr, AST_MUTEX_KIND);
-
-	res = pthread_mutex_init(pmutex, &attr);
-	pthread_mutexattr_destroy(&attr);
-	return res;
-}
-
-#define ast_pthread_mutex_init(pmutex,a) pthread_mutex_init(pmutex,a)
-
-static inline int ast_mutex_unlock(ast_mutex_t *pmutex)
-{
-	return pthread_mutex_unlock(pmutex);
-}
-
-static inline int ast_mutex_destroy(ast_mutex_t *pmutex)
-{
-	return pthread_mutex_destroy(pmutex);
-}
-
-static inline int ast_mutex_lock(ast_mutex_t *pmutex)
-{
-	__MTX_PROF(pmutex);
-}
-
-static inline int ast_mutex_trylock(ast_mutex_t *pmutex)
-{
-	return pthread_mutex_trylock(pmutex);
-}
-
-typedef pthread_cond_t ast_cond_t;
-
-static inline int ast_cond_init(ast_cond_t *cond, pthread_condattr_t *cond_attr)
-{
-	return pthread_cond_init(cond, cond_attr);
-}
-
-static inline int ast_cond_signal(ast_cond_t *cond)
-{
-	return pthread_cond_signal(cond);
-}
-
-static inline int ast_cond_broadcast(ast_cond_t *cond)
-{
-	return pthread_cond_broadcast(cond);
-}
-
-static inline int ast_cond_destroy(ast_cond_t *cond)
-{
-	return pthread_cond_destroy(cond);
-}
-
-static inline int ast_cond_wait(ast_cond_t *cond, ast_mutex_t *t)
-{
-	return pthread_cond_wait(cond, t);
-}
-
-static inline int ast_cond_timedwait(ast_cond_t *cond, ast_mutex_t *t, const struct timespec *abstime)
-{
-	return pthread_cond_timedwait(cond, t, abstime);
-}
-
-#endif /* !DEBUG_THREADS */
-
-#if defined(AST_MUTEX_INIT_W_CONSTRUCTORS)
-/*
- * If AST_MUTEX_INIT_W_CONSTRUCTORS is defined, use file scope constructors
- * and destructors to create/destroy global mutexes.
- */
-#define __AST_MUTEX_DEFINE(scope, mutex, init_val, track)	\
-	scope ast_mutex_t mutex = init_val;			\
-static void  __attribute__ ((constructor)) init_##mutex(void)	\
-{								\
-	if (track)						\
-		ast_mutex_init(&mutex);				\
-	else							\
-		ast_mutex_init_notracking(&mutex);		\
-}								\
-								\
-static void  __attribute__ ((destructor)) fini_##mutex(void)	\
-{								\
-	ast_mutex_destroy(&mutex);				\
-}
-#else /* !AST_MUTEX_INIT_W_CONSTRUCTORS */
-/* By default, use static initialization of mutexes. */ 
-#define __AST_MUTEX_DEFINE(scope, mutex, init_val, track)	scope ast_mutex_t mutex = init_val
-#endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
-
-#ifndef __CYGWIN__	/* temporary disabled for cygwin */
-#define pthread_mutex_t		use_ast_mutex_t_instead_of_pthread_mutex_t
-#define pthread_cond_t		use_ast_cond_t_instead_of_pthread_cond_t
-#endif
-#define pthread_mutex_lock	use_ast_mutex_lock_instead_of_pthread_mutex_lock
-#define pthread_mutex_unlock	use_ast_mutex_unlock_instead_of_pthread_mutex_unlock
-#define pthread_mutex_trylock	use_ast_mutex_trylock_instead_of_pthread_mutex_trylock
-#define pthread_mutex_init	use_ast_mutex_init_instead_of_pthread_mutex_init
-#define pthread_mutex_destroy	use_ast_mutex_destroy_instead_of_pthread_mutex_destroy
-#define pthread_cond_init	use_ast_cond_init_instead_of_pthread_cond_init
-#define pthread_cond_destroy	use_ast_cond_destroy_instead_of_pthread_cond_destroy
-#define pthread_cond_signal	use_ast_cond_signal_instead_of_pthread_cond_signal
-#define pthread_cond_broadcast	use_ast_cond_broadcast_instead_of_pthread_cond_broadcast
-#define pthread_cond_wait	use_ast_cond_wait_instead_of_pthread_cond_wait
-#define pthread_cond_timedwait	use_ast_cond_timedwait_instead_of_pthread_cond_timedwait
-
-#define AST_MUTEX_DEFINE_STATIC(mutex)			__AST_MUTEX_DEFINE(static, mutex, AST_MUTEX_INIT_VALUE, 1)
-#define AST_MUTEX_DEFINE_STATIC_NOTRACKING(mutex)	__AST_MUTEX_DEFINE(static, mutex, AST_MUTEX_INIT_VALUE_NOTRACKING, 0)
-
-#define AST_MUTEX_INITIALIZER __use_AST_MUTEX_DEFINE_STATIC_rather_than_AST_MUTEX_INITIALIZER__
-
-#define gethostbyname __gethostbyname__is__not__reentrant__use__ast_gethostbyname__instead__
-
-#ifndef __linux__
-#define pthread_create __use_ast_pthread_create_instead__
-#endif
-
-/*
- * Same as above, definitions of ast_rwlock_t for the various cases:
- * simple wrappers for the pthread equivalent in the non-debug case,
- * more sophisticated tracking in the debug case.
- */
-
-typedef pthread_rwlock_t ast_rwlock_t;
-
-#ifdef HAVE_PTHREAD_RWLOCK_INITIALIZER
-#define AST_RWLOCK_INIT_VALUE PTHREAD_RWLOCK_INITIALIZER
-#else
-#define AST_RWLOCK_INIT_VALUE { 0 }
+struct ast_rwlock_info {
+	pthread_rwlock_t lock;
+#ifdef HAVE_BKTR
+	struct ast_bt backtrace;
 #endif
+};
 
-#ifdef DEBUG_THREADS
+typedef struct ast_rwlock_info ast_rwlock_t;
 
 #define ast_rwlock_init(rwlock)		__ast_rwlock_init(__FILE__, __LINE__, __PRETTY_FUNCTION__, #rwlock, rwlock)
 #define ast_rwlock_destroy(rwlock)	__ast_rwlock_destroy(__FILE__, __LINE__, __PRETTY_FUNCTION__, #rwlock, rwlock)
@@ -850,6 +873,19 @@ typedef pthread_rwlock_t ast_rwlock_t;
 #define ast_rwlock_tryrdlock(a)		_ast_rwlock_tryrdlock(a, # a, __FILE__, __LINE__, __PRETTY_FUNCTION__)
 #define ast_rwlock_trywrlock(a) _ast_rwlock_trywrlock(a, # a, __FILE__, __LINE__, __PRETTY_FUNCTION__)
 
+#ifdef HAVE_PTHREAD_RWLOCK_INITIALIZER
+#ifdef HAVE_BKTR
+#define AST_RWLOCK_INIT_VALUE { PTHREAD_RWLOCK_INITIALIZER, {{0,},} }
+#else
+#define AST_RWLOCK_INIT_VALUE { PTHREAD_RWLOCK_INITIALIZER }
+#endif /* HAVE_BKTR */
+#else  /* HAVE_PTHREAD_RWLOCK_INITIALIZER */
+#ifdef HAVE_BKTR
+#define AST_RWLOCK_INIT_VALUE { 0 , {0,},}}
+#else
+#define AST_RWLOCK_INIT_VALUE { 0 }
+#endif /* HAVE_BKTR */
+#endif /* HAVE_PTHREAD_RWLOCK_INITIALIZER */
 
 static inline int __ast_rwlock_init(const char *filename, int lineno, const char *func, const char *rwlock_name, ast_rwlock_t *prwlock)
 {
@@ -870,7 +906,7 @@ static inline int __ast_rwlock_init(const char *filename, int lineno, const char
 	pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NP);
 #endif
 
-	res = pthread_rwlock_init(prwlock, &attr);
+	res = pthread_rwlock_init(&prwlock->lock, &attr);
 	pthread_rwlockattr_destroy(&attr);
 	return res;
 }
@@ -889,7 +925,7 @@ static inline int __ast_rwlock_destroy(const char *filename, int lineno, const c
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
 	
-	if ((res = pthread_rwlock_destroy(prwlock)))
+	if ((res = pthread_rwlock_destroy(&prwlock->lock)))
 		__ast_mutex_logger("%s line %d (%s): Error destroying rwlock %s: %s\n",
 				filename, lineno, func, rwlock_name, strerror(res));
 
@@ -916,8 +952,13 @@ static inline int _ast_rwlock_unlock(ast_rwlock_t *lock, const char *name,
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
 	
-	res = pthread_rwlock_unlock(lock);
+	res = pthread_rwlock_unlock(&lock->lock);
+#ifdef HAVE_BKTR
+	memset(&lock->backtrace, 0, sizeof(lock->backtrace));
+	ast_remove_lock_info(lock, NULL);
+#else
 	ast_remove_lock_info(lock);
+#endif
 	return res;
 }
 
@@ -942,13 +983,20 @@ static inline int _ast_rwlock_rdlock(ast_rwlock_t *lock, const char *name,
 		}
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
-	
+#ifdef HAVE_BKTR	
+	ast_store_lock_info(AST_RDLOCK, file, line, func, name, lock, &lock->backtrace);
+#else
 	ast_store_lock_info(AST_RDLOCK, file, line, func, name, lock);
-	res = pthread_rwlock_rdlock(lock);
+#endif
+	res = pthread_rwlock_rdlock(&lock->lock);
 	if (!res)
 		ast_mark_lock_acquired(lock);
 	else
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(lock, NULL);
+#else
 		ast_remove_lock_info(lock);
+#endif
 	return res;
 }
 
@@ -973,13 +1021,20 @@ static inline int _ast_rwlock_wrlock(ast_rwlock_t *lock, const char *name,
 		}
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
-
+#ifdef HAVE_BKTR
+	ast_store_lock_info(AST_WRLOCK, file, line, func, name, lock, &lock->backtrace);
+#else
 	ast_store_lock_info(AST_WRLOCK, file, line, func, name, lock);
-	res = pthread_rwlock_wrlock(lock);
+#endif
+	res = pthread_rwlock_wrlock(&lock->lock);
 	if (!res)
 		ast_mark_lock_acquired(lock);
 	else
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(lock, NULL);
+#else
 		ast_remove_lock_info(lock);
+#endif
 	return res;
 }
 
@@ -1004,13 +1059,20 @@ static inline int _ast_rwlock_tryrdlock(ast_rwlock_t *lock, const char *name,
 		}
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
-
+#ifdef HAVE_BKTR
+	ast_store_lock_info(AST_RDLOCK, file, line, func, name, lock, &lock->backtrace);
+#else
 	ast_store_lock_info(AST_RDLOCK, file, line, func, name, lock);
-	res = pthread_rwlock_tryrdlock(lock);
+#endif
+	res = pthread_rwlock_tryrdlock(&lock->lock);
 	if (!res)
 		ast_mark_lock_acquired(lock);
 	else
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(lock, NULL);
+#else
 		ast_remove_lock_info(lock);
+#endif
 	return res;
 }
 
@@ -1035,18 +1097,108 @@ static inline int _ast_rwlock_trywrlock(ast_rwlock_t *lock, const char *name,
 		}
 	}
 #endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
-
+#ifdef HAVE_BKTR
+	ast_store_lock_info(AST_WRLOCK, file, line, func, name, lock, &lock->backtrace);
+#else
 	ast_store_lock_info(AST_WRLOCK, file, line, func, name, lock);
-	res = pthread_rwlock_trywrlock(lock);
+#endif
+	res = pthread_rwlock_trywrlock(&lock->lock);
 	if (!res)
 		ast_mark_lock_acquired(lock);
 	else
+#ifdef HAVE_BKTR
+		ast_remove_lock_info(lock, NULL);
+#else
 		ast_remove_lock_info(lock);
+#endif
 	return res;
 }
 
 #else /* !DEBUG_THREADS */
 
+typedef pthread_mutex_t ast_mutex_t;
+
+#define AST_MUTEX_INIT_VALUE			((ast_mutex_t) PTHREAD_MUTEX_INIT_VALUE)
+#define AST_MUTEX_INIT_VALUE_NOTRACKING		((ast_mutex_t) PTHREAD_MUTEX_INIT_VALUE)
+
+#define ast_mutex_init_notracking(m)		ast_mutex_init(m)
+
+static inline int ast_mutex_init(ast_mutex_t *pmutex)
+{
+	int res;
+	pthread_mutexattr_t attr;
+
+	pthread_mutexattr_init(&attr);
+	pthread_mutexattr_settype(&attr, AST_MUTEX_KIND);
+
+	res = pthread_mutex_init(pmutex, &attr);
+	pthread_mutexattr_destroy(&attr);
+	return res;
+}
+
+#define ast_pthread_mutex_init(pmutex,a) pthread_mutex_init(pmutex,a)
+
+static inline int ast_mutex_unlock(ast_mutex_t *pmutex)
+{
+	return pthread_mutex_unlock(pmutex);
+}
+
+static inline int ast_mutex_destroy(ast_mutex_t *pmutex)
+{
+	return pthread_mutex_destroy(pmutex);
+}
+
+static inline int ast_mutex_lock(ast_mutex_t *pmutex)
+{
+	__MTX_PROF(pmutex);
+}
+
+static inline int ast_mutex_trylock(ast_mutex_t *pmutex)
+{
+	return pthread_mutex_trylock(pmutex);
+}
+
+typedef pthread_cond_t ast_cond_t;
+
+static inline int ast_cond_init(ast_cond_t *cond, pthread_condattr_t *cond_attr)
+{
+	return pthread_cond_init(cond, cond_attr);
+}
+
+static inline int ast_cond_signal(ast_cond_t *cond)
+{
+	return pthread_cond_signal(cond);
+}
+
+static inline int ast_cond_broadcast(ast_cond_t *cond)
+{
+	return pthread_cond_broadcast(cond);
+}
+
+static inline int ast_cond_destroy(ast_cond_t *cond)
+{
+	return pthread_cond_destroy(cond);
+}
+
+static inline int ast_cond_wait(ast_cond_t *cond, ast_mutex_t *t)
+{
+	return pthread_cond_wait(cond, t);
+}
+
+static inline int ast_cond_timedwait(ast_cond_t *cond, ast_mutex_t *t, const struct timespec *abstime)
+{
+	return pthread_cond_timedwait(cond, t, abstime);
+}
+
+
+typedef pthread_rwlock_t ast_rwlock_t;
+
+#ifdef HAVE_PTHREAD_RWLOCK_INITIALIZER
+#define AST_RWLOCK_INIT_VALUE PTHREAD_RWLOCK_INITIALIZER
+#else
+#define AST_RWLOCK_INIT_VALUE { 0 }
+#endif
+
 static inline int ast_rwlock_init(ast_rwlock_t *prwlock)
 {
 	int res;
@@ -1092,8 +1244,60 @@ static inline int ast_rwlock_trywrlock(ast_rwlock_t *prwlock)
 {
 	return pthread_rwlock_trywrlock(prwlock);
 }
+
 #endif /* !DEBUG_THREADS */
 
+#if defined(AST_MUTEX_INIT_W_CONSTRUCTORS)
+/*
+ * If AST_MUTEX_INIT_W_CONSTRUCTORS is defined, use file scope constructors
+ * and destructors to create/destroy global mutexes.
+ */
+#define __AST_MUTEX_DEFINE(scope, mutex, init_val, track)	\
+	scope ast_mutex_t mutex = init_val;			\
+static void  __attribute__ ((constructor)) init_##mutex(void)	\
+{								\
+	if (track)						\
+		ast_mutex_init(&mutex);				\
+	else							\
+		ast_mutex_init_notracking(&mutex);		\
+}								\
+								\
+static void  __attribute__ ((destructor)) fini_##mutex(void)	\
+{								\
+	ast_mutex_destroy(&mutex);				\
+}
+#else /* !AST_MUTEX_INIT_W_CONSTRUCTORS */
+/* By default, use static initialization of mutexes. */ 
+#define __AST_MUTEX_DEFINE(scope, mutex, init_val, track)	scope ast_mutex_t mutex = init_val
+#endif /* AST_MUTEX_INIT_W_CONSTRUCTORS */
+
+#ifndef __CYGWIN__	/* temporary disabled for cygwin */
+#define pthread_mutex_t		use_ast_mutex_t_instead_of_pthread_mutex_t
+#define pthread_cond_t		use_ast_cond_t_instead_of_pthread_cond_t
+#endif
+#define pthread_mutex_lock	use_ast_mutex_lock_instead_of_pthread_mutex_lock
+#define pthread_mutex_unlock	use_ast_mutex_unlock_instead_of_pthread_mutex_unlock
+#define pthread_mutex_trylock	use_ast_mutex_trylock_instead_of_pthread_mutex_trylock
+#define pthread_mutex_init	use_ast_mutex_init_instead_of_pthread_mutex_init
+#define pthread_mutex_destroy	use_ast_mutex_destroy_instead_of_pthread_mutex_destroy
+#define pthread_cond_init	use_ast_cond_init_instead_of_pthread_cond_init
+#define pthread_cond_destroy	use_ast_cond_destroy_instead_of_pthread_cond_destroy
+#define pthread_cond_signal	use_ast_cond_signal_instead_of_pthread_cond_signal
+#define pthread_cond_broadcast	use_ast_cond_broadcast_instead_of_pthread_cond_broadcast
+#define pthread_cond_wait	use_ast_cond_wait_instead_of_pthread_cond_wait
+#define pthread_cond_timedwait	use_ast_cond_timedwait_instead_of_pthread_cond_timedwait
+
+#define AST_MUTEX_DEFINE_STATIC(mutex)			__AST_MUTEX_DEFINE(static, mutex, AST_MUTEX_INIT_VALUE, 1)
+#define AST_MUTEX_DEFINE_STATIC_NOTRACKING(mutex)	__AST_MUTEX_DEFINE(static, mutex, AST_MUTEX_INIT_VALUE_NOTRACKING, 0)
+
+#define AST_MUTEX_INITIALIZER __use_AST_MUTEX_DEFINE_STATIC_rather_than_AST_MUTEX_INITIALIZER__
+
+#define gethostbyname __gethostbyname__is__not__reentrant__use__ast_gethostbyname__instead__
+
+#ifndef __linux__
+#define pthread_create __use_ast_pthread_create_instead__
+#endif
+
 /* Statically declared read/write locks */
 
 #ifndef HAVE_PTHREAD_RWLOCK_INITIALIZER