Totally revamp thread debugging to support locating and removing deadlocks

git-svn-id: http://svn.digium.com/svn/asterisk/trunk@1310 f38db490-d61c-443f-a65b-d21fe96a405b

6 files changed, 136 insertions, 52 deletions

diff --git a/include/asterisk/channel.h b/include/asterisk/channel.h
index 5fe9d2275..7cd2470e0 100755
--- a/include/asterisk/channel.h
+++ b/include/asterisk/channel.h
@@ -97,7 +97,7 @@ struct ast_channel {
 	/*! If anyone is blocking, this is them */
 	pthread_t blocker;			
 	/*! Lock, can be used to lock a channel for some operations */
-	pthread_mutex_t lock;			
+	ast_mutex_t lock;			
 	/*! Procedure causing blocking */
 	char *blockproc;			
 
diff --git a/include/asterisk/indications.h b/include/asterisk/indications.h
index 0ee0fac1d..9b8ebca5c 100755
--- a/include/asterisk/indications.h
+++ b/include/asterisk/indications.h
@@ -22,6 +22,8 @@
 #ifndef _ASTERISK_INDICATIONS_H
 #define _ASTERISK_INDICATIONS_H
 
+#include <asterisk/lock.h>
+
 /* forward reference */
 struct ast_channel;
 
@@ -70,6 +72,6 @@ int ast_playtones_start(struct ast_channel *chan, int vol, const char* tonelist,
 void ast_playtones_stop(struct ast_channel *chan);
 
 extern struct tone_zone *tone_zones;
-extern pthread_mutex_t tzlock;
+extern ast_mutex_t tzlock;
 
 #endif
diff --git a/include/asterisk/linkedlists.h b/include/asterisk/linkedlists.h
index c1a3d9de3..30e09a015 100755
--- a/include/asterisk/linkedlists.h
+++ b/include/asterisk/linkedlists.h
@@ -2,25 +2,26 @@
 #define ASTERISK_LINKEDLISTS_H
 
 #include <pthread.h>
+#include <asterisk/lock.h>
 
 #define AST_LIST_LOCK(head)						\
-	ast_pthread_mutex_lock(&head->lock) 
+	ast_mutex_lock(&head->lock) 
 	
 #define AST_LIST_UNLOCK(head) 						\
-	ast_pthread_mutex_unlock(&head->lock)
+	ast_mutex_unlock(&head->lock)
 
 #define AST_LIST_HEAD(name, type)					\
 struct name {								\
 	struct type *first;						\
-	pthread_mutex_t lock;						\
+	ast_mutex_t lock;						\
 }
 
 #define AST_LIST_HEAD_INITIALIZER(head)					\
-	{ NULL, PTHREAD_MUTEX_INITIALIZER }
+	{ NULL, AST_MUTEX_INITIALIZER }
 	
 #define AST_LIST_HEAD_SET(head,entry) do {				\
 	(head)->first=(entry);						\
-	pthread_mutex_init(&(head)->lock,NULL);				\
+	ast_pthread_mutex_init(&(head)->lock,NULL);				\
 } while (0)
 
 #define AST_LIST_ENTRY(type)						\
@@ -39,7 +40,7 @@ struct {								\
 
 #define AST_LIST_HEAD_INIT(head) {						\
 	(head)->first = NULL;						\
-	pthread_mutex_init(&(head)->lock,NULL);				\
+	ast_pthread_mutex_init(&(head)->lock,NULL);				\
 }
 
 #define AST_LIST_INSERT_AFTER(listelm, elm, field) do {		\
diff --git a/include/asterisk/lock.h b/include/asterisk/lock.h
index d27e295b4..092672cae 100755
--- a/include/asterisk/lock.h
+++ b/include/asterisk/lock.h
@@ -28,12 +28,12 @@
 // #define AST_MUTEX_INITIALIZER      PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
 // #define AST_MUTEX_KIND             PTHREAD_MUTEX_RECURSIVE_NP
 #ifdef PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
-#define AST_MUTEX_INITIALIZER         PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
+#define AST_MUTEX_INITIALIZER         { PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP, NULL, 0, NULL, 0 }
 #else
 #ifdef PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
-#define AST_MUTEX_INITIALIZER         PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
+#define AST_MUTEX_INITIALIZER         { PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP, NULL, 0, NULL, 0 }
 #else
-#define AST_MUTEX_INITIALIZER         PTHREAD_MUTEX_INITIALIZER
+#define AST_MUTEX_INITIALIZER         { PTHREAD_MUTEX_INITIALIZER, NULL, 0, NULL, 0 }
 #endif
 #endif
 #ifdef PTHREAD_MUTEX_ERRORCHECK_NP
@@ -42,17 +42,20 @@
 #define AST_MUTEX_KIND                PTHREAD_MUTEX_ERRORCHECK
 #endif
 
-struct mutex_info {
-	pthread_mutex_t *mutex;
+struct ast_mutex_info {
+	pthread_mutex_t mutex;
 	char *file;
 	int lineno;
 	char *func;
-	struct mutex_info *next;
+	pthread_t thread;
 };
 
-static inline int ast_pthread_mutex_init(pthread_mutex_t *t) {
+typedef struct ast_mutex_info ast_mutex_t;
+
+static inline int ast_mutex_init(ast_mutex_t *t) {
 	static pthread_mutexattr_t  attr;
 	static int  init = 1;
+	int res;
 	extern int  pthread_mutexattr_setkind_np(pthread_mutexattr_t *, int);
 
 	if (init) {
@@ -60,45 +63,87 @@ static inline int ast_pthread_mutex_init(pthread_mutex_t *t) {
 		pthread_mutexattr_setkind_np(&attr, AST_MUTEX_KIND);
 		init = 0;
 	}
-	return pthread_mutex_init(t, &attr);
+	res = pthread_mutex_init(&t->mutex, &attr);
+	t->file = NULL;
+	t->lineno = 0;
+	t->func = 0;
+	t->thread  = 0;
+	return res;
 }
 
-static inline int __ast_pthread_mutex_lock(char *filename, int lineno, char *func, pthread_mutex_t *t) {
+static inline int ast_pthread_mutex_init(ast_mutex_t *t, pthread_mutexattr_t *attr) 
+{
 	int res;
-	int tries = TRIES;
-	do {
-		res = pthread_mutex_trylock(t);
-		/* If we can't run, yield */
-		if (res) {
-			sched_yield();
-			usleep(1);
-		}
-	} while(res && tries--);
-	if (res) {
-		fprintf(stderr, "%s line %d (%s): Error obtaining mutex: %s\n", 
-				filename, lineno, func, strerror(res));
-		if ((res = pthread_mutex_lock(t)))
-                        fprintf(stderr, "%s line %d (%s): Error waiting for mutex: %s\n", 
-                               filename, lineno, func, strerror(res));
-                else
-		        fprintf(stderr, "%s line %d (%s): Got it eventually...\n",
-			       filename, lineno, func);
+	res = pthread_mutex_init(&t->mutex, attr);
+	t->file = NULL;
+	t->lineno = 0;
+	t->func = 0;
+	t->thread  = 0;
+	return res;
+}
+
+static inline int __ast_pthread_mutex_lock(char *filename, int lineno, char *func, ast_mutex_t *t) {
+	int res;
+	res = pthread_mutex_lock(&t->mutex);
+	if (!res) {
+		t->file = filename;
+		t->lineno = lineno;
+		t->func = func;
+		t->thread = pthread_self();
+	} else {
+		fprintf(stderr, "%s line %d (%s): Error obtaining mutex: %s\n",
+			filename, lineno, func, strerror(errno));
 	}
 	return res;
 }
 
-#define ast_pthread_mutex_lock(a) __ast_pthread_mutex_lock(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
+#define ast_mutex_lock(a) __ast_pthread_mutex_lock(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
 
-static inline int __ast_pthread_mutex_unlock(char *filename, int lineno, char *func, pthread_mutex_t *t) {
+static inline int __ast_pthread_mutex_trylock(char *filename, int lineno, char *func, ast_mutex_t *t) {
 	int res;
-	res = pthread_mutex_unlock(t);
+	res = pthread_mutex_trylock(&t->mutex);
+	if (!res) {
+		t->file = filename;
+		t->lineno = lineno;
+		t->func = func;
+		t->thread = pthread_self();
+	}
+	return res;
+}
+
+#define ast_mutex_trylock(a) __ast_pthread_mutex_trylock(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
+
+static inline int __ast_pthread_mutex_unlock(char *filename, int lineno, char *func, ast_mutex_t *t) {
+	int res;
+	/* Assumes lock is actually held */
+	t->file = NULL;
+	t->lineno = 0;
+	t->func = NULL;
+	t->thread = 0;
+	res = pthread_mutex_unlock(&t->mutex);
 	if (res) 
 		fprintf(stderr, "%s line %d (%s): Error releasing mutex: %s\n", 
 				filename, lineno, func, strerror(res));
 	return res;
 }
 
-#define ast_pthread_mutex_unlock(a) __ast_pthread_mutex_unlock(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
+#define ast_mutex_unlock(a) __ast_pthread_mutex_unlock(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
+
+static inline int __ast_pthread_mutex_destroy(char *filename, int lineno, char *func, ast_mutex_t *t)
+{
+	int res;
+	t->file = NULL;
+	t->lineno = 0;
+	t->func = NULL;
+	t->thread = 0;
+	res = pthread_mutex_destroy(&t->mutex);
+	if (res) 
+		fprintf(stderr, "%s line %d (%s): Error destroying mutex: %s\n",
+				filename, lineno, func, strerror(res));
+	return res;
+}
+
+#define ast_mutex_destroy(a) __ast_pthread_mutex_destroy(__FILE__, __LINE__, __PRETTY_FUNCTION__, a)
 
 #else
 
@@ -109,10 +154,44 @@ static inline int __ast_pthread_mutex_unlock(char *filename, int lineno, char *f
 #define AST_MUTEX_KIND             PTHREAD_NORMAL
 #endif
 
-#define ast_pthread_mutex_init(mutex) pthread_mutex_init(mutex, NULL)
-#define ast_pthread_mutex_lock pthread_mutex_lock
-#define ast_pthread_mutex_unlock pthread_mutex_unlock
+typedef pthread_mutex_t ast_mutex_t;
+
+static inline int ast_mutex_lock(ast_mutex_t *t)
+{
+	return pthread_mutex_lock(t);
+}
+
+static inline int ast_mutex_unlock(ast_mutex_t *t)
+{
+	return pthread_mutex_unlock(t);
+}
+
+static inline int ast_mutex_trylock(ast_mutex_t *t)
+{
+	return pthread_mutex_trylock(t);
+}
+
+static inline int ast_pthread_mutex_init(ast_mutex_t *t, const pthread_mutexattr_t *mutexattr)
+{
+	return pthread_mutex_init(t, mutexattr);
+}
+
+static inline int ast_mutex_init(ast_mutex_t *t)
+{
+	return pthread_mutex_init(t, NULL);
+}
 
+static inline int ast_mutex_destroy(ast_mutex_t *t)
+{
+	return pthread_mutex_destroy(t);
+}
 #endif
 
+#define pthread_mutex_t use_ast_mutex_t_instead_of_pthread_mutex_t
+#define pthread_mutex_lock use_ast_mutex_lock_instead_of_pthread_mutex_lock
+#define pthread_mutex_unlock use_ast_mutex_unlock_instead_of_pthread_mutex_unlock
+#define pthread_mutex_trylock use_ast_mutex_trylock_instead_of_pthread_mutex_trylock
+#define pthread_mutex_init use_ast_pthread_mutex_init_instead_of_pthread_mutex_init
+#define pthread_mutex_destroy use_ast_pthread_mutex_destroy_instead_of_pthread_mutex_destroy
+
 #endif
diff --git a/include/asterisk/manager.h b/include/asterisk/manager.h
index d5171ecc2..66da69017 100755
--- a/include/asterisk/manager.h
+++ b/include/asterisk/manager.h
@@ -23,6 +23,8 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
+#include <asterisk/lock.h>
+
 /* 
  * Call management packages are text fields of the form a: b.  There is
  * always exactly one space after the colon.
@@ -53,7 +55,7 @@
 
 struct mansession {
 	pthread_t t;
-	pthread_mutex_t lock;
+	ast_mutex_t lock;
 	struct sockaddr_in sin;
 	int fd;
 	int blocking;
diff --git a/include/asterisk/module.h b/include/asterisk/module.h
index 39eabce8e..b7a5184e8 100755
--- a/include/asterisk/module.h
+++ b/include/asterisk/module.h
@@ -152,7 +152,7 @@ void ast_unregister_atexit(void (*func)(void));
 								struct localuser *next; \
 							}
 
-#define LOCAL_USER_DECL static pthread_mutex_t localuser_lock = AST_MUTEX_INITIALIZER; \
+#define LOCAL_USER_DECL static ast_mutex_t localuser_lock = AST_MUTEX_INITIALIZER; \
 						static struct localuser *localusers = NULL; \
 						static int localusecnt = 0;
 
@@ -162,18 +162,18 @@ void ast_unregister_atexit(void (*func)(void));
 		ast_log(LOG_WARNING, "Out of memory\n"); \
 		return -1; \
 	} \
-	pthread_mutex_lock(&localuser_lock); \
+	ast_mutex_lock(&localuser_lock); \
 	u->chan = chan; \
 	u->next = localusers; \
 	localusers = u; \
 	localusecnt++; \
-	pthread_mutex_unlock(&localuser_lock); \
+	ast_mutex_unlock(&localuser_lock); \
 	ast_update_use_count(); \
 }
 
 #define LOCAL_USER_REMOVE(u) { \
 	struct localuser *uc, *ul = NULL; \
-	pthread_mutex_lock(&localuser_lock); \
+	ast_mutex_lock(&localuser_lock); \
 	uc = localusers; \
 	while (uc) { \
 		if (uc == u) { \
@@ -188,13 +188,13 @@ void ast_unregister_atexit(void (*func)(void));
 	}\
 	free(u); \
 	localusecnt--; \
-	pthread_mutex_unlock(&localuser_lock); \
+	ast_mutex_unlock(&localuser_lock); \
 	ast_update_use_count(); \
 }
 
 #define STANDARD_HANGUP_LOCALUSERS { \
 	struct localuser *u, *ul; \
-	pthread_mutex_lock(&localuser_lock); \
+	ast_mutex_lock(&localuser_lock); \
 	u = localusers; \
 	while(u) { \
 		ast_softhangup(u->chan, AST_SOFTHANGUP_APPUNLOAD); \
@@ -202,14 +202,14 @@ void ast_unregister_atexit(void (*func)(void));
 		u = u->next; \
 		free(ul); \
 	} \
-	pthread_mutex_unlock(&localuser_lock); \
+	ast_mutex_unlock(&localuser_lock); \
 	localusecnt=0; \
 }
 
 #define STANDARD_USECOUNT(res) { \
-	pthread_mutex_lock(&localuser_lock); \
+	ast_mutex_lock(&localuser_lock); \
 	res = localusecnt; \
-	pthread_mutex_unlock(&localuser_lock); \
+	ast_mutex_unlock(&localuser_lock); \
 }