add 'show threads' and 'show profile' commands.

These are momstly debugging tools for developers,
a bit documented in the header files (utils.h),
although more documentation is definitely necessary.

The performance impact is close to zero(*) so there is no
need to compile it conditionally.
(*) not completely true - thread destruction still needs
to search a list _but_ this can be easily optimized if we
end up with hundreds of active threads (in which case, though,
the problem is clearly elsewhere).



git-svn-id: http://svn.digium.com/svn/asterisk/trunk@19544 f38db490-d61c-443f-a65b-d21fe96a405b

7 files changed, 354 insertions, 4 deletions

diff --git a/asterisk.c b/asterisk.c
index 5ce65ef2f..6d995323b 100644
--- a/asterisk.c
+++ b/asterisk.c
@@ -74,6 +74,9 @@
 #include <grp.h>
 #include <pwd.h>
 #include <sys/stat.h>
+#ifdef linux
+#include <sys/prctl.h>
+#endif
 #include <regex.h>
 
 #ifdef linux
@@ -272,6 +275,208 @@ void ast_unregister_file_version(const char *file)
 		free(find);
 }
 
+struct thread_list_t {
+	AST_LIST_ENTRY(thread_list_t) list;
+	char *name;
+	pthread_t id;
+};
+
+static AST_LIST_HEAD_STATIC(thread_list, thread_list_t);
+
+static char show_threads_help[] =
+"Usage: show threads\n"
+"       List threads currently active in the system.\n";
+
+void ast_register_thread(char *name)
+{ 
+	struct thread_list_t *new = ast_calloc(1, sizeof(*new));
+
+	if (!new)
+		return;
+	new->id = pthread_self();
+	new->name = name; /* this was a copy already */
+	AST_LIST_LOCK(&thread_list);
+	AST_LIST_INSERT_HEAD(&thread_list, new, list);
+	AST_LIST_UNLOCK(&thread_list);
+}
+
+void ast_unregister_thread(void *id)
+{
+	struct thread_list_t *x;
+
+	AST_LIST_LOCK(&thread_list);
+	AST_LIST_TRAVERSE_SAFE_BEGIN(&thread_list, x, list) {
+		if ((void *)x->id == id) {
+			AST_LIST_REMOVE_CURRENT(&thread_list, list);
+			break;
+		}
+	}
+	AST_LIST_TRAVERSE_SAFE_END;
+	AST_LIST_UNLOCK(&thread_list);
+	if (x) {
+		free(x->name);
+		free(x);
+	}
+}
+
+static int handle_show_threads(int fd, int argc, char *argv[])
+{
+	int count = 0;
+	struct thread_list_t *cur;
+
+	AST_LIST_LOCK(&thread_list);
+	AST_LIST_TRAVERSE(&thread_list, cur, list) {
+		ast_cli(fd, "%p %s\n", (void *)cur->id, cur->name);
+		count++;
+	}
+        AST_LIST_UNLOCK(&thread_list);
+	ast_cli(fd, "%d threads listed.\n", count);
+	return 0;
+}
+
+struct profile_entry {
+	const char *name;
+	uint64_t	scale;	/* if non-zero, values are scaled by this */
+	int64_t	mark;
+	int64_t	value;
+	int64_t	events;
+};
+
+struct profile_data {
+	int entries;
+	int max_size;
+	struct profile_entry e[0];
+};
+
+static struct profile_data *prof_data;
+
+/*
+ * allocates a counter with a given name and scale.
+ * Returns the identifier of the counter.
+ */
+int ast_add_profile(const char *name, uint64_t scale)
+{
+	int l = sizeof(struct profile_data);
+	int n = 10;	/* default entries */
+
+	if (prof_data == NULL) {
+		prof_data = ast_calloc(1, l + n*sizeof(struct profile_entry));
+		if (prof_data == NULL)
+			return -1;
+		prof_data->entries = 0;
+		prof_data->max_size = n;
+	}
+	if (prof_data->entries >= prof_data->max_size) {
+		void *p;
+		n = prof_data->max_size + 20;
+		p = ast_realloc(prof_data, l + n*sizeof(struct profile_entry));
+		if (p == NULL)
+			return -1;
+		prof_data = p;
+		prof_data->max_size = n;
+	}
+	n = prof_data->entries++;
+	prof_data->e[n].name = ast_strdup(name);
+	prof_data->e[n].value = 0;
+	prof_data->e[n].events = 0;
+	prof_data->e[n].mark = 0;
+	prof_data->e[n].scale = scale;
+	return n;
+}
+
+int64_t ast_profile(int i, int64_t delta)
+{
+	if (!prof_data || i < 0 || i > prof_data->entries)	/* invalid index */
+		return 0;
+	if (prof_data->e[i].scale > 1)
+		delta /= prof_data->e[i].scale;
+	prof_data->e[i].value += delta;
+	prof_data->e[i].events++;
+	return prof_data->e[i].value;
+}
+
+#if defined ( __i386__) && (defined(__FreeBSD__) || defined(linux))
+#if defined(__FreeBSD__)
+#include <machine/cpufunc.h>
+#elif defined(linux)
+static __inline u_int64_t
+rdtsc(void)
+{ 
+	uint64_t rv;
+
+	__asm __volatile(".byte 0x0f, 0x31" : "=A" (rv));
+	return (rv);
+}
+#endif
+#else	/* supply a dummy function on other platforms */
+xxx
+static __inline u_int64_t
+rdtsc(void)
+{
+	return 0;
+}
+#endif
+
+int64_t ast_mark(int i, int startstop)
+{
+	if (!prof_data || i < 0 || i > prof_data->entries) /* invalid index */
+		return 0;
+	if (startstop == 1)
+		prof_data->e[i].mark = rdtsc();
+	else {
+		prof_data->e[i].mark = (rdtsc() - prof_data->e[i].mark);
+		if (prof_data->e[i].scale > 1)
+			prof_data->e[i].mark /= prof_data->e[i].scale;
+		prof_data->e[i].value += prof_data->e[i].mark;
+		prof_data->e[i].events++;
+	}
+	return prof_data->e[i].mark;
+}
+
+static int handle_show_profile(int fd, int argc, char *argv[])
+{
+	int i, min, max;
+	char *search = NULL;
+
+	if (prof_data == NULL)
+		return 0;
+
+	min = 0;
+	max = prof_data->entries;
+	if  (argc >= 3) { /* specific entries */
+		if (isdigit(argv[2][0])) {
+			min = atoi(argv[2]);
+			if (argc == 4 && strcmp(argv[3], "-"))
+				max = atoi(argv[3]);
+		} else
+			search = argv[2];
+	}
+	if (max > prof_data->entries)
+		max = prof_data->entries;
+	if (!strcmp(argv[0], "clear")) {
+		for (i= min; i < max; i++) {
+			if (!search || strstr(prof_data->e[i].name, search)) {
+				prof_data->e[i].value = 0;
+				prof_data->e[i].events = 0;
+			}
+		}
+		return 0;
+	}
+	ast_cli(fd, "profile values (%d, allocated %d)\n-------------------\n",
+		prof_data->entries, prof_data->max_size);
+	for (i = min; i < max; i++) {
+		struct profile_entry *e = &prof_data->e[i];
+		if (!search || strstr(prof_data->e[i].name, search))
+		    ast_cli(fd, "%6d: [%8ld] %10ld %12lld %12lld %s\n",
+			i,
+			(long)e->scale,
+			(long)e->events, (long long)e->value,
+			(long long)(e->events ? e->value / e->events : e->value),
+			e->name);
+	}
+	return 0;
+}
+
 static char show_version_files_help[] = 
 "Usage: show version files [like <pattern>]\n"
 "       Shows the revision numbers of the files used to build this copy of Asterisk.\n"
@@ -1231,6 +1436,12 @@ static struct ast_cli_entry core_cli[] = {
 #if !defined(LOW_MEMORY)
 	{ { "show", "version", "files", NULL }, handle_show_version_files,
 	  "Show versions of files used to build Asterisk", show_version_files_help, complete_show_version_files },
+	{ { "show", "threads", NULL }, handle_show_threads,
+	  "Show running threads", show_threads_help, NULL },
+	{ { "show", "profile", NULL }, handle_show_profile,
+	  "Show profiling info"},
+	{ { "clear", "profile", NULL }, handle_show_profile,
+	  "Clear profiling info"},
 #endif /* ! LOW_MEMORY */
 };
 
diff --git a/include/asterisk.h b/include/asterisk.h
index 38b0f67ab..7e63f608c 100644
--- a/include/asterisk.h
+++ b/include/asterisk.h
@@ -15,6 +15,8 @@
  * \brief Asterisk main include file. File version handling, generic pbx functions.
  */
 
+#include "asterisk/compat.h"
+
 #ifndef _ASTERISK_H
 #define _ASTERISK_H
 
@@ -108,6 +110,24 @@ void ast_register_file_version(const char *file, const char *version);
 void ast_unregister_file_version(const char *file);
 
 /*!
+ * \brief support for event profiling
+ * (note, this must be documented a lot more)
+ * ast_add_profile allocates a generic 'counter' with a given name,
+ * which can be shown with the command 'show profile <name>'
+ *
+ * The counter accumulates positive or negative values supplied by
+ * ast_add_profile(), dividing them by the 'scale' value passed in the
+ * create call, and also counts the number of 'events'.
+ * Values can also be taked by the TSC counter on ia32 architectures,
+ * in which case you can mark the start of an event calling ast_mark(id, 1)
+ * and then the end of the event with ast_mark(id, 0).
+ * For non-i386 architectures, these two calls return 0.
+ */
+int ast_add_profile(const char *, uint64_t scale);
+int64_t ast_profile(int, int64_t);
+int64_t ast_mark(int, int start1_stop0);
+
+/*!
  * \brief Register/unregister a source code file with the core.
  * \param file the source file name
  * \param version the version string (typically a CVS revision keyword string)
@@ -129,6 +149,20 @@ void ast_unregister_file_version(const char *file);
  * revision number.
  */
 #if defined(__GNUC__) && !defined(LOW_MEMORY)
+#ifdef MTX_PROFILE
+#define	HAVE_MTX_PROFILE	/* used in lock.h */
+#define ASTERISK_FILE_VERSION(file, version) \
+	static int mtx_prof = -1;       /* profile mutex */	\
+	static void __attribute__((constructor)) __register_file_version(void) \
+	{ \
+		mtx_prof = ast_add_profile("mtx_lock_" file, 0);	\
+		ast_register_file_version(file, version); \
+	} \
+	static void __attribute__((destructor)) __unregister_file_version(void) \
+	{ \
+		ast_unregister_file_version(file); \
+	}
+#else
 #define ASTERISK_FILE_VERSION(file, version) \
 	static void __attribute__((constructor)) __register_file_version(void) \
 	{ \
@@ -138,6 +172,7 @@ void ast_unregister_file_version(const char *file);
 	{ \
 		ast_unregister_file_version(file); \
 	}
+#endif
 #elif !defined(LOW_MEMORY) /* ! __GNUC__  && ! LOW_MEMORY*/
 #define ASTERISK_FILE_VERSION(file, x) static const char __file_version[] = x;
 #else /* LOW_MEMORY */
diff --git a/include/asterisk/compat.h b/include/asterisk/compat.h
index 4ed0b7a5d..79d23e7e7 100644
--- a/include/asterisk/compat.h
+++ b/include/asterisk/compat.h
@@ -80,6 +80,7 @@ int unsetenv(const char *name);
 #endif
 
 #ifdef __linux__
+#include <inttypes.h>
 #define HAVE_STRCASESTR
 #define HAVE_STRNDUP
 #define HAVE_STRNLEN
diff --git a/include/asterisk/lock.h b/include/asterisk/lock.h
index f0192e7ba..e86462306 100644
--- a/include/asterisk/lock.h
+++ b/include/asterisk/lock.h
@@ -56,6 +56,23 @@
 #ifndef _ASTERISK_LOCK_H
 #define _ASTERISK_LOCK_H
 
+/* internal macro to profile mutexes. Only computes the delay on
+ * non-blocking calls.
+ */
+#ifndef	HAVE_MTX_PROFILE
+#define	__MTX_PROF	/* nothing */
+#else
+#define	__MTX_PROF	{			\
+	int i;					\
+	/* profile only non-blocking events */	\
+	ast_mark(mtx_prof, 1);			\
+	i = pthread_mutex_trylock(pmutex);	\
+	ast_mark(mtx_prof, 0);			\
+	if (!i)					\
+		return i;			\
+	}
+#endif	/* HAVE_MTX_PROFILE */
+
 #include <pthread.h>
 #include <netdb.h>
 #include <time.h>
@@ -75,7 +92,7 @@
 #endif
 
 #ifdef BSD
-#ifdef __GNUC__
+#if 0 && defined( __GNUC__)
 #define AST_MUTEX_INIT_W_CONSTRUCTORS
 #else
 #define AST_MUTEX_INIT_ON_FIRST_USE
@@ -264,7 +281,13 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 		time_t seconds = time(NULL);
 		time_t current;
 		do {
+#ifdef	HAVE_MTX_PROFILE
+			ast_mark(mtx_prof, 1);
+#endif
 			res = pthread_mutex_trylock(&t->mutex);
+#ifdef	HAVE_MTX_PROFILE
+			ast_mark(mtx_prof, 0);
+#endif
 			if (res == EBUSY) {
 				current = time(NULL);
 				if ((current - seconds) && (!((current - seconds) % 5))) {
@@ -279,6 +302,12 @@ static inline int __ast_pthread_mutex_lock(const char *filename, int lineno, con
 		} while (res == EBUSY);
 	}
 #else
+#ifdef	HAVE_MTX_PROFILE
+	ast_mark(mtx_prof, 1);
+	res = pthread_mutex_trylock(&t->mutex);
+	ast_mark(mtx_prof, 0);
+	if (res)
+#endif
 	res = pthread_mutex_lock(&t->mutex);
 #endif /* DETECT_DEADLOCKS */
 
@@ -581,6 +610,7 @@ static void  __attribute__ ((destructor)) fini_##mutex(void) \
 
 static inline int ast_mutex_lock(ast_mutex_t *pmutex)
 {
+	__MTX_PROF
 	return pthread_mutex_lock(pmutex);
 }
 
@@ -601,8 +631,10 @@ static inline int ast_mutex_lock(ast_mutex_t *pmutex)
 {
 	if (*pmutex == (ast_mutex_t)AST_MUTEX_KIND)
 		ast_mutex_init(pmutex);
+	__MTX_PROF
 	return pthread_mutex_lock(pmutex);
 }
+
 static inline int ast_mutex_trylock(ast_mutex_t *pmutex)
 {
 	if (*pmutex == (ast_mutex_t)AST_MUTEX_KIND)
@@ -616,6 +648,7 @@ static inline int ast_mutex_trylock(ast_mutex_t *pmutex)
 
 static inline int ast_mutex_lock(ast_mutex_t *pmutex)
 {
+	__MTX_PROF
 	return pthread_mutex_lock(pmutex);
 }
 
diff --git a/include/asterisk/utils.h b/include/asterisk/utils.h
index c702fa6fb..35f2539ef 100644
--- a/include/asterisk/utils.h
+++ b/include/asterisk/utils.h
@@ -225,8 +225,14 @@ static force_inline int inaddrcmp(const struct sockaddr_in *sin1, const struct s
 }
 
 #define AST_STACKSIZE 256 * 1024
-#define ast_pthread_create(a,b,c,d) ast_pthread_create_stack(a,b,c,d,0)
-int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine)(void *), void *data, size_t stacksize);
+
+void ast_register_thread(char *name);
+void ast_unregister_thread(void *id);
+
+#define ast_pthread_create(a,b,c,d) ast_pthread_create_stack(a,b,c,d,0, \
+	 __FILE__, __FUNCTION__, __LINE__, #c)
+int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine)(void *), void *data, size_t stacksize,
+	const char *file, const char *caller, int line, const char *start_fn);
 
 /*!
 	\brief Process a string to find and replace characters
diff --git a/utils.c b/utils.c
index ae4023538..3bd3623d3 100644
--- a/utils.c
+++ b/utils.c
@@ -509,8 +509,42 @@ int ast_utils_init(void)
 #undef pthread_create /* For ast_pthread_create function only */
 #endif /* !__linux__ */
 
-int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine)(void *), void *data, size_t stacksize)
+/*
+ * support for 'show threads'. The start routine is wrapped by
+ * dummy_start(), so that ast_register_thread() and
+ * ast_unregister_thread() know the thread identifier.
+ */
+struct thr_arg {
+	void *(*start_routine)(void *);
+	void *data;
+	char *name;
+};
+
+/*
+ * on OS/X, pthread_cleanup_push() and pthread_cleanup_pop()
+ * are odd macros which start and end a block, so they _must_ be
+ * used in pairs (the latter with a '1' argument to call the
+ * handler on exit.
+ * On BSD we don't need this, but we keep it for compatibility with the MAC.
+ */
+static void *dummy_start(void *data)
 {
+	void *ret;
+	struct thr_arg a = *((struct thr_arg *)data);	/* make a local copy */
+
+	free(data);
+	ast_register_thread(a.name);
+	pthread_cleanup_push(ast_unregister_thread, (void *)pthread_self());	/* on unregister */
+	ret = a.start_routine(a.data);
+	pthread_cleanup_pop(1);
+	return ret;
+}
+
+int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine)(void *), void *data, size_t stacksize,
+	const char *file, const char *caller, int line, const char *start_fn)
+{
+	struct thr_arg *a;
+
 	pthread_attr_t lattr;
 	if (!attr) {
 		pthread_attr_init(&lattr);
@@ -534,6 +568,17 @@ int ast_pthread_create_stack(pthread_t *thread, pthread_attr_t *attr, void *(*st
 	errno = pthread_attr_setstacksize(attr, stacksize);
 	if (errno)
 		ast_log(LOG_WARNING, "pthread_attr_setstacksize returned non-zero: %s\n", strerror(errno));
+	a = ast_malloc(sizeof(*a));
+	if (!a)
+		ast_log(LOG_WARNING, "no memory, thread %s will not be listed\n", start_fn);
+	else {	/* remap parameters */
+		a->start_routine = start_routine;
+		a->data = data;
+		start_routine = dummy_start;
+		asprintf(&a->name, "%-20s started at [%5d] %s %s()",
+			start_fn, line, file, caller);
+		data = a;
+	}
 	return pthread_create(thread, attr, start_routine, data); /* We're in ast_pthread_create, so it's okay */
 }
 
diff --git a/utils/astman.c b/utils/astman.c
index d1631ed14..427b581b2 100644
--- a/utils/astman.c
+++ b/utils/astman.c
@@ -84,6 +84,25 @@ void ast_unregister_file_version(const char *file)
 {
 }
 
+int ast_add_profile(const char *, uint64_t scale);
+int ast_add_profile(const char *s, uint64_t scale)
+{
+	return -1;
+}
+
+int64_t ast_profile(int, int64_t);
+int64_t ast_profile(int key, int64_t val)
+{
+	return 0;
+}
+int64_t ast_mark(int, int start1_stop0);
+int64_t ast_mark(int key, int start1_stop0)
+{
+	return 0;
+}
+
+/* end of dummy functions */
+
 static struct ast_chan *find_chan(char *name)
 {
 	struct ast_chan *chan;