/****************************/

/* THIS IS OPEN SOURCE CODE */

/****************************/

/*

* File:    threads.c

* Author:  Philip Mucci

*          mucci@cs.utk.edu

* Mods:    Kevin London

*          london@cs.utk.edu

*/

/* This file contains thread allocation and bookkeeping functions */

#include "papi.h"

#include "papi_internal.h"

#include "papi_vector.h"

#include "papi_memory.h"

#include <string.h>

#include <unistd.h>

/*****************/

/* BEGIN GLOBALS */

/*****************/

/* The following globals get initialized and cleared by:

   extern int _papi_hwi_init_global_threads(void);

   extern int _papi_hwi_shutdown_thread(ThreadInfo_t *thread); */

/* list of threads, gets initialized to master process with TID of getpid() */

volatile ThreadInfo_t *_papi_hwi_thread_head;

/* If we have TLS, this variable ALWAYS points to our thread descriptor. It's like magic! */

#if defined(HAVE_THREAD_LOCAL_STORAGE)

THREAD_LOCAL_STORAGE_KEYWORD ThreadInfo_t *_papi_hwi_my_thread;

#endif

/* Function that returns and unsigned long thread identifier */

unsigned long ( *_papi_hwi_thread_id_fn ) ( void );

/* Function that sends a signal to other threads */

#ifdef ANY_THREAD_GETS_SIGNAL

int ( *_papi_hwi_thread_kill_fn ) ( int, int );

#endif

/*****************/

/*  END  GLOBALS */

/*****************/

static int

lookup_and_set_thread_symbols( void )

{

#if defined(ANY_THREAD_GETS_SIGNAL)

	int retval;

	char *error_ptc = NULL, *error_ptk = NULL;

	void *symbol_ptc = NULL, *symbol_ptk = NULL, *handle = NULL;

	handle = dlopen( NULL, RTLD_LAZY );

	if ( handle == NULL ) {

		PAPIERROR( "Error from dlopen(NULL, RTLD_LAZY): %d %s", errno,

				   dlerror(  ) );

		return ( PAPI_ESYS );

	}

	symbol_ptc = dlsym( handle, "pthread_self" );

	if ( symbol_ptc == NULL ) {

		error_ptc = dlerror(  );

		THRDBG( "dlsym(%p,pthread_self) returned NULL: %s\n",

				( error_ptc ? error_ptc : "No error, NULL symbol!" ) );

	}

	symbol_ptk = dlsym( handle, "pthread_kill" );

	if ( symbol_ptk == NULL ) {

		error_ptk = dlerror(  );

		THRDBG( "dlsym(%p,pthread_kill) returned NULL: %s\n",

				( error_ptk ? error_ptk : "No error, NULL symbol!" ) );

	}

	dlclose( handle );

	if ( !( ( _papi_hwi_thread_kill_fn && _papi_hwi_thread_id_fn ) ||

			( !_papi_hwi_thread_kill_fn && !_papi_hwi_thread_id_fn ) ) )

		return ( PAPI_EMISC );

	_papi_hwi_thread_kill_fn = ( int ( * )( int, int ) ) symbol_ptk;

	_papi_hwi_thread_id_fn = ( unsigned long ( * )( void ) ) symbol_ptc;

#endif

	return ( PAPI_OK );

}

static ThreadInfo_t *

allocate_thread( int tid )

{

	ThreadInfo_t *thread;

	int i;

	/* The Thread EventSet is special. It is not in the EventSet list, but is pointed

	   to by each EventSet of that particular thread. */

	thread = ( ThreadInfo_t * ) papi_malloc( sizeof ( ThreadInfo_t ) );

	if ( thread == NULL )

		return ( NULL );

	memset( thread, 0x00, sizeof ( ThreadInfo_t ) );

	thread->context =

		( hwd_context_t ** ) papi_malloc( sizeof ( hwd_context_t * ) *

										  ( size_t ) papi_num_components );

	if ( !thread->context ) {

		papi_free( thread );

		return ( NULL );

	}

	thread->running_eventset =

		( EventSetInfo_t ** ) papi_malloc( sizeof ( EventSetInfo_t * ) *

										   ( size_t ) papi_num_components );

	if ( !thread->running_eventset ) {

		papi_free( thread->context );

		papi_free( thread );

		return ( NULL );

	}

	for ( i = 0; i < papi_num_components; i++ ) {

		thread->context[i] =

			( void * ) papi_malloc( ( size_t ) _papi_hwd[i]->size.context );

		thread->running_eventset[i] = NULL;

		if ( thread->context[i] == NULL ) {

			for ( i--; i >= 0; i-- )

				papi_free( thread->context[i] );

			papi_free( thread->context );

			papi_free( thread );

			return ( NULL );

		}

		memset( thread->context[i], 0x00,

				( size_t ) _papi_hwd[i]->size.context );

	}

	if ( _papi_hwi_thread_id_fn ) {

           thread->tid = ( *_papi_hwi_thread_id_fn ) (  );

	}

	else {

	   thread->tid = ( unsigned long ) getpid(  );

	}

	thread->allocator_tid=thread->tid;

	if (tid == 0 ) {

	}

	else {

	  thread->tid=tid;

	}

	THRDBG( "Allocated thread %ld at %p, allocator: %ld\n", thread->tid, 

		thread,

		thread->allocator_tid );

	return thread;

}

static void

free_thread( ThreadInfo_t ** thread )

{

	int i;

	THRDBG( "Freeing thread %ld at %p\n", ( *thread )->tid, *thread );

	for ( i = 0; i < papi_num_components; i++ ) {

		if ( ( *thread )->context[i] )

			papi_free( ( *thread )->context[i] );

	}

	if ( ( *thread )->context )

		papi_free( ( *thread )->context );

	if ( ( *thread )->running_eventset )

		papi_free( ( *thread )->running_eventset );

	memset( *thread, 0x00, sizeof ( ThreadInfo_t ) );

	papi_free( *thread );

	*thread = NULL;

}

static void

insert_thread( ThreadInfo_t * entry, int tid )

{

	_papi_hwi_lock( THREADS_LOCK );

	if ( _papi_hwi_thread_head == NULL ) {	/* 0 elements */

		THRDBG( "_papi_hwi_thread_head is NULL\n" );

		entry->next = entry;

	} else if ( _papi_hwi_thread_head->next == _papi_hwi_thread_head ) {	/* 1 elements */

		THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",

				_papi_hwi_thread_head->tid, _papi_hwi_thread_head );

		_papi_hwi_thread_head->next = entry;

		entry->next = ( ThreadInfo_t * ) _papi_hwi_thread_head;

	} else {				 /* 2+ elements */

		THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",

				_papi_hwi_thread_head->tid, _papi_hwi_thread_head );

		entry->next = _papi_hwi_thread_head->next;

		_papi_hwi_thread_head->next = entry;

	}

	_papi_hwi_thread_head = entry;

	THRDBG( "_papi_hwi_thread_head now thread %ld at %p\n",

			_papi_hwi_thread_head->tid, _papi_hwi_thread_head );

	_papi_hwi_unlock( THREADS_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)

	/* Don't set the current local thread if we are a fake attach thread */

        if (tid==0) {

	   _papi_hwi_my_thread = entry;

	   THRDBG( "TLS for thread %ld is now %p\n", entry->tid,

			_papi_hwi_my_thread );

	}

#else

	( void ) tid;

#endif

}

static int

remove_thread( ThreadInfo_t * entry )

{

	ThreadInfo_t *tmp = NULL, *prev = NULL;

	_papi_hwi_lock( THREADS_LOCK );

	THRDBG( "_papi_hwi_thread_head was thread %ld at %p\n",

			_papi_hwi_thread_head->tid, _papi_hwi_thread_head );

	/* Find the preceding element and the matched element,

	   short circuit if we've seen the head twice */

	for ( tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;

		  ( entry != tmp ) || ( prev == NULL ); tmp = tmp->next ) {

		prev = tmp;

	}

	if ( tmp != entry ) {

		THRDBG( "Thread %ld at %p was not found in the thread list!\n",

				entry->tid, entry );

		return ( PAPI_EBUG );

	}

	/* Only 1 element in list */

	if ( prev == tmp ) {

		_papi_hwi_thread_head = NULL;

		tmp->next = NULL;

		THRDBG( "_papi_hwi_thread_head now NULL\n" );

	} else {

		prev->next = tmp->next;

		/* If we're removing the head, better advance it! */

		if ( _papi_hwi_thread_head == tmp ) {

			_papi_hwi_thread_head = tmp->next;

			THRDBG( "_papi_hwi_thread_head now thread %ld at %p\n",

					_papi_hwi_thread_head->tid, _papi_hwi_thread_head );

		}

		THRDBG( "Removed thread %p from list\n", tmp );

	}

	_papi_hwi_unlock( THREADS_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)

	_papi_hwi_my_thread = NULL;

	THRDBG( "TLS for thread %ld is now %p\n", entry->tid,

			_papi_hwi_my_thread );

#endif

	return PAPI_OK;

}

int

_papi_hwi_initialize_thread( ThreadInfo_t ** dest, int tid )

{

	int retval;

	ThreadInfo_t *thread;

	int i;

	if ( ( thread = allocate_thread( tid  ) ) == NULL ) {

		*dest = NULL;

		return PAPI_ENOMEM;

	}

	/* Call the component to fill in anything special. */

	for ( i = 0; i < papi_num_components; i++ ) {

	    if (_papi_hwd[i]->cmp_info.disabled) continue;

	    retval = _papi_hwd[i]->init_thread( thread->context[i] );

	    if ( retval ) {

	       free_thread( &thread );

	       *dest = NULL;

	       return retval;

	    }

	}

	insert_thread( thread, tid );

	*dest = thread;

	return PAPI_OK;

}

#if defined(ANY_THREAD_GETS_SIGNAL)

/* This is ONLY defined for systems that enable ANY_THREAD_GETS_SIGNAL

   since we must forward signals sent to non-PAPI threads.

   This is NOT compatible with thread local storage, since to broadcast

   the signal, we need a list of threads. */

int

_papi_hwi_broadcast_signal( unsigned int mytid )

{

	int i, retval, didsomething = 0;

	volatile ThreadInfo_t *foo = NULL;

	_papi_hwi_lock( THREADS_LOCK );

	for ( foo = _papi_hwi_thread_head; foo != NULL; foo = foo->next ) {

		/* xxxx Should this be hardcoded to index 0 or walk the list or what? */

		for ( i = 0; i < papi_num_components; i++ ) {

			if ( ( foo->tid != mytid ) && ( foo->running_eventset[i] ) &&

				 ( foo->running_eventset[i]->

				   state & ( PAPI_OVERFLOWING | PAPI_MULTIPLEXING ) ) ) {

				/* xxxx mpx_info inside _papi_mdi_t _papi_hwi_system_info is commented out.

				   See papi_internal.h for details. The multiplex_timer_sig value is now part of that structure */

			  THRDBG("Thread %ld sending signal %d to thread %ld\n",mytid,foo->tid,

				  (foo->running_eventset[i]->state & PAPI_OVERFLOWING ? _papi_hwd[i]->cmp_info.hardware_intr_sig : _papi_os_info.itimer_sig));

			  retval = (*_papi_hwi_thread_kill_fn)(foo->tid, 

				  (foo->running_eventset[i]->state & PAPI_OVERFLOWING ? _papi_hwd[i]->cmp_info.hardware_intr_sig : _papi_os_info.itimer_sig));

			  if (retval != 0)

				return(PAPI_EMISC);

			}

		}

		if ( foo->next == _papi_hwi_thread_head )

		  break;

	}

	_papi_hwi_unlock( THREADS_LOCK );

	return ( PAPI_OK );

}

#endif

/* This is undefined for systems that enable ANY_THREAD_GETS_SIGNAL

   since we always must enable threads for safety. */

int

_papi_hwi_set_thread_id_fn( unsigned long ( *id_fn ) ( void ) )

{

#if !defined(ANY_THREAD_GETS_SIGNAL)

	/* Check for multiple threads still in the list, if so, we can't change it */

	if ( _papi_hwi_thread_head->next != _papi_hwi_thread_head )

		return ( PAPI_EINVAL );

	/* We can't change the thread id function from one to another, 

	   only NULL to non-NULL and vice versa. */

	if ( ( id_fn != NULL ) && ( _papi_hwi_thread_id_fn != NULL ) )

		return ( PAPI_EINVAL );

	_papi_hwi_thread_id_fn = id_fn;

	THRDBG( "Set new thread id function to %p\n", id_fn );

	if ( id_fn )

		_papi_hwi_thread_head->tid = ( *_papi_hwi_thread_id_fn ) (  );

	else

		_papi_hwi_thread_head->tid = ( unsigned long ) getpid(  );

	THRDBG( "New master tid is %ld\n", _papi_hwi_thread_head->tid );

#else

	THRDBG( "Skipping set of thread id function\n" );

#endif

	return PAPI_OK;

}

static int _papi_hwi_thread_free_eventsets(long tid) {

   EventSetInfo_t *ESI;

   ThreadInfo_t *master;

   DynamicArray_t *map = &_papi_hwi_system_info.global_eventset_map;

   int i;

   master = _papi_hwi_lookup_thread( tid );

   _papi_hwi_lock( INTERNAL_LOCK );

   for( i = 0; i < map->totalSlots; i++ ) {

      ESI = map->dataSlotArray[i];

      if ( ( ESI ) && (ESI->master!=NULL) ) {

	 if ( ESI->master == master ) {

	    THRDBG("Attempting to remove %d from tid %ld\n",ESI->EventSetIndex,tid);

	    /* Code copied from _papi_hwi_remove_EventSet(ESI);      */

	    _papi_hwi_free_EventSet( ESI );

	    map->dataSlotArray[i] = NULL;

	    map->availSlots++;

	    map->fullSlots--;

	 }

      }

   }

   _papi_hwi_unlock( INTERNAL_LOCK );

   return PAPI_OK;

}

int

_papi_hwi_shutdown_thread( ThreadInfo_t * thread, int force_shutdown )

{

	int retval = PAPI_OK;

	unsigned long tid;

	int i, failure = 0;

	if ( _papi_hwi_thread_id_fn )

		tid = ( *_papi_hwi_thread_id_fn ) (  );

	else

		tid = ( unsigned long ) getpid(  );

        THRDBG("Want to shutdown thread %ld, alloc %ld, our_tid: %ld\n",

	       thread->tid,

	       thread->allocator_tid,

	       tid);

	if ((thread->tid==tid) || ( thread->allocator_tid == tid ) || force_shutdown) {

                _papi_hwi_thread_free_eventsets(tid);

		remove_thread( thread );

		THRDBG( "Shutting down thread %ld at %p\n", thread->tid, thread );

		for( i = 0; i < papi_num_components; i++ ) {

		   if (_papi_hwd[i]->cmp_info.disabled) continue;

		   retval = _papi_hwd[i]->shutdown_thread( thread->context[i]);

		   if ( retval != PAPI_OK ) failure = retval;

		}

		free_thread( &thread );

		return ( failure );

	}

	THRDBG( "Skipping shutdown thread %ld at %p, thread %ld not allocator!\n",

			thread->tid, thread, tid );

	return PAPI_EBUG;

}

/* THESE MUST BE CALLED WITH A GLOBAL LOCK */

int

_papi_hwi_shutdown_global_threads( void )

{

        int err,num_threads,i;

	ThreadInfo_t *tmp,*next;

	unsigned long our_tid;

	tmp = _papi_hwi_lookup_thread( 0 );

	if ( tmp == NULL ) {

	   THRDBG( "Did not find my thread for shutdown!\n" );

	   err = PAPI_EBUG;

	}

	else {

	   our_tid=tmp->tid;

	   (void)our_tid;

	   THRDBG("Shutting down %ld\n",our_tid);

	   err = _papi_hwi_shutdown_thread( tmp, 1 );

	   /* count threads */

	   tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;

	   num_threads=0;

	   while(tmp!=NULL) {

	      num_threads++;

	      if (tmp->next==_papi_hwi_thread_head) break;

	      tmp=tmp->next;

	   }

	   /* Shut down all threads allocated by this thread */

	   /* Urgh it's a circular list where we removed in the loop  */

	   /* so the only sane way to do it is get a count in advance */

	   tmp = ( ThreadInfo_t * ) _papi_hwi_thread_head;

	   for(i=0;i

	      next=tmp->next;

	      THRDBG("looking at #%d %ld our_tid: %ld alloc_tid: %ld\n",

		     i,tmp->tid,our_tid,tmp->allocator_tid);

		 THRDBG("Also removing thread %ld\n",tmp->tid);

	         err = _papi_hwi_shutdown_thread( tmp, 1 );

	      tmp=next;

	   }

	}

#ifdef DEBUG

	if ( ISLEVEL( DEBUG_THREADS ) ) {

		if ( _papi_hwi_thread_head ) {

			THRDBG( "Thread head %p still exists!\n", _papi_hwi_thread_head );

		}

	}

#endif

#if defined(HAVE_THREAD_LOCAL_STORAGE)

	_papi_hwi_my_thread = NULL;

#endif

	_papi_hwi_thread_head = NULL;

	_papi_hwi_thread_id_fn = NULL;

#if defined(ANY_THREAD_GETS_SIGNAL)

	_papi_hwi_thread_kill_fn = NULL;

#endif

	return err;

}

int

_papi_hwi_init_global_threads( void )

{

	int retval;

	ThreadInfo_t *tmp;

	_papi_hwi_lock( GLOBAL_LOCK );

#if defined(HAVE_THREAD_LOCAL_STORAGE)

	_papi_hwi_my_thread = NULL;

#endif

	_papi_hwi_thread_head = NULL;

	_papi_hwi_thread_id_fn = NULL;

#if defined(ANY_THREAD_GETS_SIGNAL)

	_papi_hwi_thread_kill_fn = NULL;

#endif

	retval = _papi_hwi_initialize_thread( &tmp , 0);

	if ( retval == PAPI_OK ) {

	   retval = lookup_and_set_thread_symbols(  );

	}

	_papi_hwi_unlock( GLOBAL_LOCK );

	return ( retval );

}

int

_papi_hwi_gather_all_thrspec_data( int tag, PAPI_all_thr_spec_t * where )

{

	int didsomething = 0;

	ThreadInfo_t *foo = NULL;

	_papi_hwi_lock( THREADS_LOCK );

	for ( foo = ( ThreadInfo_t * ) _papi_hwi_thread_head; foo != NULL;

		  foo = foo->next ) {

		/* If we want thread ID's */

		if ( where->id )

			memcpy( &where->id[didsomething], &foo->tid,

					sizeof ( where->id[didsomething] ) );

		/* If we want data pointers */

		if ( where->data )

			where->data[didsomething] = foo->thread_storage[tag];

		didsomething++;

		if ( ( where->id ) || ( where->data ) ) {

			if ( didsomething >= where->num )

				break;

		}

		if ( foo->next == _papi_hwi_thread_head )

			break;

	}

	where->num = didsomething;

	_papi_hwi_unlock( THREADS_LOCK );

	return ( PAPI_OK );

}