/*

* File:    perf_event.c

*

* Author:  Corey Ashford

*          cjashfor@us.ibm.com

*          - based upon perfmon.c written by -

*          Philip Mucci

*          mucci@cs.utk.edu

* Mods:    Gary Mohr

*          gary.mohr@bull.com

* Mods:    Vince Weaver

*          vweaver1@eecs.utk.edu

* Mods:	   Philip Mucci

*	   mucci@eecs.utk.edu

* Mods:    Gary Mohr

*          gary.mohr@bull.com

*          Modified the perf_event component to use PFM_OS_PERF_EVENT_EXT mode in libpfm4.

*          This adds several new event masks, including cpu=, u=, and k= which give the user

*          the ability to set cpu number to use or control the domain (user, kernel, or both)

*          in which the counter should be incremented.  These are event masks so it is now 

*          possible to have multiple events in the same event set that count activity from 

*          differennt cpu's or count activity in different domains.

*/

#include <fcntl.h>

#include <string.h>

#include <errno.h>

#include <signal.h>

#include <syscall.h>

#include <sys/utsname.h>

#include <sys/mman.h>

#include <sys/ioctl.h>

/* PAPI-specific includes */

#include "papi.h"

#include "papi_memory.h"

#include "papi_internal.h"

#include "papi_vector.h"

#include "extras.h"

/* libpfm4 includes */

#include "papi_libpfm4_events.h"

#include "pe_libpfm4_events.h"

#include "perfmon/pfmlib.h"

#include PEINCLUDE

/* Linux-specific includes */

#include "mb.h"

#include "linux-memory.h"

#include "linux-timer.h"

#include "linux-common.h"

#include "linux-context.h"

#include "perf_event_lib.h"

#include "perf_helpers.h"

/* Set to enable pre-Linux 2.6.34 perf_event workarounds   */

/* If disabling them gets no complaints then we can remove */

/* These in a future version of PAPI.                      */

#define OBSOLETE_WORKAROUNDS 0

/* Defines for ctx->state */

#define PERF_EVENTS_OPENED  0x01

#define PERF_EVENTS_RUNNING 0x02

/* Forward declaration */

papi_vector_t _perf_event_vector;

/* Globals */

struct native_event_table_t perf_native_event_table;

static int our_cidx;

static int exclude_guest_unsupported;

/* The kernel developers say to never use a refresh value of 0        */

/* See https://lkml.org/lkml/2011/5/24/172                            */

/* However, on some platforms (like Power) a value of 1 does not work */

/* We're still tracking down why this happens.                        */

#if defined(__powerpc__)

#define PAPI_REFRESH_VALUE 0

#else

#define PAPI_REFRESH_VALUE 1

#endif

static int _pe_set_domain( hwd_control_state_t *ctl, int domain);

#if (OBSOLETE_WORKAROUNDS==1)

/* Check for processor support */

/* Can be used for generic checking, though in general we only     */

/* check for pentium4 here because support was broken for multiple */

/* kernel releases and the usual standard detections did not       */

/* handle this.  So we check for pentium 4 explicitly.             */

static int

processor_supported(int vendor, int family) {

   /* Error out if kernel too early to support p4 */

   if (( vendor == PAPI_VENDOR_INTEL ) && (family == 15)) {

      if (_papi_os_info.os_version < LINUX_VERSION(2,6,35)) {

         PAPIERROR("Pentium 4 not supported on kernels before 2.6.35");

         return PAPI_ENOSUPP;

      }

   }

   return PAPI_OK;

}

#endif

/* Fix up the config based on what CPU/Vendor we are running on */

static int

pe_vendor_fixups(papi_vector_t *vector)

{

     /* powerpc */

     /* On IBM and Power6 Machines default domain should include supervisor */

  if ( _papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_IBM ) {

     vector->cmp_info.available_domains |=

                  PAPI_DOM_KERNEL | PAPI_DOM_SUPERVISOR;

     if (strcmp(_papi_hwi_system_info.hw_info.model_string, "POWER6" ) == 0 ) {

        vector->cmp_info.default_domain =

                  PAPI_DOM_USER | PAPI_DOM_KERNEL | PAPI_DOM_SUPERVISOR;

     }

  }

  if ( _papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_MIPS ) {

     vector->cmp_info.available_domains |= PAPI_DOM_KERNEL;

  }

  if ((_papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_INTEL) ||

      (_papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_AMD)) {

     vector->cmp_info.fast_real_timer = 1;

  }

	/* ARM */

	if ( _papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_ARM) {

		/* Some ARMv7 and earlier could not measure	*/

		/* KERNEL and USER separately.			*/

		/* Whitelist CortexA7 and CortexA15		*/

		/* There might be more				*/

		if ((_papi_hwi_system_info.hw_info.cpuid_family < 8) &&

			(_papi_hwi_system_info.hw_info.cpuid_model!=0xc07) &&

			(_papi_hwi_system_info.hw_info.cpuid_model!=0xc0f)) {

			vector->cmp_info.available_domains |=

				PAPI_DOM_USER | PAPI_DOM_KERNEL | PAPI_DOM_SUPERVISOR;

			vector->cmp_info.default_domain =

				PAPI_DOM_USER | PAPI_DOM_KERNEL | PAPI_DOM_SUPERVISOR;

		}

	}

	/* CRAY */

	if ( _papi_hwi_system_info.hw_info.vendor == PAPI_VENDOR_CRAY ) {

		vector->cmp_info.available_domains |= PAPI_DOM_OTHER;

	}

	return PAPI_OK;

}

/******************************************************************/

/******** Kernel Version Dependent Routines  **********************/

/******************************************************************/

/* PERF_FORMAT_GROUP allows reading an entire group's counts at once   */

/* before 2.6.34 PERF_FORMAT_GROUP did not work when reading results   */

/*  from attached processes.  We are lazy and disable it for all cases */

/*  commit was:  050735b08ca8a016bbace4445fa025b88fee770b              */

static int

bug_format_group(void) {

#if (OBSOLETE_WORKAROUNDS==1)

	if (_papi_os_info.os_version < LINUX_VERSION(2,6,34)) return 1;

#endif

	/* MIPS, as of version 3.1, does not support this properly */

	/* FIXME: is this still true? */

#if defined(__mips__)

  return 1;

#endif

  return 0;

}

#if (OBSOLETE_WORKAROUNDS==1)

/* There's a bug prior to Linux 2.6.33 where if you are using */

/* PERF_FORMAT_GROUP, the TOTAL_TIME_ENABLED and              */

/* TOTAL_TIME_RUNNING fields will be zero unless you disable  */

/* the counters first                                         */

static int

bug_sync_read(void) {

  if (_papi_os_info.os_version < LINUX_VERSION(2,6,33)) return 1;

  return 0;

}

#endif

/* Set the F_SETOWN_EX flag on the fd.                          */

/* This affects which thread an overflow signal gets sent to    */

/* Handled in a subroutine to handle the fact that the behavior */

/* is dependent on kernel version.                              */

static int

fcntl_setown_fd(int fd) {

	int ret;

	struct f_owner_ex fown_ex;

	/* F_SETOWN_EX is not available until 2.6.32 */

	/* but PAPI perf_event support didn't work on 2.6.31 anyay */

	/* set ownership of the descriptor */

	fown_ex.type = F_OWNER_TID;

	fown_ex.pid  = mygettid();

	ret = fcntl(fd, F_SETOWN_EX, (unsigned long)&fown_ex );

	if ( ret == -1 ) {

		PAPIERROR( "cannot fcntl(F_SETOWN_EX) on %d: %s",

			fd, strerror( errno ) );

		return PAPI_ESYS;

	}

	return PAPI_OK;

}

/* The read format on perf_event varies based on various flags that */

/* are passed into it.  This helper avoids copying this logic       */

/* multiple places.                                                 */

static unsigned int

get_read_format( unsigned int multiplex,

		 unsigned int inherit,

		 int format_group )

{

   unsigned int format = 0;

   /* if we need read format options for multiplexing, add them now */

   if (multiplex) {

      format |= PERF_FORMAT_TOTAL_TIME_ENABLED;

      format |= PERF_FORMAT_TOTAL_TIME_RUNNING;

   }

   /* if our kernel supports it and we are not using inherit, */

   /* add the group read options                              */

   if ( (!bug_format_group()) && !inherit) {

      if (format_group) {

	 format |= PERF_FORMAT_GROUP;

      }

   }

   SUBDBG("multiplex: %d, inherit: %d, group_leader: %d, format: %#x\n",

	  multiplex, inherit, format_group, format);

   return format;

}

/* attr.exclude_guest is enabled by default in recent libpfm4 */

/* however older kernels will reject events with it set */

/* because the reserved field is not all zeros */

static int

check_exclude_guest( void )

{

	int ev_fd;

	struct perf_event_attr attr;

	exclude_guest_unsupported=0;

	/* First check that we can open a plain instructions event */

	memset(&attr, 0 , sizeof(attr));

	attr.config = PERF_COUNT_HW_INSTRUCTIONS;

	ev_fd = sys_perf_event_open( &attr, 0, -1, -1, 0 );

	if ( ev_fd == -1 ) {

		PAPIERROR("Couldn't open hw_instructions in exclude_guest=0 test");

		return -1;

	}

	close(ev_fd);

	/* Now try again with excude_guest */

	memset(&attr, 0 , sizeof(attr));

	attr.config = PERF_COUNT_HW_INSTRUCTIONS;

	attr.exclude_guest=1;

	ev_fd = sys_perf_event_open( &attr, 0, -1, -1, 0 );

	if ( ev_fd == -1 ) {

		if (errno==EINVAL) {

			exclude_guest_unsupported=1;

		}

		else {

		  PAPIERROR("Couldn't open hw_instructions in exclude_guest=1 test");

		}

	} else {

		exclude_guest_unsupported=0;

		close(ev_fd);

	}

	return PAPI_OK;

}

/*****************************************************************/

/********* End Kernel-version Dependent Routines  ****************/

/*****************************************************************/

/*****************************************************************/

/********* Begin perf_event low-level code ***********************/

/*****************************************************************/

static void perf_event_dump_attr( struct perf_event_attr *hw_event,

	pid_t pid, int cpu, int group_fd, unsigned long int flags) {

	/* Mark parameters as not used                   */

	/* In the common case (no SUBDBG) the function   */

	/* compiles into an empty function and complains */

	/* about unused variables.                       */

	(void)hw_event;

	(void)pid;

	(void)cpu;

	(void)group_fd;

	(void)flags;

	SUBDBG("sys_perf_event_open(hw_event: %p, pid: %d, cpu: %d, "

		"group_fd: %d, flags: %lx\n",

		hw_event, pid, cpu, group_fd, flags);

	SUBDBG("   type: %d\n",hw_event->type);

	SUBDBG("   size: %d\n",hw_event->size);

	SUBDBG("   config: %"PRIx64" (%"PRIu64")\n",

		hw_event->config, hw_event->config);

	SUBDBG("   sample_period: %"PRIu64"\n",hw_event->sample_period);

	SUBDBG("   sample_type: %"PRIu64"\n",hw_event->sample_type);

	SUBDBG("   read_format: %"PRIu64"\n",hw_event->read_format);

	SUBDBG("   disabled: %d\n",hw_event->disabled);

	SUBDBG("   inherit: %d\n",hw_event->inherit);

	SUBDBG("   pinned: %d\n",hw_event->pinned);

	SUBDBG("   exclusive: %d\n",hw_event->exclusive);

	SUBDBG("   exclude_user: %d\n",hw_event->exclude_user);

	SUBDBG("   exclude_kernel: %d\n",hw_event->exclude_kernel);

	SUBDBG("   exclude_hv: %d\n",hw_event->exclude_hv);

	SUBDBG("   exclude_idle: %d\n",hw_event->exclude_idle);

	SUBDBG("   mmap: %d\n",hw_event->mmap);

	SUBDBG("   comm: %d\n",hw_event->comm);

	SUBDBG("   freq: %d\n",hw_event->freq);

	SUBDBG("   inherit_stat: %d\n",hw_event->inherit_stat);

	SUBDBG("   enable_on_exec: %d\n",hw_event->enable_on_exec);

	SUBDBG("   task: %d\n",hw_event->task);

	SUBDBG("   watermark: %d\n",hw_event->watermark);

	SUBDBG("   precise_ip: %d\n",hw_event->precise_ip);

	SUBDBG("   mmap_data: %d\n",hw_event->mmap_data);

	SUBDBG("   sample_id_all: %d\n",hw_event->sample_id_all);

	SUBDBG("   exclude_host: %d\n",hw_event->exclude_host);

	SUBDBG("   exclude_guest: %d\n",hw_event->exclude_guest);

	SUBDBG("   exclude_callchain_kernel: %d\n",

		hw_event->exclude_callchain_kernel);

	SUBDBG("   exclude_callchain_user: %d\n",

		hw_event->exclude_callchain_user);

	SUBDBG("   wakeup_events: %"PRIx32" (%"PRIu32")\n",

		hw_event->wakeup_events, hw_event->wakeup_events);

	SUBDBG("   bp_type: %"PRIx32" (%"PRIu32")\n",

		hw_event->bp_type, hw_event->bp_type);

	SUBDBG("   config1: %"PRIx64" (%"PRIu64")\n",

		hw_event->config1, hw_event->config1);

	SUBDBG("   config2: %"PRIx64" (%"PRIu64")\n",

		hw_event->config2, hw_event->config2);

	SUBDBG("   branch_sample_type: %"PRIx64" (%"PRIu64")\n",

		hw_event->branch_sample_type, hw_event->branch_sample_type);

	SUBDBG("   sample_regs_user: %"PRIx64" (%"PRIu64")\n",

		hw_event->sample_regs_user, hw_event->sample_regs_user);

	SUBDBG("   sample_stack_user: %"PRIx32" (%"PRIu32")\n",

		hw_event->sample_stack_user, hw_event->sample_stack_user);

}

static int map_perf_event_errors_to_papi(int perf_event_error) {

   int ret;

   /* These mappings are approximate.

      EINVAL in particular can mean lots of different things */

   switch(perf_event_error) {

      case EPERM:

      case EACCES:

           ret = PAPI_EPERM;

	   break;

      case ENODEV:

      case EOPNOTSUPP:

	   ret = PAPI_ENOSUPP;

           break;

      case ENOENT:

	   ret = PAPI_ENOEVNT;

           break;

      case ENOSYS:

      case EAGAIN:

      case EBUSY:

      case E2BIG:	/* Only happens if attr is the wrong size somehow */

      case EBADF:	/* We are attempting to group with an invalid file descriptor */

	   ret = PAPI_ESYS;

	   break;

      case ENOMEM:

	   ret = PAPI_ENOMEM;

	   break;

      case EMFILE:	/* Out of file descriptors.  Typically max out at 1024 */

           ret = PAPI_ECOUNT;

           break;

      case EINVAL:

      default:

	   ret = PAPI_EINVAL;

           break;

   }

   return ret;

}

/** Check if the current set of options is supported by  */

/*  perf_events.                                         */

/*  We do this by temporarily opening an event with the  */

/*  desired options then closing it again.  We use the   */

/*  PERF_COUNT_HW_INSTRUCTION event as a dummy event     */

/*  on the assumption it is available on all             */

/*  platforms.                                           */

static int

check_permissions( unsigned long tid,

		   unsigned int cpu_num,

		   unsigned int domain,

		   unsigned int granularity,

		   unsigned int multiplex,

		   unsigned int inherit )

{

   int ev_fd;

   struct perf_event_attr attr;

   long pid;

   /* clearing this will set a type of hardware and to count all domains */

   memset(&attr, '\0', sizeof(attr));

   attr.read_format = get_read_format(multiplex, inherit, 1);

   /* set the event id (config field) to instructios */

   /* (an event that should always exist)            */

   /* This was cycles but that is missing on Niagara */

   attr.config = PERF_COUNT_HW_INSTRUCTIONS;

   /* now set up domains this event set will be counting */

   if (!(domain & PAPI_DOM_SUPERVISOR)) {

      attr.exclude_hv = 1;

   }

   if (!(domain & PAPI_DOM_USER)) {

      attr.exclude_user = 1;

   }

   if (!(domain & PAPI_DOM_KERNEL)) {

      attr.exclude_kernel = 1;

   }

   if (granularity==PAPI_GRN_SYS) {

      pid = -1;

   } else {

      pid = tid;

   }

   SUBDBG("Calling sys_perf_event_open() from check_permissions\n");

	perf_event_dump_attr( &attr, pid, cpu_num, -1, 0 );

   ev_fd = sys_perf_event_open( &attr, pid, cpu_num, -1, 0 );

   if ( ev_fd == -1 ) {

      SUBDBG("sys_perf_event_open returned error.  Linux says, %s", 

	     strerror( errno ) );

      return map_perf_event_errors_to_papi(errno);

   }

   /* now close it, this was just to make sure we have permissions */

   /* to set these options                                         */

   close(ev_fd);

   return PAPI_OK;

}

/* Maximum size we ever expect to read from a perf_event fd   */

/*  (this is the number of 64-bit values)                     */

/* We use this to size the read buffers                       */

/* The three is for event count, time_enabled, time_running   */

/*  and the counter term is count value and count id for each */

/*  possible counter value.                                   */

#define READ_BUFFER_SIZE (3 + (2 * PERF_EVENT_MAX_MPX_COUNTERS))

/* KERNEL_CHECKS_SCHEDUABILITY_UPON_OPEN is a work-around for kernel arch */

/* implementations (e.g. x86 before 2.6.33) which don't do a static event */

/* scheduability check in sys_perf_event_open.  It is also needed if the  */

/* kernel is stealing an event, such as when NMI watchdog is enabled.     */

static int

check_scheduability( pe_context_t *ctx, pe_control_t *ctl, int idx )

{

   int retval = 0, cnt = -1;

   ( void ) ctx;			 /*unused */

   long long papi_pe_buffer[READ_BUFFER_SIZE];

   int i,group_leader_fd;

   /* If the kernel isn't tracking scheduability right       */

   /* Then we need to start/stop/read to force the event     */

   /* to be scheduled and see if an error condition happens. */

   /* get the proper fd to start */

   group_leader_fd=ctl->events[idx].group_leader_fd;

   if (group_leader_fd==-1) group_leader_fd=ctl->events[idx].event_fd;

   /* start the event */

   retval = ioctl( group_leader_fd, PERF_EVENT_IOC_ENABLE, NULL );

   if (retval == -1) {

      PAPIERROR("ioctl(PERF_EVENT_IOC_ENABLE) failed");

      return PAPI_ESYS;

   }

   /* stop the event */

   retval = ioctl(group_leader_fd, PERF_EVENT_IOC_DISABLE, NULL );

   if (retval == -1) {

      PAPIERROR( "ioctl(PERF_EVENT_IOC_DISABLE) failed" );

      return PAPI_ESYS;

   }

   /* See if a read returns any results */

   cnt = read( group_leader_fd, papi_pe_buffer, sizeof(papi_pe_buffer));

   if ( cnt == -1 ) {

      SUBDBG( "read returned an error!  Should never happen.\n" );

      return PAPI_ESYS;

   }

   if ( cnt == 0 ) {

      /* We read 0 bytes if we could not schedule the event */

      /* The kernel should have detected this at open       */

      /* but various bugs (including NMI watchdog)          */

      /* result in this behavior                            */

      return PAPI_ECNFLCT;

   } else {

      /* Reset all of the counters (opened so far) back to zero      */

      /* from the above brief enable/disable call pair.              */

      /* We have to reset all events because reset of group leader      */

      /* does not reset all.                                            */

      /* we assume that the events are being added one by one and that  */

      /* we do not need to reset higher events (doing so may reset ones */

      /* that have not been initialized yet.                            */

      /* Note... PERF_EVENT_IOC_RESET does not reset time running       */

      /* info if multiplexing, so we should avoid coming here if        */

      /* we are multiplexing the event.                                 */

      for( i = 0; i < idx; i++) {

	 retval=ioctl( ctl->events[i].event_fd, PERF_EVENT_IOC_RESET, NULL );

	 if (retval == -1) {

	    PAPIERROR( "ioctl(PERF_EVENT_IOC_RESET) #%d/%d %d "

		       "(fd %d)failed",

		       i,ctl->num_events,idx,ctl->events[i].event_fd);

	    return PAPI_ESYS;

	 }

      }

   }

   return PAPI_OK;

}

/* Do some extra work on a perf_event fd if we're doing sampling  */

/* This mostly means setting up the mmap buffer.                  */

static int

configure_fd_for_sampling( pe_control_t *ctl, int evt_idx )

{

   int ret;

   int fd = ctl->events[evt_idx].event_fd;

   /* Register that we would like a SIGIO notification when a mmap'd page */

   /* becomes full.                                                       */

   ret = fcntl( fd, F_SETFL, O_ASYNC | O_NONBLOCK );

   if ( ret ) {

      PAPIERROR ( "fcntl(%d, F_SETFL, O_ASYNC | O_NONBLOCK) "

		  "returned error: %s", fd, strerror( errno ) );

      return PAPI_ESYS;

   }

   /* Set the F_SETOWN_EX flag on the fd.                          */

   /* This affects which thread an overflow signal gets sent to.   */

   ret=fcntl_setown_fd(fd);

   if (ret!=PAPI_OK) return ret;

   /* Set FD_CLOEXEC.  Otherwise if we do an exec with an overflow */

   /* running, the overflow handler will continue into the exec()'d*/

   /* process and kill it because no signal handler is set up.     */

   ret=fcntl(fd, F_SETFD, FD_CLOEXEC);

   if (ret) {

      return PAPI_ESYS;

   }

   /* when you explicitely declare that you want a particular signal,  */

   /* even with you use the default signal, the kernel will send more  */

   /* information concerning the event to the signal handler.          */

   /*                                                                  */

   /* In particular, it will send the file descriptor from which the   */

   /* event is originating which can be quite useful when monitoring   */

   /* multiple tasks from a single thread.                             */

   ret = fcntl( fd, F_SETSIG, ctl->overflow_signal );

   if ( ret == -1 ) {

      PAPIERROR( "cannot fcntl(F_SETSIG,%d) on %d: %s",

		 ctl->overflow_signal, fd,

		 strerror( errno ) );

      return PAPI_ESYS;

   }

	return PAPI_OK;

}

static int

set_up_mmap( pe_control_t *ctl, int evt_idx)

{

	void *buf_addr;

	int fd = ctl->events[evt_idx].event_fd;

	/* mmap() the sample buffer */

	buf_addr = mmap( NULL,

			ctl->events[evt_idx].nr_mmap_pages * getpagesize(),

			PROT_READ | PROT_WRITE,

			MAP_SHARED,

			fd, 0 );

	/* This may happen if we go over the limit in	*/

	/* /proc/sys/kernel/perf_event_mlock_kb		*/

	/* which defaults to 516k			*/

	/* with regular rdpmc events on 4k page archs	*/

	/* this is roughly 128 events			*/

	/* We sholdn't fail, just fall back to non-rdpmc	*/

	/* Although not sure what happens if it's a sample	*/

	/* event that fails to mmap.				*/

	if ( buf_addr == MAP_FAILED ) {

		SUBDBG( "mmap(NULL,%d,%d,%d,%d,0): %s",

			ctl->events[evt_idx].nr_mmap_pages * getpagesize(),

			PROT_READ | PROT_WRITE,

			MAP_SHARED,

			fd, strerror( errno ) );

		ctl->events[evt_idx].mmap_buf = NULL;

		/* Easier to just globally disable this, as it should	*/

		/* be a fairly uncommon case hopefully.			*/

		if (_perf_event_vector.cmp_info.fast_counter_read) {

			PAPIERROR("Can't mmap, disabling fast_counter_read\n");

			_perf_event_vector.cmp_info.fast_counter_read=0;

		}

		return PAPI_ESYS;

	}

	SUBDBG( "Sample buffer for fd %d is located at %p\n", fd, buf_addr );

	/* Set up the mmap buffer and its associated helpers */

	ctl->events[evt_idx].mmap_buf = (struct perf_counter_mmap_page *) buf_addr;

	ctl->events[evt_idx].tail = 0;

	ctl->events[evt_idx].mask =

		( ctl->events[evt_idx].nr_mmap_pages - 1 ) * getpagesize() - 1;

	return PAPI_OK;

}

/* Open all events in the control state */

static int

open_pe_events( pe_context_t *ctx, pe_control_t *ctl )

{

	int i, ret = PAPI_OK;

	long pid;

	if (ctl->granularity==PAPI_GRN_SYS) {

		pid = -1;

	}

	else {

		pid = ctl->tid;

	}

	for( i = 0; i < ctl->num_events; i++ ) {

		ctl->events[i].event_opened=0;

		/* set up the attr structure.			*/

		/* We don't set up all fields here		*/

		/* as some have already been set up previously.	*/

		/* Handle the broken exclude_guest problem */

		/* libpfm4 sets this by default (PEBS events depend on it) */

		/* but on older kernels that dont know about exclude_guest */

		/* perf_event_open() will error out as a "reserved"        */

		/* unknown bit is set to 1.                                */

		/* Do we need to also watch for exclude_host, exclude_idle */

		/* exclude_callchain*?					   */

		if ((ctl->events[i].attr.exclude_guest) &&

			(exclude_guest_unsupported)) {

			SUBDBG("Disabling exclude_guest in event %d\n",i);

			ctl->events[i].attr.exclude_guest=0;

		}

		/* group leader (event 0) is special                */

		/* If we're multiplexed, everyone is a group leader */

		if (( i == 0 ) || (ctl->multiplexed)) {

			ctl->events[i].attr.pinned = !ctl->multiplexed;

			ctl->events[i].attr.disabled = 1;

			ctl->events[i].group_leader_fd=-1;

			ctl->events[i].attr.read_format = get_read_format(

							ctl->multiplexed,

							ctl->inherit,

							!ctl->multiplexed );

		} else {

			ctl->events[i].attr.pinned=0;

			ctl->events[i].attr.disabled = 0;

			ctl->events[i].group_leader_fd=ctl->events[0].event_fd;

			ctl->events[i].attr.read_format = get_read_format(

							ctl->multiplexed,

							ctl->inherit,

							0 );

		}

		/* try to open */

		perf_event_dump_attr(

				&ctl->events[i].attr,

				pid,

				ctl->events[i].cpu,

				ctl->events[i].group_leader_fd,

				0 /* flags */ );

		ctl->events[i].event_fd = sys_perf_event_open(

				&ctl->events[i].attr,

				pid,

				ctl->events[i].cpu,

				ctl->events[i].group_leader_fd,

				0 /* flags */ );

		/* Try to match Linux errors to PAPI errors */

		if ( ctl->events[i].event_fd == -1 ) {

			SUBDBG("sys_perf_event_open returned error "

				"on event #%d.  Error: %s\n",

				i, strerror( errno ) );

			ret=map_perf_event_errors_to_papi(errno);

			goto open_pe_cleanup;

		}

		SUBDBG ("sys_perf_event_open: tid: %ld, cpu_num: %d,"

			" group_leader/fd: %d, event_fd: %d,"

			" read_format: %"PRIu64"\n",

			pid, ctl->events[i].cpu,

			ctl->events[i].group_leader_fd,

			ctl->events[i].event_fd,

			ctl->events[i].attr.read_format);

		/* in many situations the kernel will indicate we opened fine */

		/* yet things will fail later.  So we need to double check    */

		/* we actually can use the events we've set up.               */

		/* This is not necessary if we are multiplexing, and in fact */

		/* we cannot do this properly if multiplexed because         */

		/* PERF_EVENT_IOC_RESET does not reset the time running info */

		if (!ctl->multiplexed) {

			ret = check_scheduability( ctx, ctl, i );

			if ( ret != PAPI_OK ) {

				/* the last event did open, so we need to    */

				/* bump the counter before doing the cleanup */

				i++;

				goto open_pe_cleanup;

			}

		}

		ctl->events[i].event_opened=1;

	}

	/* Now that we've successfully opened all of the events, do whatever  */

	/* "tune-up" is needed to attach the mmap'd buffers, signal handlers, */

	/* and so on.                                                         */

	/* Make things easier and give each event a mmap() buffer */

	/* Keeping separate tracking for rdpmc vs regular events  */

	/* Would be a pain.  Also perf always gives every event a */

	/* mmap buffer.						  */

	for ( i = 0; i < ctl->num_events; i++ ) {

		/* Can't mmap() inherited events :( */

		if (ctl->inherit) {

			ctl->events[i].nr_mmap_pages = 0;

			ctl->events[i].mmap_buf = NULL;

		}

		else {

			/* Just a guess at how many pages would make this   */

			/* relatively efficient.                            */

			/* Note that it's "1 +" because of the need for a   */

			/* control page, and the number following the "+"   */

			/* must be a power of 2 (1, 4, 8, 16, etc) or zero. */

			/* This is required to optimize dealing with        */

			/* circular buffer wrapping of the mapped pages.    */

			if (ctl->events[i].sampling) {

				ctl->events[i].nr_mmap_pages = 1 + 2;

			}

			else if (_perf_event_vector.cmp_info.fast_counter_read) {

				ctl->events[i].nr_mmap_pages = 1;

			}

			else {

				ctl->events[i].nr_mmap_pages = 0;

			}

			/* Set up the MMAP sample pages */

			if (ctl->events[i].nr_mmap_pages) {

				set_up_mmap(ctl,i);

			} else {

				ctl->events[i].mmap_buf = NULL;

			}

		}

	}

	for ( i = 0; i < ctl->num_events; i++ ) {

		/* If sampling is enabled, hook up signal handler */

		if (ctl->events[i].attr.sample_period) {

			ret = configure_fd_for_sampling( ctl, i );

			if ( ret != PAPI_OK ) {

				/* We failed, and all of the fds are open */

				/* so we need to clean up all of them */

				i = ctl->num_events;

				goto open_pe_cleanup;

			}

		}

	}

	/* Set num_evts only if completely successful */

	ctx->state |= PERF_EVENTS_OPENED;

	return PAPI_OK;

open_pe_cleanup:

	/* We encountered an error, close up the fds we successfully opened.  */

	/* We go backward in an attempt to close group leaders last, although */

	/* That's probably not strictly necessary.                            */

	while ( i > 0 ) {

		i--;

		if (ctl->events[i].event_fd>=0) {

			close( ctl->events[i].event_fd );

			ctl->events[i].event_opened=0;

		}

	}

	return ret;

}

/* TODO: make code clearer -- vmw */

static int

close_event( pe_event_info_t *event )

{

	int munmap_error=0,close_error=0;

	if ( event->mmap_buf ) {

		if (event->nr_mmap_pages==0) {

			PAPIERROR("munmap and num pages is zero");

		}

		if ( munmap ( event->mmap_buf,

				event->nr_mmap_pages * getpagesize() ) ) {

			PAPIERROR( "munmap of fd = %d returned error: %s",