/** * @file linux-mx.c * @brief A component for Myricom MX (Myrinet Express) */ #include "papi.h" #include "papi_internal.h" #include "papi_vector.h" #include #include #include #include #include #define MX_MAX_COUNTERS 100 #define MX_MAX_COUNTER_TERMS MX_MAX_COUNTERS #define LINELEN 128 typedef struct MX_register { /* indicate which counters this event can live on */ unsigned int selector; } MX_register_t; typedef struct MX_native_event_entry { /* description of the resources required by this native event */ MX_register_t resources; /* If it exists, then this is the name of this event */ char *name; /* If it exists, then this is the description of this event */ char *description; } MX_native_event_entry_t; typedef struct MX_reg_alloc { MX_register_t ra_bits; } MX_reg_alloc_t; typedef struct MX_control_state { long long start_count[MX_MAX_COUNTERS]; long long current_count[MX_MAX_COUNTERS]; long long difference[MX_MAX_COUNTERS]; int which_counter[MX_MAX_COUNTERS]; int num_events; } MX_control_state_t; typedef struct MX_context { MX_control_state_t state; } MX_context_t; static const MX_native_event_entry_t mx_native_table[] = { {{1, }, "LANAI_UPTIME", "Lanai uptime (seconds)"}, {{2, }, "COUNTERS_UPTIME", "Counters uptime (seconds)"}, {{3, }, "BAD_CRC8", "Bad CRC8 (Port 0)"}, {{4, }, "BAD_CRC32", "Bad CRC32 (Port 0)"}, {{5, }, "UNSTRIPPED_ROUTE", "Unstripped route (Port 0)"}, {{6, }, "PKT_DESC_INVALID", "pkt_desc_invalid (Port 0)"}, {{7, }, "RECV_PKT_ERRORS", "recv_pkt_errors (Port 0)"}, {{8, }, "PKT_MISROUTED", "pkt_misrouted (Port 0)"}, {{9, }, "DATA_SRC_UNKNOWN", "data_src_unknown"}, {{10, }, "DATA_BAD_ENDPT", "data_bad_endpt"}, {{11, }, "DATA_ENDPT_CLOSED", "data_endpt_closed"}, {{12, }, "DATA_BAD_SESSION", "data_bad_session"}, {{13, }, "PUSH_BAD_WINDOW", "push_bad_window"}, {{14, }, "PUSH_DUPLICATE", "push_duplicate"}, {{15, }, "PUSH_OBSOLETE", "push_obsolete"}, {{16, }, "PUSH_RACE_DRIVER", "push_race_driver"}, {{17, }, "PUSH_BAD_SEND_HANDLE_MAGIC", "push_bad_send_handle_magic"}, {{18, }, "PUSH_BAD_SRC_MAGIC", "push_bad_src_magic"}, {{19, }, "PULL_OBSOLETE", "pull_obsolete"}, {{20, }, "PULL_NOTIFY_OBSOLETE", "pull_notify_obsolete"}, {{21, }, "PULL_RACE_DRIVER", "pull_race_driver"}, {{22, }, "ACK_BAD_TYPE", "ack_bad_type"}, {{23, }, "ACK_BAD_MAGIC", "ack_bad_magic"}, {{24, }, "ACK_RESEND_RACE", "ack_resend_race"}, {{25, }, "LATE_ACK", "Late ack"}, {{26, }, "ACK_NACK_FRAMES_IN_PIPE", "ack_nack_frames_in_pipe"}, {{27, }, "NACK_BAD_ENDPT", "nack_bad_endpt"}, {{28, }, "NACK_ENDPT_CLOSED", "nack_endpt_closed"}, {{29, }, "NACK_BAD_SESSION", "nack_bad_session"}, {{30, }, "NACK_BAD_RDMAWIN", "nack_bad_rdmawin"}, {{31, }, "NACK_EVENTQ_FULL", "nack_eventq_full"}, {{32, }, "SEND_BAD_RDMAWIN", "send_bad_rdmawin"}, {{33, }, "CONNECT_TIMEOUT", "connect_timeout"}, {{34, }, "CONNECT_SRC_UNKNOWN", "connect_src_unknown"}, {{35, }, "QUERY_BAD_MAGIC", "query_bad_magic"}, {{36, }, "QUERY_TIMED_OUT", "query_timed_out"}, {{37, }, "QUERY_SRC_UNKNOWN", "query_src_unknown"}, {{38, }, "RAW_SENDS", "Raw sends (Port 0)"}, {{39, }, "RAW_RECEIVES", "Raw receives (Port 0)"}, {{40, }, "RAW_OVERSIZED_PACKETS", "Raw oversized packets (Port 0)"}, {{41, }, "RAW_RECV_OVERRUN", "raw_recv_overrun"}, {{42, }, "RAW_DISABLED", "raw_disabled"}, {{43, }, "CONNECT_SEND", "connect_send"}, {{44, }, "CONNECT_RECV", "connect_recv"}, {{45, }, "ACK_SEND", "ack_send (Port 0)"}, {{46, }, "ACK_RECV", "ack_recv (Port 0)"}, {{47, }, "PUSH_SEND", "push_send (Port 0)"}, {{48, }, "PUSH_RECV", "push_recv (Port 0)"}, {{49, }, "QUERY_SEND", "query_send (Port 0)"}, {{50, }, "QUERY_RECV", "query_recv (Port 0)"}, {{51, }, "REPLY_SEND", "reply_send (Port 0)"}, {{52, }, "REPLY_RECV", "reply_recv (Port 0)"}, {{53, }, "QUERY_UNKNOWN", "query_unknown (Port 0)"}, /* {{ 54, }, "QUERY_UNKNOWN", "query_unknown (Port 0)"},*/ {{55, }, "DATA_SEND_NULL", "data_send_null (Port 0)"}, {{56, }, "DATA_SEND_SMALL", "data_send_small (Port 0)"}, {{57, }, "DATA_SEND_MEDIUM", "data_send_medium (Port 0)"}, {{58, }, "DATA_SEND_RNDV", "data_send_rndv (Port 0)"}, {{59, }, "DATA_SEND_PULL", "data_send_pull (Port 0)"}, {{60, }, "DATA_RECV_NULL", "data_recv_null (Port 0)"}, {{61, }, "DATA_RECV_SMALL_INLINE", "data_recv_small_inline (Port 0)"}, {{62, }, "DATA_RECV_SMALL_COPY", "data_recv_small_copy (Port 0)"}, {{63, }, "DATA_RECV_MEDIUM", "data_recv_medium (Port 0)"}, {{64, }, "DATA_RECV_RNDV", "data_recv_rndv (Port 0)"}, {{65, }, "DATA_RECV_PULL", "data_recv_pull (Port 0)"}, {{66, }, "ETHER_SEND_UNICAST_CNT", "ether_send_unicast_cnt (Port 0)"}, {{67, }, "ETHER_SEND_MULTICAST_CNT", "ether_send_multicast_cnt (Port 0)"}, {{68, }, "ETHER_RECV_SMALL_CNT", "ether_recv_small_cnt (Port 0)"}, {{69, }, "ETHER_RECV_BIG_CNT", "ether_recv_big_cnt (Port 0)"}, {{70, }, "ETHER_OVERRUN", "ether_overrun"}, {{71, }, "ETHER_OVERSIZED", "ether_oversized"}, {{72, }, "DATA_RECV_NO_CREDITS", "data_recv_no_credits"}, {{73, }, "PACKETS_RECENT", "Packets resent"}, {{74, }, "PACKETS_DROPPED", "Packets dropped (data send side)"}, {{75, }, "MAPPER_ROUTES_UPDATE", "Mapper routes update"}, {{76, }, "ROUTE_DISPERSION", "Route dispersion (Port 0)"}, {{77, }, "OUT_OF_SEND_HANDLES", "out_of_send_handles"}, {{78, }, "OUT_OF_PULL_HANDLES", "out_of_pull_handles"}, {{79, }, "OUT_OF_PUSH_HANDLES", "out_of_push_handles"}, {{80, }, "MEDIUM_CONT_RACE", "medium_cont_race"}, {{81, }, "CMD_TYPE_UNKNOWN", "cmd_type_unknown"}, {{82, }, "UREQ_TYPE_UNKNOWN", "ureq_type_unknown"}, {{83, }, "INTERRUPTS_OVERRUN", "Interrupts overrun"}, {{84, }, "WAITING_FOR_INTERRUPT_DMA", "Waiting for interrupt DMA"}, {{85, }, "WAITING_FOR_INTERRUPT_ACK", "Waiting for interrupt Ack"}, {{86, }, "WAITING_FOR_INTERRUPT_TIMER", "Waiting for interrupt Timer"}, {{87, }, "SLABS_RECYCLING", "Slabs recycling"}, {{88, }, "SLABS_PRESSURE", "Slabs pressure"}, {{89, }, "SLABS_STARVATION", "Slabs starvation"}, {{90, }, "OUT_OF_RDMA_HANDLES", "out_of_rdma handles"}, {{91, }, "EVENTQ_FULL", "eventq_full"}, {{92, }, "BUFFER_DROP", "buffer_drop (Port 0)"}, {{93, }, "MEMORY_DROP", "memory_drop (Port 0)"}, {{94, }, "HARDWARE_FLOW_CONTROL", "Hardware flow control (Port 0)"}, {{95, }, "SIMULATED_PACKETS_LOST", "(Devel) Simulated packets lost (Port 0)"}, {{96, }, "LOGGING_FRAMES_DUMPED", "(Logging) Logging frames dumped"}, {{97, }, "WAKE_INTERRUPTS", "Wake interrupts"}, {{98, }, "AVERTED_WAKEUP_RACE", "Averted wakeup race"}, {{99, }, "DMA_METADATA_RACE", "Dma metadata race"}, {{0, }, "", ""} }; static int num_events=0; papi_vector_t _mx_vector; static char mx_counters_exe[BUFSIZ]; static int read_mx_counters( long long *counters ) { FILE *fp; char line[LINELEN]; int i, linenum; /* Open a pipe to the mx_counters executable */ fp = popen( mx_counters_exe, "r" ); if ( !fp ) { perror( "popen" ); return PAPI_ECMP; } /* A line of output looks something similar to: */ /* " Lanai uptime (seconds): 766268 (0xbb13c)" */ /* This code may fail if number of ports on card > 1 */ linenum = 0; while ( fgets( line, LINELEN, fp ) ) { // printf("%s",line); for(i=0; line[i]!= '\0' && i=MX_MAX_COUNTERS) break; } pclose( fp ); return PAPI_OK; } /* * Component setup and shutdown */ /* Initialize hardware counters, setup the function vector table * and get hardware information, this routine is called when the * PAPI process is initialized (IE PAPI_library_init) */ static int _mx_init_component( int cidx ) { FILE *fff; char test_string[BUFSIZ]; /* detect if MX available */ strncpy(mx_counters_exe,"mx_counters 2> /dev/null",BUFSIZ); fff=popen(mx_counters_exe,"r"); /* popen only returns NULL if "sh" fails, not the actual command */ if (fgets(test_string,BUFSIZ,fff)==NULL) { pclose(fff); strncpy(mx_counters_exe,"./components/mx/utils/fake_mx_counters 2> /dev/null",BUFSIZ); fff=popen(mx_counters_exe,"r"); if (fgets(test_string,BUFSIZ,fff)==NULL) { pclose(fff); /* neither real nor fake found */ strncpy(_mx_vector.cmp_info.disabled_reason, "No MX utilities found",PAPI_MAX_STR_LEN); return PAPI_ECMP; } } pclose(fff); num_events=MX_MAX_COUNTERS; _mx_vector.cmp_info.num_native_events=num_events; /* Export the component id */ _mx_vector.cmp_info.CmpIdx = cidx; return PAPI_OK; } /* * This is called whenever a thread is initialized */ static int _mx_init_thread( hwd_context_t * ctx ) { ( void ) ctx; /*unused */ return PAPI_OK; } static int _mx_shutdown_component(void) { return PAPI_OK; } static int _mx_shutdown_thread( hwd_context_t * ctx ) { ( void ) ctx; /*unused */ return PAPI_OK; } /* * Control of counters (Reading/Writing/Starting/Stopping/Setup) * functions */ static int _mx_init_control_state( hwd_control_state_t *ctl ) { ( void ) ctl; /*unused */ return PAPI_OK; } static int _mx_update_control_state( hwd_control_state_t *ctl, NativeInfo_t *native, int count, hwd_context_t *ctx ) { ( void ) ctx; /*unused */ int i, index; MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl; for(i=0; iwhich_counter[i]=index; // printf("Mapping event# %d to HW counter %d (count=%d)\n", // i,index,count); native[i].ni_position = i; } mx_ctl->num_events=count; return PAPI_OK; } static int _mx_start( hwd_context_t *ctx, hwd_control_state_t *ctl ) { long long mx_counters[MX_MAX_COUNTERS]; ( void ) ctx; /*unused */ MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl; int i; read_mx_counters( mx_counters ); // for(i=0;inum_events;i++) { mx_ctl->current_count[i]= mx_counters[mx_ctl->which_counter[i]]; mx_ctl->start_count[i]=mx_ctl->current_count[i]; } return PAPI_OK; } static int _mx_stop( hwd_context_t *ctx, hwd_control_state_t *ctl ) { ( void ) ctx; /*unused */ long long mx_counters[MX_MAX_COUNTERS]; MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl; int i; read_mx_counters( mx_counters ); for(i=0;inum_events;i++) { mx_ctl->current_count[i]= mx_counters[mx_ctl->which_counter[i]]; } return PAPI_OK; } static int _mx_read( hwd_context_t *ctx, hwd_control_state_t *ctl, long long **events, int flags ) { ( void ) ctx; /*unused */ ( void ) flags; /*unused */ int i; long long mx_counters[MX_MAX_COUNTERS]; MX_control_state_t *mx_ctl = (MX_control_state_t *)ctl; read_mx_counters( mx_counters ); for ( i = 0; i < mx_ctl->num_events; i++ ) { mx_ctl->current_count[i]= mx_counters[mx_ctl->which_counter[i]]; mx_ctl->difference[i] = mx_ctl->current_count[i]- mx_ctl->start_count[i]; } *events = mx_ctl->difference; return PAPI_OK; } static int _mx_reset( hwd_context_t * ctx, hwd_control_state_t * ctrl ) { _mx_start( ctx, ctrl ); return PAPI_OK; } /* Unused write function */ /* static int */ /* _mx_write( hwd_context_t * ctx, hwd_control_state_t * ctrl, long long *from ) */ /* { */ /* ( void ) ctx; /\*unused *\/ */ /* ( void ) ctrl; /\*unused *\/ */ /* ( void ) from; /\*unused *\/ */ /* return PAPI_OK; */ /* } */ /* * Functions for setting up various options */ /* This function sets various options in the component * The valid codes being passed in are PAPI_SET_DEFDOM, * PAPI_SET_DOMAIN, PAPI_SETDEFGRN, PAPI_SET_GRANUL * and PAPI_SET_INHERIT */ static int _mx_ctl( hwd_context_t * ctx, int code, _papi_int_option_t * option ) { ( void ) ctx; /*unused */ ( void ) code; /*unused */ ( void ) option; /*unused */ return PAPI_OK; } /* * This function has to set the bits needed to count different domains * In particular: PAPI_DOM_USER, PAPI_DOM_KERNEL PAPI_DOM_OTHER * By default return PAPI_EINVAL if none of those are specified * and PAPI_OK with success * PAPI_DOM_USER is only user context is counted * PAPI_DOM_KERNEL is only the Kernel/OS context is counted * PAPI_DOM_OTHER is Exception/transient mode (like user TLB misses) * PAPI_DOM_ALL is all of the domains */ static int _mx_set_domain( hwd_control_state_t * cntrl, int domain ) { ( void ) cntrl; /*unused */ if ( PAPI_DOM_ALL != domain ) { return PAPI_EINVAL; } return PAPI_OK; } static int _mx_ntv_code_to_name( unsigned int EventCode, char *name, int len ) { int event=EventCode; if (event >=0 && event < num_events) { strncpy( name, mx_native_table[event].name, len ); return PAPI_OK; } return PAPI_ENOEVNT; } static int _mx_ntv_code_to_descr( unsigned int EventCode, char *name, int len ) { int event=EventCode; if (event >=0 && event < num_events) { strncpy( name, mx_native_table[event].description, len ); return PAPI_OK; } return PAPI_ENOEVNT; } static int _mx_ntv_enum_events( unsigned int *EventCode, int modifier ) { if ( modifier == PAPI_ENUM_FIRST ) { if (num_events==0) return PAPI_ENOEVNT; *EventCode = 0; return PAPI_OK; } if ( modifier == PAPI_ENUM_EVENTS ) { int index = *EventCode; if ( mx_native_table[index + 1].resources.selector ) { *EventCode = *EventCode + 1; return PAPI_OK; } else { return PAPI_ENOEVNT; } } return PAPI_EINVAL; } papi_vector_t _mx_vector = { .cmp_info = { .name = "mx", .short_name = "mx", .version = "1.4", .description = "Myricom MX (Myrinet Express) statistics", .num_mpx_cntrs = MX_MAX_COUNTERS, .num_cntrs = MX_MAX_COUNTERS, .default_domain = PAPI_DOM_ALL, .default_granularity = PAPI_GRN_SYS, .available_granularities = PAPI_GRN_SYS, .hardware_intr_sig = PAPI_INT_SIGNAL, /* component specific cmp_info initializations */ .fast_real_timer = 0, .fast_virtual_timer = 0, .attach = 0, .attach_must_ptrace = 0, .available_domains = PAPI_DOM_ALL, }, /* sizes of framework-opaque component-private structures */ .size = { .context = sizeof ( MX_context_t ), .control_state = sizeof ( MX_control_state_t ), .reg_value = sizeof ( MX_register_t ), .reg_alloc = sizeof ( MX_reg_alloc_t ), }, /* function pointers in this component */ .init_thread = _mx_init_thread, .init_component = _mx_init_component, .init_control_state = _mx_init_control_state, .start = _mx_start, .stop = _mx_stop, .read = _mx_read, .shutdown_thread = _mx_shutdown_thread, .shutdown_component = _mx_shutdown_component, .ctl = _mx_ctl, .update_control_state = _mx_update_control_state, .set_domain = _mx_set_domain, .reset = _mx_reset, .ntv_enum_events = _mx_ntv_enum_events, .ntv_code_to_name = _mx_ntv_code_to_name, .ntv_code_to_descr = _mx_ntv_code_to_descr, };