/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */

/*

 *  (C) 2013 by Argonne National Laboratory.

 *      See COPYRIGHT in top-level directory.

 */

/* Types and interfaces in this file are internally used by MPIR_T itself.

 * Other modules should call higher level interfaces in mpit.h instead.

 */

#ifndef MPITIMPL_H_INCLUDED

#define MPITIMPL_H_INCLUDED

#include "mpi.h"

#include "mpir_strerror.h"

#include "mpir_type_defs.h"

#include "mpir_assert.h"

#include "mpir_pointers.h"

#include "utarray.h"

#include "uthash.h"

#include "mpir_objects.h"

#ifdef HAVE_ERROR_CHECKING

typedef enum {

    MPIR_T_OBJECT_INVALID = 0,

    MPIR_T_ENUM_HANDLE,

    MPIR_T_CVAR_HANDLE,

    MPIR_T_PVAR_HANDLE,

    MPIR_T_PVAR_SESSION

} MPIR_T_object_kind;

#endif

/* MPI_T enum

 */

typedef struct enum_item_s {

    const char *name;

    int value;

} enum_item_t;

typedef struct MPIR_T_enum_s {

#ifdef HAVE_ERROR_CHECKING

    MPIR_T_object_kind kind;

#endif

    const char *name;

    UT_array *items;

} MPIR_T_enum_t;

/* MPI_T category (cat)

 */

typedef struct {

    const char *name;

    UT_array *cvar_indices;

    UT_array *pvar_indices;

    UT_array *subcat_indices;

    const char *desc;

} cat_table_entry_t;

/* Hash names to indices in a table */

typedef struct {

    const char *name;

    unsigned idx;

    UT_hash_handle hh;          /* Makes this structure hashable */

} name2index_hash_t;

/* MPI_T control variable (cvar)

 */

typedef struct MPIR_T_cvar_range_value_s {

    int low;

    int high;

} MPIR_T_cvar_range_value_t;

/* Type used to represent cvar default values */

typedef union MPIR_T_cvar_value_s {

    int d;

    unsigned u;

    unsigned ul;

    unsigned ull;

    MPI_Count c;

    const char *str;

    double f;

    MPIR_T_cvar_range_value_t range;

} MPIR_T_cvar_value_t;

typedef int MPIR_T_cvar_get_addr_cb(void *obj_handle, void **addr);

typedef int MPIR_T_cvar_get_count_cb(void *obj_handle, int *count);

typedef struct cvar_table_entry_s {

    /* Is the cvar currently in use? False if the cvar is unregistered */

    int active;

    /* cvar name */

    const char *name;

    /* Address of the cvar. May be NULL when get_addr != NULL */

    void *addr;

    /* cvar data type */

    MPI_Datatype datatype;

    /* Num. of elements of the cvar. May be ignored when get_count != NULL */

    int count;

    /* Properties of the cvar */

    MPI_T_enum enumtype;

    MPIR_T_verbosity_t verbosity;

    MPIR_T_bind_t bind;

    MPIR_T_scope_t scope;

    /* Default value */

    MPIR_T_cvar_value_t defaultval;

    /* If not NULL, components provide this callback to get addr of the cvar */

    MPIR_T_cvar_get_addr_cb *get_addr;

    /* If not NULL, components provide this callback to get count */

    MPIR_T_cvar_get_count_cb *get_count;

    /* Description of the cvar */

    const char *desc;

} cvar_table_entry_t;

typedef struct MPIR_T_cvar_handle_s {

#ifdef HAVE_ERROR_CHECKING

    MPIR_T_object_kind kind;

#endif

    /* Address and count of the cvar. Set at handle allocation time */

    void *addr;

    int count;

    /* Cached value from cvar_table_entry_t to avoid indirection */

    MPI_Datatype datatype;

    MPIR_T_scope_t scope;

} MPIR_T_cvar_handle_t;

void MPIR_T_CVAR_REGISTER_impl(MPI_Datatype dtype, const char *name, const void *addr, int count,

                               MPIR_T_enum_t * etype, MPIR_T_verbosity_t verb, MPIR_T_bind_t bind,

                               MPIR_T_scope_t scope, MPIR_T_cvar_get_addr_cb get_addr,

                               MPIR_T_cvar_get_count_cb get_count, MPIR_T_cvar_value_t defaultval,

                               const char *cat, const char *desc);

/* MPI_T performance variable (pvar)

 */

/* Forward declaration */

struct MPIR_T_pvar_handle_s;

struct MPIR_T_pvar_session_s;

typedef void MPIR_T_pvar_get_value_cb(void *addr, void *obj_handle, int count, void *buf);

typedef void MPIR_T_pvar_get_count_cb(void *addr, void *obj_handle, int *count);

/* Basic pvar flags defined by MPI_T standard */

#define MPIR_T_PVAR_FLAG_READONLY      0x01

#define MPIR_T_PVAR_FLAG_CONTINUOUS    0x02

#define MPIR_T_PVAR_FLAG_ATOMIC        0x04

/* Auxlilary flags used by MPIR_T */

/* pvar is MPI_T_PVAR_CLASS_{COUNTER, TIMER, AGGREGATE} */

#define MPIR_T_PVAR_FLAG_SUM           0x08

/* pvar is MPI_T_PVAR_CLASS_{HIGH, LOW}WATERMARK */

#define MPIR_T_PVAR_FLAG_WATERMARK     0x10

/* pvar is continuous. If not, it has been started at least once */

#define MPIR_T_PVAR_FLAG_ONCESTARTED   0x20

/* pvar is continuous. If not, it is started */

#define MPIR_T_PVAR_FLAG_STARTED       0x40

/* Used only for watermark handles. Set if a pvar handle is the

 * first handle of an associated watermark.

 */

#define MPIR_T_PVAR_FLAG_FIRST         0x80

/* MPI_T performance variable (pvar) stuff */

typedef struct {

    /* Is the pvar in use (i.e., not unregistered)? */

    int active;

    /* pvar name */

    const char *name;

    /* If not NULL, it is address of the pvar */

    void *addr;

    /* pvar data type */

    MPI_Datatype datatype;

    /* Num. of elements of the pvar */

    int count;

    /* Properties of the pvar */

    MPIR_T_pvar_class_t varclass;

    MPIR_T_verbosity_t verbosity;

    MPIR_T_enum_t *enumtype;

    MPIR_T_bind_t bind;

    /* Basic flags of the pvar */

    int flags;

    /* If not NULL, components provide this callback to read the pvar */

    MPIR_T_pvar_get_value_cb *get_value;

    /* If not NULL, components provide this callback to get count */

    MPIR_T_pvar_get_count_cb *get_count;

    /* Description of the pvar */

    const char *desc;

} pvar_table_entry_t;

/*

 The following two macros do not work since C preprocessor does not support

 nested ifdefs. So we use another woarkable but a little ugly approach.

#define PVAR_GATED_ACTION(MODULE, action_) \

    do { \

        #ifdef ENABLE_PVAR_##MODULE \

            action_; \

        #endif \

    } while (0)

*/

/* ENABLE_PVAR_##MODULE must be defined by configure script either to 0 or 1 */

#define PVAR_GATED_ACTION(MODULE, action_) \

    do { \

        if (ENABLE_PVAR_##MODULE) { \

            action_; \

        } \

    } while (0)

/* For some classes of pvars, internally we can not represent them

 * in basic data types. So come the following typedefs.

 */

/* Timer type */

typedef struct {

    /* Accumulated time */

    MPID_Time_t total;

    /* Time when the timer was started recently */

    MPID_Time_t curstart;

    /* A counter recording how many times the timer is started */

    unsigned long long count;

} MPIR_T_pvar_timer_t;

/* An union to represent a watermark value */

typedef union {

    double f;

    unsigned u;

    unsigned long ul;

    unsigned long long ull;

} watermark_value_t;

/* Watermark type */

typedef struct {

    /* current -- current resource utilization level

     * waterarmk -- cached value for the first pvar handle

     */

    watermark_value_t current, watermark;

    /* Datatype of the watermark */

    MPI_Datatype datatype;

    /* Is the cached value (i.e, watermark) in use by a pvar handle? */

    int first_used;

    /* Is the first pvar handle started? */

    int first_started;

    /* A double-linked list of handles of the pvar */

    struct MPIR_T_pvar_handle_s *hlist;

} MPIR_T_pvar_watermark_t;

typedef struct MPIR_T_pvar_handle_s {

#ifdef HAVE_ERROR_CHECKING

    MPIR_T_object_kind kind;

#endif

    /* These are cached fields from pvar table. Do so to avoid extra

     * indirection when accessing them through pvar handles.

     */

    void *addr;

    MPI_Datatype datatype;

    int count;

    MPIR_T_pvar_get_value_cb *get_value;

    MPIR_T_pvar_class_t varclass;

    /* Bytes of an element of datatype */

    int bytes;

    /* Basic flags copied from pvar info + auxilary flags in pvar handle */

    int flags;

    /* Store info here in case we need other fields */

    const pvar_table_entry_t *info;

    /* Owner session from which the handle is allocated */

    struct MPIR_T_pvar_session_s *session;

    /* Object which this pvar is bound to. NULL if no binding */

    void *obj_handle;

    /* This is how we support pvar sessions.

     *

     * For pvars of counter, timer or aggregate type, we cache their value at

     * the last start time in offset, their current value in current, and

     * their accumlated value in accum. Generally, when such a pvar is running,

     * reading the pvar should return

     *      accum[i] + current[i] - offset[i], 0 <= i < count - 1.

     * When the pvar is stopped, reading just returns accum.

     *

     * For pvars of high/lowwatermark type, above method does not work.

     * We have a copy of such a pvar in every handle of the pvar.

     * Handles are registered to the pvar. Whenever a watermark changes,

     * its copies in non-stopped handles are updated. That sounds non-scalable.

     * Considering single-session is common, we reserve room in watermark

     * themselves for cache buffer for the first handle. So when such a pvar

     * changes, it also updates the watermark close to it in memory.

     *

     * For pvars of other classes,  since they are supposed to be readonly

     * and continuous (FIXME: Is it true?), caching is not needed.

     */

    void *accum;

    void *offset;

    void *current;

    watermark_value_t watermark;

    /* To chain handles in a session */

    struct MPIR_T_pvar_handle_s *prev, *next;

    /* To chain handles of a watermark pvar */

    struct MPIR_T_pvar_handle_s *prev2, *next2;

} MPIR_T_pvar_handle_t;

typedef struct MPIR_T_pvar_session_s {

#ifdef HAVE_ERROR_CHECKING

    MPIR_T_object_kind kind;

#endif

    /* A linked list of pvar handles */

    MPIR_T_pvar_handle_t *hlist;

} MPIR_T_pvar_session_t;

extern void MPIR_T_PVAR_REGISTER_impl(MPIR_T_pvar_class_t varclass, MPI_Datatype dtype,

                                      const char *name, void *addr, int count,

                                      MPIR_T_enum_t * etype, MPIR_T_verbosity_t verb,

                                      MPIR_T_bind_t bind, int flags,

                                      MPIR_T_pvar_get_value_cb get_value,

                                      MPIR_T_pvar_get_count_cb get_count, const char *cat,

                                      const char *desc);

/* For static pvars (i.e., pvars with static storage), we embed their class name

 * into their variable name, so that users can declare pvars with the same name

 * for different classes, without worry of name conflict. "class + pvar name"

 * should be unique as required by MPI_T.

 */

/* MPI_T_PVAR_CLASS_STATE (continuous only)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_STATE_SET_VAR_impl(ptr_, val_) \

    do { *(ptr_) = (val_); } while (0)

#define MPIR_T_PVAR_STATE_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_STATE_SET_impl(name_, val_) \

    MPIR_T_PVAR_STATE_SET_VAR_impl(&PVAR_STATE_##name_, val_)

#define MPIR_T_PVAR_STATE_GET_impl(name_) \

    MPIR_T_PVAR_STATE_GET_VAR_impl(&PVAR_STATE_##name_)

/* Registration AND initialization for static pvar */

#define MPIR_T_PVAR_STATE_REGISTER_STATIC_impl(dtype_, name_, \

            initval_, etype_, verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_INT); \

        /* Double check if dtype_ and name_ match */ \

        MPIR_Assert(sizeof(PVAR_STATE_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        /* State pvars should be describled further by an enum */ \

        MPIR_Assert((etype_) != MPI_T_ENUM_NULL); \

        PVAR_STATE_##name_ = (initval_); \

        addr_ = &PVAR_STATE_##name_; \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_STATE, dtype_, #name_, \

            addr_, 1, etype_, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_STATE_REGISTER_DYNAMIC_impl(dtype_, name_, addr_, count_, \

            etype_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_INT); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        MPIR_Assert((etype_) != MPI_T_ENUM_NULL); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_STATE, dtype_, #name_, \

            addr_, count_, etype_, verb_, bind_, flags_, get_value_, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_LEVEL (continuous only)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_LEVEL_SET_VAR_impl(ptr_, val_) \

    do { *(ptr_) = (val_); } while (0)

#define MPIR_T_PVAR_LEVEL_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_LEVEL_INC_VAR_impl(ptr_, val_) \

    do { *(ptr_) += (val_); } while (0)

#define MPIR_T_PVAR_LEVEL_DEC_VAR_impl(ptr_, val_) \

    do { *(ptr_) -= (val_); } while (0)

#define MPIR_T_PVAR_LEVEL_SET_impl(name_, val_) \

    MPIR_T_PVAR_LEVEL_SET_VAR_impl(&PVAR_LEVEL_##name_, val_)

#define MPIR_T_PVAR_LEVEL_GET_impl(name_) \

    MPIR_T_PVAR_LEVEL_GET_VAR_impl(&PVAR_LEVEL_##name_)

#define MPIR_T_PVAR_LEVEL_INC_impl(name_, val_) \

    MPIR_T_PVAR_LEVEL_INC_VAR_impl(&PVAR_LEVEL_##name_, val_)

#define MPIR_T_PVAR_LEVEL_DEC_impl(name_, val_) \

    MPIR_T_PVAR_LEVEL_DEC_VAR_impl(&PVAR_LEVEL_##name_, val_)

/* Registration AND initialization for static pvar */

#define MPIR_T_PVAR_LEVEL_REGISTER_STATIC_impl(dtype_, name_, \

            initval_, verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        /* Double check if dtype_ and name_ match */ \

        MPIR_Assert(sizeof(PVAR_LEVEL_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        PVAR_LEVEL_##name_ = (initval_); \

        addr_ = &PVAR_LEVEL_##name_; \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LEVEL, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_LEVEL_REGISTER_DYNAMIC_impl(dtype_, name_, \

            addr_, count_, verb_, bind_, flags_, get_value_, get_count, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_LEVEL, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \

            get_count, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_SIZE (continuous only)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_SIZE_SET_VAR_impl(ptr_, val_) \

    do { *(ptr_) = (val_); } while (0)

#define MPIR_T_PVAR_SIZE_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_SIZE_SET_impl(name_, val_) \

    MPIR_T_PVAR_SIZE_SET_VAR_impl(&PVAR_SIZE_##name_, val_)

#define MPIR_T_PVAR_SIZE_GET_impl(name_) \

    MPIR_T_PVAR_SIZE_GET_VAR_impl(&PVAR_SIZE_##name_)

/* Registration AND initialization for static pvar */

#define MPIR_T_PVAR_SIZE_REGISTER_STATIC_impl(dtype_, name_, \

            initval_, verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        /* Double check if dtype_ and name_ match */ \

        MPIR_Assert(sizeof(PVAR_SIZE_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        PVAR_SIZE_##name_ = (initval_); \

        addr_ = &PVAR_SIZE_##name_; \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_SIZE, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_SIZE_REGISTER_DYNAMIC_impl(dtype_, name_, \

            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_SIZE, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \

            get_count_, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_PERCENTAGE (continuous only)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_PERCENTAGE_SET_VAR_impl(ptr_, val_) \

    do { \

        MPIR_Assert(0.0 <= (val_) && (val_) <= 1.0); \

        *(ptr_) = (val_); \

    } while (0)

#define MPIR_T_PVAR_PERCENTAGE_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_PERCENTAGE_SET_impl(name_, val_) \

    MPIR_T_PVAR_PERCENTAGE_SET_VAR_impl(&PVAR_PERCENTAGE_##name_, val_)

#define MPIR_T_PVAR_PERCENTAGE_GET_impl(name_) \

    MPIR_T_PVAR_PERCENTAGE_GET_VAR_impl(&PVAR_PERCENTAGE_##name_)

/* Registration AND initialization for static pvar */

#define MPIR_T_PVAR_PERCENTAGE_REGISTER_STATIC_impl(dtype_, name_, \

            initval_, verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_DOUBLE); \

        /* Double check if dtype_ and name_ match */ \

        MPIR_Assert(sizeof(PVAR_PERCENTAGE_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        addr_ = &PVAR_PERCENTAGE_##name_; \

        PVAR_PERCENTAGE_##name_ = (initval_); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_PERCENTAGE, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_PERCENTAGE_REGISTER_DYNAMIC_impl(dtype_, name_, \

            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_DOUBLE); \

        MPIR_Assert((flags_) & MPIR_T_PVAR_FLAG_CONTINUOUS); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_PERCENTAGE, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \

            get_count_, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_COUNTER (continuous or not)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_COUNTER_INIT_VAR_impl(ptr_) \

    do { *(ptr_) = 0; } while (0)

/* _COUNTER_SET is intentionally not provided. Users should only INC counters */

#define MPIR_T_PVAR_COUNTER_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_COUNTER_INC_VAR_impl(ptr_, inc_) \

    do { *(ptr_) += (inc_); } while (0)

#define MPIR_T_PVAR_COUNTER_INIT_impl(name_) \

    MPIR_T_PVAR_COUNTER_INIT_VAR_impl(&PVAR_COUNTER_##name_)

#define MPIR_T_PVAR_COUNTER_GET_impl(name_) \

    MPIR_T_PVAR_COUNTER_GET_VAR_impl(&PVAR_COUNTER_##name_)

#define MPIR_T_PVAR_COUNTER_INC_impl(name_, inc_) \

    MPIR_T_PVAR_COUNTER_INC_VAR_impl(&PVAR_COUNTER_##name_, inc_)

#define MPIR_T_PVAR_COUNTER_ADDR_impl(name_) \

    (&PVAR_COUNTER_##name_)

/* Registration AND initialization to zero for static pvar.  */

#define MPIR_T_PVAR_COUNTER_REGISTER_STATIC_impl(dtype_, name_, \

            verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG); \

        /* Double check if dtype_ and name_ match*/ \

        MPIR_Assert(sizeof(PVAR_COUNTER_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        PVAR_COUNTER_##name_ = 0; \

        addr_ = &PVAR_COUNTER_##name_; \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC_impl(dtype_, name_, \

            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \

            get_count_, cat_, desc_); \

    } while (0)

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR_impl(ptr_, count_) \

    do { \

        int idx_; \

        idx_ = 0; \

        for (; idx_ < (count_); idx_++) \

            *((ptr_) + idx_) = 0; \

    } while (0)

#define MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR_impl(ptr_, idx_) \

    *((ptr_) + (idx_))

#define MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR_impl(ptr_, idx_, inc_) \

    do { *((ptr_) + (idx_)) += (inc_); } while (0)

#define MPIR_T_PVAR_COUNTER_ARRAY_INIT_impl(name_) \

    do { \

        int count_; \

        count_ = sizeof(PVAR_COUNTER_##name_)/sizeof(PVAR_COUNTER_##name_[0]); \

        MPIR_T_PVAR_COUNTER_ARRAY_INIT_VAR_impl(PVAR_COUNTER_##name_, count_); \

    } while (0)

#define MPIR_T_PVAR_COUNTER_ARRAY_GET_impl(name_, idx_) \

    MPIR_T_PVAR_COUNTER_ARRAY_GET_VAR_impl(PVAR_COUNTER_##name_, idx_)

#define MPIR_T_PVAR_COUNTER_ARRAY_INC_impl(ptr_, idx_, inc_) \

    MPIR_T_PVAR_COUNTER_ARRAY_INC_VAR_impl(PVAR_COUNTER_##name_, idx_, inc_)

/* Registration AND initialization to zero for static counter array  */

#define MPIR_T_PVAR_COUNTER_ARRAY_REGISTER_STATIC_impl(dtype_, name_, \

            verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        int count_;  \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG); \

        /* Double check if dtype_ and name_ match */ \

        MPIR_Assert(sizeof(PVAR_COUNTER_##name_[0]) == MPIR_Datatype_get_basic_size(dtype_)); \

        addr_ = PVAR_COUNTER_##name_; \

        MPIR_T_PVAR_COUNTER_ARRAY_INIT_impl(name_); \

        count_ = sizeof(PVAR_COUNTER_##name_)/sizeof(mpit_pvar_##name_[0]); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Dynamic counter array is already provided by MPIR_T_PVAR_COUNTER_REGISTER_DYNAMIC */

/* MPI_T_PVAR_CLASS_AGGREGATE (continuous or not)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_AGGREGATE_INIT_VAR_impl(ptr_) \

    do { *(ptr_) = 0; } while (0)

/* _AGGREGATE_SET is intentionally not provided as for counters */

#define MPIR_T_PVAR_AGGREGATE_GET_VAR_impl(ptr_) \

    (*(ptr_))

#define MPIR_T_PVAR_AGGREGATE_INC_VAR_impl(ptr_, inc_) \

    do { *(ptr_) += (inc_); } while (0)

#define MPIR_T_PVAR_AGGREGATE_INIT_impl(name_) \

    MPIR_T_PVAR_AGGREGATE_INIT_VAR_impl(&PVAR_AGGREGATE_##name_)

#define MPIR_T_PVAR_AGGREGATE_GET_impl(name_) \

    MPIR_T_PVAR_AGGREGATE_GET_VAR_impl(&PVAR_AGGREGATE_##name_)

#define MPIR_T_PVAR_AGGREGATE_INC_impl(name_, inc_) \

    MPIR_T_PVAR_AGGREGATE_INC_VAR_impl(&PVAR_AGGREGATE_##name_, inc_)

/* Registration AND initialization to zero for static aggregate  */

#define MPIR_T_PVAR_AGGREGATE_REGISTER_STATIC_impl(dtype_, name_, \

            verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        /* Double check if dtype_ and name_ match*/ \

        MPIR_Assert(sizeof(PVAR_AGGREGATE_##name_) == MPIR_Datatype_get_basic_size(dtype_)); \

        PVAR_AGGREGATE_##name_ = 0; \

        addr_ = &PVAR_AGGREGATE_##name_; \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_AGGREGATE, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, NULL, NULL, cat_, desc_); \

    } while (0)

/* Registration for dynamic pvar w/ or w/o callback. Init is left to users */

#define MPIR_T_PVAR_AGGREGATE_REGISTER_DYNAMIC_impl(dtype_, name_, \

            addr_, count_, verb_, bind_, flags_, get_value_, get_count_, cat_, desc_) \

    do { \

        /* Allowable datatypes */ \

        MPIR_Assert((dtype_) == MPI_UNSIGNED || (dtype_) == MPI_UNSIGNED_LONG || \

                    (dtype_) == MPI_UNSIGNED_LONG_LONG || (dtype_) == MPI_DOUBLE); \

        MPIR_Assert((addr_) != NULL || (get_value_) != NULL); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_AGGREGATE, dtype_, #name_, \

            addr_, count_, MPI_T_ENUM_NULL, verb_, bind_, flags_, get_value_, \

            get_count_, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_TIMER (continuous or not)

 */

/* Interfaces through pointer or name */

#define MPIR_T_PVAR_TIMER_INIT_VAR_impl(ptr_) \

    do { \

        /* FIXME: need a generic approach to init a timer */ \

        memset(&((ptr_)->total), 0, sizeof(MPID_Time_t)); \

    } while (0)

#define MPIR_T_PVAR_TIMER_GET_VAR_impl(ptr_, buf) \

    do { \

        MPID_Wtime_todouble(&((ptr_)->total), buf); \

    } while (0)

#define MPIR_T_PVAR_TIMER_START_VAR_impl(ptr_) \

    do { \

        MPID_Wtime(&((ptr_)->curstart)); \

        (ptr_)->count++; \

    } while (0)

#define MPIR_T_PVAR_TIMER_END_VAR_impl(ptr_) \

    do { \

        MPID_Time_t tmp_; \

        MPID_Wtime(&tmp_); \

        MPID_Wtime_acc(&((ptr_)->curstart), &tmp_, &((ptr_)->total)); \

    } while (0)

#define MPIR_T_PVAR_TIMER_INIT_impl(name_) \

    MPIR_T_PVAR_TIMER_INIT_VAR_impl(&PVAR_TIMER_##name_)

#define MPIR_T_PVAR_TIMER_GET_impl(name_, buf_) \

    MPIR_T_PVAR_TIMER_GET_VAR_impl(&PVAR_TIMER_##name_, buf_)

#define MPIR_T_PVAR_TIMER_START_impl(name_) \

    MPIR_T_PVAR_TIMER_START_VAR_impl(&PVAR_TIMER_##name_)

#define MPIR_T_PVAR_TIMER_END_impl(name_) \

    MPIR_T_PVAR_TIMER_END_VAR_impl(&PVAR_TIMER_##name_)

#define MPIR_T_PVAR_TIMER_ADDR_impl(name_) \

    (&PVAR_TIMER_##name_)

/* Customized get_value() for MPIR_T_pvar_timer_t */

static inline

    void get_timer_time_in_double(MPIR_T_pvar_timer_t * timer, void *obj_handle,

                                  int count, double *buf)

{

    int i;

    for (i = 0; i < count; i++)

        MPID_Wtime_todouble(&(timer[i].total), &buf[i]);

}

/* Registration for static storage */

#define MPIR_T_PVAR_TIMER_REGISTER_STATIC_impl(dtype_, name_, \

            verb_, bind_, flags_, cat_, desc_) \

    do { \

        void *addr_; \

        void *count_addr_; \

        /* Allowable datatypes only */ \

        MPIR_Assert((dtype_) == MPI_DOUBLE); \

        MPIR_T_PVAR_TIMER_INIT_impl(name_); \

        addr_ = &PVAR_TIMER_##name_; \

        count_addr_ = &(PVAR_TIMER_##name_.count); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_TIMER, dtype_, #name_, \

            addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, \

            (MPIR_T_pvar_get_value_cb *)&get_timer_time_in_double, NULL, cat_, desc_); \

        MPIR_T_PVAR_REGISTER_impl(MPI_T_PVAR_CLASS_COUNTER, MPI_UNSIGNED_LONG_LONG, #name_, \

            count_addr_, 1, MPI_T_ENUM_NULL, verb_, bind_, flags_, \

            NULL, NULL, cat_, desc_); \

    } while (0)

/* MPI_T_PVAR_CLASS_HIGHWATERMARK (continuous or not)

 */

/* Interfaces through pointer or name.

 * In contrast to previous pvar classes, for each type we create a set

 * of interfaces. That is because we have a pointer and a union in the

 * struct. We need to know types to (de)reference them.

*/

#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \

    do { \

        (ptr_)->datatype = MPI_UNSIGNED; \

        (ptr_)->current.u = (val_); \

        (ptr_)->first_started = 0;    \

        (ptr_)->first_used = 0; \

        (ptr_)->hlist = NULL;  \

    } while (0)

#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \

    do { \

        (ptr_)->datatype = MPI_UNSIGNED_LONG; \

        (ptr_)->current.ul = (val_); \

        (ptr_)->first_started = 0;    \

        (ptr_)->first_used = 0; \

        (ptr_)->hlist = NULL;  \

    } while (0)

#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \

    do { \

        (ptr_)->datatype = MPI_UNSIGNED_LONG_LONG; \

        (ptr_)->current.ull = (val_); \

        (ptr_)->first_started = 0;    \

        (ptr_)->first_used = 0; \

        (ptr_)->hlist = NULL;  \

    } while (0)

#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_VAR_impl(ptr_, val_) \

    do { \

        (ptr_)->datatype = MPI_DOUBLE; \

        (ptr_)->current.f = (val_); \

        (ptr_)->first_started = 0;    \

        (ptr_)->first_used = 0; \

        (ptr_)->hlist = NULL;  \

    } while (0)

#define MPIR_T_PVAR_UINT_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \

    do { \

        MPIR_T_pvar_handle_t *head; \

        (ptr_)->current.u = (val_); \

        if ((ptr_)->first_used && (ptr_)->first_started) { \

            if ((val_) > (ptr_)->watermark.u) \

                (ptr_)->watermark.u = (val_); \

        } \

        head = (ptr_)->hlist; \

        while (head != NULL) { \

            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.u) { \

                head->watermark.u = (val_); \

            } \

            head = head->next2; \

        } \

    } while (0)

#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \

    do { \

        MPIR_T_pvar_handle_t *head; \

        (ptr_)->current.ul = (val_); \

        if ((ptr_)->first_used && (ptr_)->first_started) { \

            if ((val_) > (ptr_)->watermark.ul) \

                (ptr_)->watermark.ul = (val_); \

        } \

        head = (ptr_)->hlist; \

        while (head != NULL) { \

            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.ul) { \

                head->watermark.ul = (val_); \

            } \

            head = head->next2; \

        } \

    } while (0)

#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \

    do { \

        MPIR_T_pvar_handle_t *head; \

        (ptr_)->current.ull = (val_); \

        if ((ptr_)->first_used && (ptr_)->first_started) { \

            if ((val_) > (ptr_)->watermark.ull) \

                (ptr_)->watermark.ull = (val_); \

        } \

        head = (ptr_)->hlist; \

        while (head != NULL) { \

            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.ull) { \

                head->watermark.ull = (val_); \

            } \

            head = head->next2; \

        } \

    } while (0)

#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_UPDATE_VAR_impl(ptr_, val_) \

    do { \

        MPIR_T_pvar_handle_t *head; \

        (ptr_)->current.f = (val_); \

        if ((ptr_)->first_used && (ptr_)->first_started) { \

            if ((val_) > (ptr_)->watermark.f) \

                (ptr_)->watermark.f = (val_); \

        } \

        head = (ptr_)->hlist; \

        while (head != NULL) { \

            if (MPIR_T_pvar_is_started(head) && (val_) > head->watermark.f) { \

                head->watermark.f = (val_); \

            } \

            head = head->next2; \

        } \

    } while (0)

#define MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_impl(name_, val_) \

    MPIR_T_PVAR_UINT_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)

#define MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_impl(name_, val_) \

    MPIR_T_PVAR_ULONG_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)

#define MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_impl(name_, val_) \

    MPIR_T_PVAR_ULONG2_HIGHWATERMARK_INIT_VAR_impl(&PVAR_HIGHWATERMARK_##name_, val_)

#define MPIR_T_PVAR_DOUBLE_HIGHWATERMARK_INIT_impl(name_, val_) \