/*
This file is provided under a dual BSD/GPLv2 license. When using or
redistributing this file, you may do so under either license.
GPL LICENSE SUMMARY
Copyright(c) 2015 Intel Corporation.
This program is free software; you can redistribute it and/or modify
it under the terms of version 2 of the GNU General Public License as
published by the Free Software Foundation.
This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
General Public License for more details.
Contact Information:
Intel Corporation, www.intel.com
BSD LICENSE
Copyright(c) 2015 Intel Corporation.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
* Neither the name of Intel Corporation nor the names of its
contributors may be used to endorse or promote products derived
from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
/* Copyright (c) 2003-2015 Intel Corporation. All rights reserved. */
#ifndef PSMI_AM_H
#define PSMI_AM_H
#include "am_config.h"
#include "../psm_am_internal.h"
#define AMSH_DIRBLOCK_SIZE 128
typedef
struct am_epaddr {
/*
* epaddr must be the first field to have the same address as this
* structure
*/
struct psm2_epaddr epaddr;
uint16_t shmidx;
uint16_t return_shmidx;
uint32_t cstate_outgoing:3;
uint32_t cstate_incoming:3;
uint32_t pid:22;
/*
* Device number of GPU used by given EP, only used when CUDA is
* enabled. There is no gain from #ifdefing it out, since it does not
* use any extra space.
*/
uint32_t gpuid:4;
} am_epaddr_t;
/* Up to NSHORT_ARGS are supported via am_pkt_short_t; the remaining
arguments are passed using space in am_pkt_bulk_t. One additional argument
is added for passing the internal ptl_am handler index. */
#define NSHORT_ARGS 6
#define NBULK_ARGS (PSMI_AM_MAX_ARGS - NSHORT_ARGS + 1)
typedef
struct amsh_am_token {
struct psmi_am_token tok;
ptl_t *ptl; /**> What PTL was it received on */
psm2_mq_t mq; /**> What matched queue is this for ? */
uint16_t shmidx; /**> what shmidx sent this */
} amsh_am_token_t;
typedef void (*psmi_handler_fn_t) (void *token, psm2_amarg_t *args, int nargs,
void *src, size_t len);
typedef struct psmi_handlertab {
psmi_handler_fn_t fn;
} psmi_handlertab_t;
#define PSMI_AM_CONN_REQ 1
#define PSMI_AM_CONN_REP 2
#define PSMI_AM_DISC_REQ 3
#define PSMI_AM_DISC_REP 4
#define PSMI_KASSIST_OFF 0x0
#define PSMI_KASSIST_CMA_GET 0x1
#define PSMI_KASSIST_CMA_PUT 0x2
#define PSMI_KASSIST_CMA 0x3
#define PSMI_KASSIST_GET 0x1
#define PSMI_KASSIST_PUT 0x2
#define PSMI_KASSIST_MASK 0x3
int psmi_epaddr_pid(psm2_epaddr_t epaddr);
/*
* Eventually, we will allow users to register handlers as "don't reply", which
* may save on some of the buffering requirements
*/
#define PSMI_HANDLER_NEEDS_REPLY(handler) 1
#define PSMI_VALIDATE_REPLY(handler) assert(PSMI_HANDLER_NEEDS_REPLY(handler))
int psmi_amsh_poll(ptl_t *ptl, int replyonly);
/* Shared memory AM, forward decls */
int
psmi_amsh_short_request(ptl_t *ptl, psm2_epaddr_t epaddr,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
const void *src, size_t len, int flags);
void
psmi_amsh_short_reply(amsh_am_token_t *tok,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
const void *src, size_t len, int flags);
int
psmi_amsh_long_request(ptl_t *ptl, psm2_epaddr_t epaddr,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
const void *src, size_t len, void *dest, int flags);
void
psmi_amsh_long_reply(amsh_am_token_t *tok,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
const void *src, size_t len, void *dest, int flags);
void psmi_am_mq_handler(void *toki, psm2_amarg_t *args, int narg, void *buf,
size_t len);
void psmi_am_mq_handler(void *toki, psm2_amarg_t *args, int narg, void *buf,
size_t len);
void psmi_am_mq_handler_data(void *toki, psm2_amarg_t *args, int narg,
void *buf, size_t len);
void psmi_am_mq_handler_complete(void *toki, psm2_amarg_t *args, int narg,
void *buf, size_t len);
void psmi_am_mq_handler_rtsmatch(void *toki, psm2_amarg_t *args, int narg,
void *buf, size_t len);
void psmi_am_mq_handler_rtsdone(void *toki, psm2_amarg_t *args, int narg,
void *buf, size_t len);
void psmi_am_handler(void *toki, psm2_amarg_t *args, int narg, void *buf,
size_t len);
/* AM over shared memory (forward decls) */
psm2_error_t
psmi_amsh_am_get_parameters(psm2_ep_t ep, struct psm2_am_parameters *parameters);
psm2_error_t
psmi_amsh_am_short_request(psm2_epaddr_t epaddr,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
void *src, size_t len, int flags,
psm2_am_completion_fn_t completion_fn,
void *completion_ctxt);
psm2_error_t
psmi_amsh_am_short_reply(psm2_am_token_t tok,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
void *src, size_t len, int flags,
psm2_am_completion_fn_t completion_fn,
void *completion_ctxt);
#define amsh_conn_handler_hidx 1
#define mq_handler_hidx 2
#define mq_handler_data_hidx 3
#define mq_handler_rtsmatch_hidx 4
#define mq_handler_rtsdone_hidx 5
#define am_handler_hidx 6
#define AMREQUEST_SHORT 0
#define AMREQUEST_LONG 1
#define AMREPLY_SHORT 2
#define AMREPLY_LONG 3
#define AM_IS_REPLY(x) ((x)&0x2)
#define AM_IS_REQUEST(x) (!AM_IS_REPLY(x))
#define AM_IS_LONG(x) ((x)&0x1)
#define AM_IS_SHORT(x) (!AM_IS_LONG(x))
#define AM_FLAG_SRC_ASYNC 0x1
#define AM_FLAG_SRC_TEMP 0x2
/*
* Request Fifo.
*/
typedef
struct am_reqq {
struct am_reqq *next;
ptl_t *ptl;
psm2_epaddr_t epaddr;
int amtype;
psm2_handler_t handler;
psm2_amarg_t args[8];
int nargs;
uint32_t len;
void *src;
void *dest;
int amflags;
int flags;
} am_reqq_t;
struct am_reqq_fifo_t {
am_reqq_t *first;
am_reqq_t **lastp;
};
psm2_error_t psmi_am_reqq_drain(ptl_t *ptl);
void psmi_am_reqq_add(int amtype, ptl_t *ptl, psm2_epaddr_t epaddr,
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
void *src, size_t len, void *dest, int flags);
/*
* Shared memory Active Messages, implementation derived from
* Lumetta, Mainwaring, Culler. Multi-Protocol Active Messages on a Cluster of
* SMP's. Supercomputing 1997.
*
* We support multiple endpoints in shared memory, but we only support one
* shared memory context with up to AMSH_MAX_LOCAL_PROCS local endpoints. Some
* structures are endpoint specific (as denoted * with amsh_ep_) and others are
* specific to the single shared memory context * (amsh_ global variables).
*
* Each endpoint maintains a shared request block and a shared reply block.
* Each block is composed of queues for small, medium and large messages.
*/
#define QFREE 0
#define QUSED 1
#define QREADY 2
#define QREADYMED 3
#define QREADYLONG 4
#define QISEMPTY(flag) (flag < QREADY)
#if defined(__x86_64__) || defined(__i386__)
# define _QMARK_FLAG_FENCE() asm volatile("" : : : "memory") /* compilerfence */
#else
# error No _QMARK_FLAG_FENCE() defined for this platform
#endif
#define _QMARK_FLAG(pkt_ptr, _flag) \
do { \
_QMARK_FLAG_FENCE(); \
(pkt_ptr)->flag = (_flag); \
} while (0)
#define QMARKFREE(pkt_ptr) _QMARK_FLAG(pkt_ptr, QFREE)
#define QMARKREADY(pkt_ptr) _QMARK_FLAG(pkt_ptr, QREADY)
#define QMARKUSED(pkt_ptr) _QMARK_FLAG(pkt_ptr, QUSED)
#define AMFMT_SYSTEM 1
#define AMFMT_SHORT_INLINE 2
#define AMFMT_SHORT 3
#define AMFMT_LONG 4
#define AMFMT_LONG_END 5
#define AMSH_CMASK_NONE 0
#define AMSH_CMASK_PREREQ 1
#define AMSH_CMASK_POSTREQ 2
#define AMSH_CMASK_DONE 3
#define AMSH_CSTATE_OUTGOING_NONE 1
#define AMSH_CSTATE_OUTGOING_REPLIED 2
#define AMSH_CSTATE_OUTGOING_ESTABLISHED 3
#define AMSH_CSTATE_OUTGOING_DISC_REPLIED 4
#define AMSH_CSTATE_OUTGOING_DISC_REQUESTED 5
#define AMSH_CSTATE_INCOMING_NONE 1
#define AMSH_CSTATE_INCOMING_DISC_REQUESTED 4
#define AMSH_CSTATE_INCOMING_ESTABLISHED 5
#define AMSH_PID_UNKNOWN 0
/**********************************
* Shared memory packet formats
**********************************/
typedef
struct am_pkt_short {
uint32_t flag; /**> Packet state */
union {
uint32_t bulkidx; /**> index in bulk packet queue */
uint32_t length; /**> length when no bulkidx used */
};
uint16_t shmidx; /**> index in shared segment */
uint16_t type;
uint16_t nargs;
uint16_t handleridx;
psm2_amarg_t args[NSHORT_ARGS]; /* AM arguments */
/* We eventually will expose up to 8 arguments, but this isn't implemented
* For now. >6 args will probably require a medium instead of a short */
} __attribute__ ((aligned(64)))
am_pkt_short_t;
PSMI_STRICT_SIZE_DECL(am_pkt_short_t, 64);
typedef struct am_pkt_bulk {
uint32_t flag;
uint32_t idx;
uintptr_t dest; /* Destination pointer in "longs" */
uint32_t dest_off; /* Destination pointer offset */
uint32_t len; /* Destination length within offset */
psm2_amarg_t args[NBULK_ARGS]; /* Additional "spillover" for >6 args */
uint8_t payload[0];
} am_pkt_bulk_t;
/* No strict size decl, used for mediums and longs */
/****************************************************
* Shared memory header and block control structures
***************************************************/
/* Each pkt queue has the same header format, although the queue
* consumers don't use the 'head' index in the same manner. */
typedef struct am_ctl_qhdr {
uint32_t head; /* Touched only by 1 consumer */
uint8_t _pad0[64 - 4];
pthread_spinlock_t lock;
uint32_t tail; /* XXX candidate for fetch-and-incr */
uint32_t elem_cnt;
uint32_t elem_sz;
uint8_t _pad1[64 - 3 * 4 - sizeof(pthread_spinlock_t)];
} am_ctl_qhdr_t;
PSMI_STRICT_SIZE_DECL(am_ctl_qhdr_t, 128);
/* Each process has a reply qhdr and a request qhdr */
typedef struct am_ctl_blockhdr {
volatile am_ctl_qhdr_t shortq;
volatile am_ctl_qhdr_t longbulkq;
} am_ctl_blockhdr_t;
PSMI_STRICT_SIZE_DECL(am_ctl_blockhdr_t, 128 * 2);
/* We cache the "shorts" because that's what we poll on in the critical path.
* We take care to always update these pointers whenever the segment is remapped.
*/
typedef struct am_ctl_qshort_cache {
volatile am_pkt_short_t *base;
volatile am_pkt_short_t *head;
volatile am_pkt_short_t *end;
} am_ctl_qshort_cache_t;
/******************************************
* Shared segment local directory (global)
******************************************
*
* Each process keeps a directory for where request and reply structures are
* located at its peers. This directory must be re-initialized every time the
* shared segment moves in the VM, and the segment moves every time we remap()
* for additional memory.
*/
struct amsh_qdirectory {
am_ctl_blockhdr_t *qreqH;
am_pkt_short_t *qreqFifoShort;
am_pkt_bulk_t *qreqFifoLong;
am_ctl_blockhdr_t *qrepH;
am_pkt_short_t *qrepFifoShort;
am_pkt_bulk_t *qrepFifoLong;
} __attribute__ ((aligned(64)));
/******************************************
* Shared fifo element counts and sizes
******************************************
* These values are context-wide, they can only be set early on and can't be *
* modified at runtime. All endpoints are expected to use the same values.
*/
typedef
struct amsh_qinfo {
int qreqFifoShort;
int qreqFifoLong;
int qrepFifoShort;
int qrepFifoLong;
} amsh_qinfo_t;
/******************************************
* Per-endpoint structures (ep-local)
******************************************
* Each endpoint keeps its own information as to where it resides in the
* directory, and maintains its own cached copies of where the short header
* resides in shared memory.
*
* This structure is carefully arranged to optimize cache locality and
* performance. Do not modify without careful and thorough analysis.
*/
struct am_ctl_nodeinfo {
uint16_t psm_verno;
volatile uint16_t is_init;
volatile pid_t pid;
psm2_epid_t epid;
psm2_epaddr_t epaddr;
uintptr_t amsh_shmbase;
amsh_qinfo_t amsh_qsizes;
uint32_t amsh_features;
struct amsh_qdirectory qdir;
} __attribute__((aligned(64)));
struct ptl_am {
psm2_ep_t ep;
psm2_epid_t epid;
psm2_epaddr_t epaddr;
ptl_ctl_t *ctl;
int connect_phase;
int connect_outgoing;
int connect_incoming;
int zero_polls;
int amsh_only_polls;
int max_ep_idx, am_ep_size;
int psmi_kassist_mode;
char *amsh_keyname;
/* These three items carefully picked to fit in one cache line. */
am_ctl_qshort_cache_t reqH;
am_ctl_qshort_cache_t repH;
struct am_reqq_fifo_t psmi_am_reqq_fifo;
am_pkt_short_t amsh_empty_shortpkt;
struct am_ctl_nodeinfo *self_nodeinfo;
struct am_ctl_nodeinfo *am_ep;
} __attribute__((aligned(64)));
#endif