|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This file is provided under a dual BSD/GPLv2 license. When using or
|
|
Packit Service |
155747 |
redistributing this file, you may do so under either license.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
GPL LICENSE SUMMARY
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2016 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is free software; you can redistribute it and/or modify
|
|
Packit Service |
155747 |
it under the terms of version 2 of the GNU General Public License as
|
|
Packit Service |
155747 |
published by the Free Software Foundation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
This program is distributed in the hope that it will be useful, but
|
|
Packit Service |
155747 |
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit Service |
155747 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit Service |
155747 |
General Public License for more details.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Contact Information:
|
|
Packit Service |
155747 |
Intel Corporation, www.intel.com
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
BSD LICENSE
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Copyright(c) 2016 Intel Corporation.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
Redistribution and use in source and binary forms, with or without
|
|
Packit Service |
155747 |
modification, are permitted provided that the following conditions
|
|
Packit Service |
155747 |
are met:
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
* Redistributions of source code must retain the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer.
|
|
Packit Service |
155747 |
* Redistributions in binary form must reproduce the above copyright
|
|
Packit Service |
155747 |
notice, this list of conditions and the following disclaimer in
|
|
Packit Service |
155747 |
the documentation and/or other materials provided with the
|
|
Packit Service |
155747 |
distribution.
|
|
Packit Service |
155747 |
* Neither the name of Intel Corporation nor the names of its
|
|
Packit Service |
155747 |
contributors may be used to endorse or promote products derived
|
|
Packit Service |
155747 |
from this software without specific prior written permission.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
Packit Service |
155747 |
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
Packit Service |
155747 |
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
Packit Service |
155747 |
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
Packit Service |
155747 |
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit Service |
155747 |
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
Packit Service |
155747 |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
Packit Service |
155747 |
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
Packit Service |
155747 |
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
Packit Service |
155747 |
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Copyright (c) 2003-2016 Intel Corporation. All rights reserved. */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include <sys/types.h> /* shm_open and signal handling */
|
|
Packit Service |
155747 |
#include <sys/mman.h>
|
|
Packit Service |
155747 |
#include <sys/stat.h>
|
|
Packit Service |
155747 |
#include <fcntl.h>
|
|
Packit Service |
155747 |
#include <signal.h>
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#include "psm_user.h"
|
|
Packit Service |
155747 |
#include "psm_mq_internal.h"
|
|
Packit Service |
155747 |
#include "psm_am_internal.h"
|
|
Packit Service |
155747 |
#include "cmarw.h"
|
|
Packit Service |
155747 |
#include "psmi_wrappers.h"
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
#include "am_cuda_memhandle_cache.h"
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int psmi_shm_mq_rv_thresh = PSMI_MQ_RV_THRESH_NO_KASSIST;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static const amsh_qinfo_t amsh_qcounts = {
|
|
Packit Service |
155747 |
.qreqFifoShort = 1024,
|
|
Packit Service |
155747 |
.qreqFifoLong = 256,
|
|
Packit Service |
155747 |
.qrepFifoShort = 1024,
|
|
Packit Service |
155747 |
.qrepFifoLong = 256
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static const amsh_qinfo_t amsh_qelemsz = {
|
|
Packit Service |
155747 |
.qreqFifoShort = sizeof(am_pkt_short_t),
|
|
Packit Service |
155747 |
.qreqFifoLong = AMLONG_SZ,
|
|
Packit Service |
155747 |
.qrepFifoShort = sizeof(am_pkt_short_t),
|
|
Packit Service |
155747 |
.qrepFifoLong = AMLONG_SZ
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ustatic struct {
|
|
Packit Service |
155747 |
void *addr;
|
|
Packit Service |
155747 |
size_t len;
|
|
Packit Service |
155747 |
struct sigaction SIGSEGV_old_act;
|
|
Packit Service |
155747 |
struct sigaction SIGBUS_old_act;
|
|
Packit Service |
155747 |
} action_stash;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static psm2_error_t amsh_poll(ptl_t *ptl, int replyonly);
|
|
Packit Service |
155747 |
static void process_packet(ptl_t *ptl, am_pkt_short_t *pkt, int isreq);
|
|
Packit Service |
155747 |
static void amsh_conn_handler(void *toki, psm2_amarg_t *args, int narg,
|
|
Packit Service |
155747 |
void *buf, size_t len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Kassist helper functions */
|
|
Packit Service |
155747 |
#if _HFI_DEBUGGING
|
|
Packit Service |
155747 |
static const char *psmi_kassist_getmode(int mode);
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
static int psmi_get_kassist_mode();
|
|
Packit Service |
155747 |
int psmi_epaddr_pid(psm2_epaddr_t epaddr);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static inline void
|
|
Packit Service |
155747 |
am_ctl_qhdr_init(volatile am_ctl_qhdr_t *q, int elem_cnt, int elem_sz)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
pthread_spin_init(&q->lock, PTHREAD_PROCESS_SHARED);
|
|
Packit Service |
155747 |
q->head = 0;
|
|
Packit Service |
155747 |
q->tail = 0;
|
|
Packit Service |
155747 |
q->elem_cnt = elem_cnt;
|
|
Packit Service |
155747 |
q->elem_sz = elem_sz;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static void
|
|
Packit Service |
155747 |
am_ctl_bulkpkt_init(am_pkt_bulk_t *base_ptr, size_t elemsz, int nelems)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
am_pkt_bulk_t *bulkpkt;
|
|
Packit Service |
155747 |
uintptr_t bulkptr = (uintptr_t) base_ptr;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i < nelems; i++, bulkptr += elemsz) {
|
|
Packit Service |
155747 |
bulkpkt = (am_pkt_bulk_t *) bulkptr;
|
|
Packit Service |
155747 |
bulkpkt->idx = i;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define _PA(type) PSMI_ALIGNUP(amsh_qcounts.q ## type * amsh_qelemsz.q ## type, \
|
|
Packit Service |
155747 |
PSMI_PAGESIZE)
|
|
Packit Service |
155747 |
static inline uintptr_t am_ctl_sizeof_block()
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return PSMI_ALIGNUP(
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(AMSH_BLOCK_HEADER_SIZE, PSMI_PAGESIZE) +
|
|
Packit Service |
155747 |
/* reqctrl block */
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(sizeof(am_ctl_blockhdr_t), PSMI_PAGESIZE) +
|
|
Packit Service |
155747 |
_PA(reqFifoShort) + _PA(reqFifoLong) +
|
|
Packit Service |
155747 |
/*reqctrl block */
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(sizeof(am_ctl_blockhdr_t), PSMI_PAGESIZE) +
|
|
Packit Service |
155747 |
/* align to page size */
|
|
Packit Service |
155747 |
_PA(repFifoShort) + _PA(repFifoLong), PSMI_PAGESIZE);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#undef _PA
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static void am_update_directory(struct am_ctl_nodeinfo *);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void amsh_atexit()
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
static pthread_mutex_t mutex_once = PTHREAD_MUTEX_INITIALIZER;
|
|
Packit Service |
155747 |
static int atexit_once;
|
|
Packit Service |
155747 |
psm2_ep_t ep;
|
|
Packit Service |
155747 |
struct ptl_am *ptl;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
pthread_mutex_lock(&mutex_once);
|
|
Packit Service |
155747 |
if (atexit_once) {
|
|
Packit Service |
155747 |
pthread_mutex_unlock(&mutex_once);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
atexit_once = 1;
|
|
Packit Service |
155747 |
pthread_mutex_unlock(&mutex_once);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ep = psmi_opened_endpoint;
|
|
Packit Service |
155747 |
while (ep) {
|
|
Packit Service |
155747 |
ptl = (struct ptl_am *)(ep->ptl_amsh.ptl);
|
|
Packit Service |
155747 |
if (ptl->self_nodeinfo &&
|
|
Packit Service |
155747 |
ptl->amsh_keyname != NULL) {
|
|
Packit Service |
155747 |
_HFI_VDBG("unlinking shm file %s\n",
|
|
Packit Service |
155747 |
ptl->amsh_keyname);
|
|
Packit Service |
155747 |
shm_unlink(ptl->amsh_keyname);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
ep = ep->user_ep_next;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ustatic
|
|
Packit Service |
155747 |
void amsh_mmap_fault(int signo, siginfo_t *siginfo, void *context)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
if ((unsigned long int) siginfo->si_addr >= (unsigned long int) action_stash.addr &&
|
|
Packit Service |
155747 |
(unsigned long int) siginfo->si_addr < (unsigned long int) action_stash.addr + (unsigned long int) action_stash.len) {
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static char shm_errmsg[256];
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
snprintf(shm_errmsg, sizeof(shm_errmsg),
|
|
Packit Service |
155747 |
"%s: Unable to allocate shared memory for intra-node messaging.\n"
|
|
Packit Service |
155747 |
"%s: Delete stale shared memory files in /dev/shm.\n",
|
|
Packit Service |
155747 |
psmi_gethostname(), psmi_gethostname());
|
|
Packit Service |
155747 |
amsh_atexit();
|
|
Packit Service |
155747 |
if (psmi_write(2, shm_errmsg, strlen(shm_errmsg) + 1) == -1)
|
|
Packit Service |
155747 |
psmi_exit(2);
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
psmi_exit(1); /* XXX revisit this... there's probably a better way to exit */
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
if (signo == SIGSEGV) {
|
|
Packit Service |
155747 |
if (action_stash.SIGSEGV_old_act.sa_sigaction == (void*) SIG_DFL) {
|
|
Packit Service |
155747 |
psmi_sigaction(SIGSEGV, &action_stash.SIGSEGV_old_act, NULL);
|
|
Packit Service |
155747 |
raise(SIGSEGV);
|
|
Packit Service |
155747 |
struct sigaction act;
|
|
Packit Service |
155747 |
act.sa_sigaction = amsh_mmap_fault;
|
|
Packit Service |
155747 |
act.sa_flags = SA_SIGINFO;
|
|
Packit Service |
155747 |
psmi_sigaction(SIGSEGV, &act, NULL);
|
|
Packit Service |
155747 |
} else if (action_stash.SIGSEGV_old_act.sa_sigaction == (void*) SIG_IGN) {
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
action_stash.SIGSEGV_old_act.sa_sigaction(signo, siginfo, context);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else if (signo == SIGBUS) {
|
|
Packit Service |
155747 |
if (action_stash.SIGBUS_old_act.sa_sigaction == (void*) SIG_DFL) {
|
|
Packit Service |
155747 |
psmi_sigaction(SIGBUS, &action_stash.SIGBUS_old_act, NULL);
|
|
Packit Service |
155747 |
raise(SIGBUS);
|
|
Packit Service |
155747 |
struct sigaction act;
|
|
Packit Service |
155747 |
act.sa_sigaction = amsh_mmap_fault;
|
|
Packit Service |
155747 |
act.sa_flags = SA_SIGINFO;
|
|
Packit Service |
155747 |
psmi_sigaction(SIGBUS, &act, NULL);
|
|
Packit Service |
155747 |
} else if (action_stash.SIGBUS_old_act.sa_sigaction == (void*) SIG_IGN) {
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
action_stash.SIGBUS_old_act.sa_sigaction(signo, siginfo, context);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_exit(signo);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Create endpoint shared-memory object, containing ep's info
|
|
Packit Service |
155747 |
* and message queues.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
psm2_error_t psmi_shm_create(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_ep_t ep = ptl->ep;
|
|
Packit Service |
155747 |
char shmbuf[256];
|
|
Packit Service |
155747 |
void *mapptr;
|
|
Packit Service |
155747 |
size_t segsz;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
int shmfd = -1;
|
|
Packit Service |
155747 |
char *amsh_keyname;
|
|
Packit Service |
155747 |
int iterator;
|
|
Packit Service |
155747 |
/* Get which kassist mode to use. */
|
|
Packit Service |
155747 |
ptl->psmi_kassist_mode = psmi_get_kassist_mode();
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (_HFI_PRDBG_ON) {
|
|
Packit Service |
155747 |
_HFI_PRDBG_ALWAYS
|
|
Packit Service |
155747 |
("kassist_mode %d %s use_kassist %d\n",
|
|
Packit Service |
155747 |
ptl->psmi_kassist_mode,
|
|
Packit Service |
155747 |
psmi_kassist_getmode(ptl->psmi_kassist_mode),
|
|
Packit Service |
155747 |
(ptl->psmi_kassist_mode != PSMI_KASSIST_OFF));
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
segsz = am_ctl_sizeof_block();
|
|
Packit Service |
155747 |
for (iterator = 0; iterator <= INT_MAX; iterator++) {
|
|
Packit Service |
155747 |
snprintf(shmbuf,
|
|
Packit Service |
155747 |
sizeof(shmbuf),
|
|
Packit Service |
155747 |
"/psm2_shm.%ld%016lx%d",
|
|
Packit Service |
155747 |
(long int) getuid(),
|
|
Packit Service |
155747 |
ep->epid,
|
|
Packit Service |
155747 |
iterator);
|
|
Packit Service |
155747 |
amsh_keyname = psmi_strdup(NULL, shmbuf);
|
|
Packit Service |
155747 |
if (amsh_keyname == NULL) {
|
|
Packit Service |
155747 |
err = PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
shmfd =
|
|
Packit Service |
155747 |
shm_open(amsh_keyname, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
|
|
Packit Service |
155747 |
if (shmfd < 0) {
|
|
Packit Service |
155747 |
if (errno == EACCES && iterator < INT_MAX)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
else {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error creating shared "
|
|
Packit Service |
155747 |
"memory object in "
|
|
Packit Service |
155747 |
"shm_open: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
struct stat st;
|
|
Packit Service |
155747 |
if (fstat(shmfd, &st) == -1) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error validating "
|
|
Packit Service |
155747 |
"shared memory object "
|
|
Packit Service |
155747 |
"with fstat: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (getuid() == st.st_uid) {
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
err = PSM2_SHMEM_SEGMENT_ERR;
|
|
Packit Service |
155747 |
close(shmfd);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (err) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error creating shared memory object "
|
|
Packit Service |
155747 |
"in shm_open: namespace exhausted.");
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Now register the atexit handler for cleanup, whether master or slave */
|
|
Packit Service |
155747 |
atexit(amsh_atexit);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_PRDBG("Opened shmfile %s\n", amsh_keyname);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ftruncate(shmfd, segsz) != 0) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error setting size of shared memory object to %u bytes in "
|
|
Packit Service |
155747 |
"ftruncate: %s\n",
|
|
Packit Service |
155747 |
(uint32_t) segsz,
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
mapptr = mmap(NULL, segsz,
|
|
Packit Service |
155747 |
PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0);
|
|
Packit Service |
155747 |
if (mapptr == MAP_FAILED) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error mmapping shared memory: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
memset((void *) mapptr, 0, segsz); /* touch all of my pages */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Our own ep's info for ptl_am resides at the start of the
|
|
Packit Service |
155747 |
shm object. Other processes need some of this info to
|
|
Packit Service |
155747 |
understand the rest of the queue structure and other details. */
|
|
Packit Service |
155747 |
ptl->self_nodeinfo = (struct am_ctl_nodeinfo *) mapptr;
|
|
Packit Service |
155747 |
ptl->amsh_keyname = amsh_keyname;
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_shmbase = (uintptr_t) mapptr;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
if (shmfd >= 0) close(shmfd);
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psm2_error_t psmi_epdir_extend(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
struct am_ctl_nodeinfo *new = NULL;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
new = (struct am_ctl_nodeinfo *)
|
|
Packit Service |
155747 |
psmi_memalign(ptl->ep, PER_PEER_ENDPOINT, 64,
|
|
Packit Service |
155747 |
(ptl->am_ep_size + AMSH_DIRBLOCK_SIZE) *
|
|
Packit Service |
155747 |
sizeof(struct am_ctl_nodeinfo));
|
|
Packit Service |
155747 |
if (new == NULL)
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
memcpy(new, ptl->am_ep,
|
|
Packit Service |
155747 |
ptl->am_ep_size * sizeof(struct am_ctl_nodeinfo));
|
|
Packit Service |
155747 |
memset(new + ptl->am_ep_size, 0,
|
|
Packit Service |
155747 |
AMSH_DIRBLOCK_SIZE * sizeof(struct am_ctl_nodeinfo));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_free(ptl->am_ep);
|
|
Packit Service |
155747 |
ptl->am_ep = new;
|
|
Packit Service |
155747 |
ptl->am_ep_size += AMSH_DIRBLOCK_SIZE;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Unmap shm regions upon proper disconnect with other processes
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
psm2_error_t psmi_do_unmap(uintptr_t shmbase)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
if (munmap((void *)shmbase, am_ctl_sizeof_block())) {
|
|
Packit Service |
155747 |
err =
|
|
Packit Service |
155747 |
psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error with munmap of shared segment: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Map a remote process' shared memory object.
|
|
Packit Service |
155747 |
*
|
|
Packit Service |
155747 |
* If the remote process has a shared memory object available, add it to our own
|
|
Packit Service |
155747 |
* directory and return the shmidx. If the shared memory object does not exist,
|
|
Packit Service |
155747 |
* return -1, and the connect poll function will try to map again later.
|
|
Packit Service |
155747 |
*
|
|
Packit Service |
155747 |
* If force_remap is true, then clear the entry that matches the epid.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
psm2_error_t psmi_shm_map_remote(ptl_t *ptl_gen, psm2_epid_t epid, uint16_t *shmidx_o, int force_remap)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
int use_kassist;
|
|
Packit Service |
155747 |
uint16_t shmidx;
|
|
Packit Service |
155747 |
char shmbuf[256];
|
|
Packit Service |
155747 |
void *dest_mapptr;
|
|
Packit Service |
155747 |
size_t segsz;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
int dest_shmfd;
|
|
Packit Service |
155747 |
struct am_ctl_nodeinfo *dest_nodeinfo;
|
|
Packit Service |
155747 |
int iterator;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
shmidx = *shmidx_o = -1;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i <= ptl->max_ep_idx; i++) {
|
|
Packit Service |
155747 |
if (ptl->am_ep[i].epid == epid) {
|
|
Packit Service |
155747 |
if (force_remap) {
|
|
Packit Service |
155747 |
ptl->am_ep[i].epaddr = NULL;
|
|
Packit Service |
155747 |
ptl->am_ep[i].epid = 0;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
*shmidx_o = shmidx = i;
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
use_kassist = (ptl->psmi_kassist_mode != PSMI_KASSIST_OFF);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
segsz = am_ctl_sizeof_block();
|
|
Packit Service |
155747 |
for (iterator = 0; iterator <= INT_MAX; iterator++) {
|
|
Packit Service |
155747 |
snprintf(shmbuf,
|
|
Packit Service |
155747 |
sizeof(shmbuf),
|
|
Packit Service |
155747 |
"/psm2_shm.%ld%016lx%d",
|
|
Packit Service |
155747 |
(long int) getuid(),
|
|
Packit Service |
155747 |
epid,
|
|
Packit Service |
155747 |
iterator);
|
|
Packit Service |
155747 |
dest_shmfd = shm_open(shmbuf, O_RDWR, S_IRWXU);
|
|
Packit Service |
155747 |
if (dest_shmfd < 0) {
|
|
Packit Service |
155747 |
if (errno == EACCES && iterator < INT_MAX)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
else {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error opening remote "
|
|
Packit Service |
155747 |
"shared memory object "
|
|
Packit Service |
155747 |
"in shm_open: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
struct stat st;
|
|
Packit Service |
155747 |
if (fstat(dest_shmfd, &st) == -1) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error validating "
|
|
Packit Service |
155747 |
"shared memory object "
|
|
Packit Service |
155747 |
"with fstat: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (getuid() == st.st_uid) {
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
err = PSM2_SHMEM_SEGMENT_ERR;
|
|
Packit Service |
155747 |
close(dest_shmfd);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (err) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL,
|
|
Packit Service |
155747 |
PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error opening remote shared "
|
|
Packit Service |
155747 |
"memory object in shm_open: "
|
|
Packit Service |
155747 |
"namespace exhausted.");
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
dest_mapptr = mmap(NULL, segsz,
|
|
Packit Service |
155747 |
PROT_READ | PROT_WRITE, MAP_SHARED, dest_shmfd, 0);
|
|
Packit Service |
155747 |
if (dest_mapptr == MAP_FAILED) {
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error mmapping remote shared memory: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
close(dest_shmfd);
|
|
Packit Service |
155747 |
dest_nodeinfo = (struct am_ctl_nodeinfo *)dest_mapptr;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* We core dump right after here if we don't check the mmap */
|
|
Packit Service |
155747 |
action_stash.addr = dest_mapptr;
|
|
Packit Service |
155747 |
action_stash.len = segsz;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
struct sigaction act = { .sa_sigaction = amsh_mmap_fault, .sa_flags = SA_SIGINFO };
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
sigaction(SIGSEGV, &act, &action_stash.SIGSEGV_old_act);
|
|
Packit Service |
155747 |
sigaction(SIGBUS, &act, &action_stash.SIGBUS_old_act);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
volatile uint16_t *is_init = &dest_nodeinfo->is_init;
|
|
Packit Service |
155747 |
while (*is_init == 0)
|
|
Packit Service |
155747 |
usleep(1);
|
|
Packit Service |
155747 |
ips_sync_reads();
|
|
Packit Service |
155747 |
_HFI_PRDBG("Got a published remote dirpage page at "
|
|
Packit Service |
155747 |
"%p, size=%dn", dest_mapptr, (int)segsz);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
shmidx = -1;
|
|
Packit Service |
155747 |
if ((ptl->max_ep_idx + 1) == ptl->am_ep_size) {
|
|
Packit Service |
155747 |
err = psmi_epdir_extend(ptl_gen);
|
|
Packit Service |
155747 |
if (err)
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i <= ptl->max_ep_idx; i++) {
|
|
Packit Service |
155747 |
if (ptl->am_ep[i].epid != 0)
|
|
Packit Service |
155747 |
am_update_directory(&ptl->am_ep[i]);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
for (i = 0; i < ptl->am_ep_size; i++) {
|
|
Packit Service |
155747 |
psmi_assert(ptl->am_ep[i].epid != epid);
|
|
Packit Service |
155747 |
if (ptl->am_ep[i].epid == 0) {
|
|
Packit Service |
155747 |
ptl->am_ep[i].epid = epid;
|
|
Packit Service |
155747 |
ptl->am_ep[i].psm_verno = dest_nodeinfo->psm_verno;
|
|
Packit Service |
155747 |
ptl->am_ep[i].pid = dest_nodeinfo->pid;
|
|
Packit Service |
155747 |
if (use_kassist) {
|
|
Packit Service |
155747 |
/* If we are able to use CMA assume everyone
|
|
Packit Service |
155747 |
* else on the node can also use it.
|
|
Packit Service |
155747 |
* Advertise that CMA is active via the
|
|
Packit Service |
155747 |
* feature flag.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (cma_available()) {
|
|
Packit Service |
155747 |
ptl->am_ep[i].amsh_features |=
|
|
Packit Service |
155747 |
AMSH_HAVE_CMA;
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_CMA;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
ptl->psmi_kassist_mode =
|
|
Packit Service |
155747 |
PSMI_KASSIST_OFF;
|
|
Packit Service |
155747 |
use_kassist = 0;
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_NO_KASSIST;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_NO_KASSIST;
|
|
Packit Service |
155747 |
_HFI_PRDBG("KASSIST MODE: %s\n",
|
|
Packit Service |
155747 |
psmi_kassist_getmode(ptl->psmi_kassist_mode));
|
|
Packit Service |
155747 |
shmidx = *shmidx_o = i;
|
|
Packit Service |
155747 |
_HFI_PRDBG("Mapped epid %lx into shmidx %d\n", epid, shmidx);
|
|
Packit Service |
155747 |
ptl->am_ep[i].amsh_shmbase = (uintptr_t) dest_mapptr;
|
|
Packit Service |
155747 |
ptl->am_ep[i].amsh_qsizes = dest_nodeinfo->amsh_qsizes;
|
|
Packit Service |
155747 |
if (i > ptl->max_ep_idx)
|
|
Packit Service |
155747 |
ptl->max_ep_idx = i;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* install the old sighandler back */
|
|
Packit Service |
155747 |
sigaction(SIGSEGV, &action_stash.SIGSEGV_old_act, NULL);
|
|
Packit Service |
155747 |
sigaction(SIGBUS, &action_stash.SIGBUS_old_act, NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (shmidx == (uint16_t)-1)
|
|
Packit Service |
155747 |
err = psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Could not connect to local endpoint"); fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Initialize pointer structure and locks for endpoint shared-memory AM.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define AMSH_QSIZE(type) \
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(amsh_qelemsz.q ## type * amsh_qcounts.q ## type, \
|
|
Packit Service |
155747 |
PSMI_PAGESIZE)
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static psm2_error_t amsh_init_segment(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Preconditions */
|
|
Packit Service |
155747 |
psmi_assert_always(ptl != NULL);
|
|
Packit Service |
155747 |
psmi_assert_always(ptl->ep != NULL);
|
|
Packit Service |
155747 |
psmi_assert_always(ptl->epaddr != NULL);
|
|
Packit Service |
155747 |
psmi_assert_always(ptl->ep->epid != 0);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if ((err = psmi_shm_create(ptl_gen)))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_qsizes.qreqFifoShort = AMSH_QSIZE(reqFifoShort);
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_qsizes.qreqFifoLong = AMSH_QSIZE(reqFifoLong);
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_qsizes.qrepFifoShort = AMSH_QSIZE(repFifoShort);
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_qsizes.qrepFifoLong = AMSH_QSIZE(repFifoLong);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* We core dump right after here if we don't check the mmap */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
struct sigaction act;
|
|
Packit Service |
155747 |
act.sa_sigaction = amsh_mmap_fault;
|
|
Packit Service |
155747 |
act.sa_flags = SA_SIGINFO;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
sigaction(SIGSEGV, &act, &action_stash.SIGSEGV_old_act);
|
|
Packit Service |
155747 |
sigaction(SIGBUS, &act, &action_stash.SIGBUS_old_act);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* Now that we know our epid, update it in the shmidx array
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
ptl->reqH.base = ptl->reqH.head = ptl->reqH.end = NULL;
|
|
Packit Service |
155747 |
ptl->repH.base = ptl->repH.head = ptl->repH.end = NULL;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
am_update_directory(ptl->self_nodeinfo);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->reqH.head = ptl->reqH.base = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
(((uintptr_t)ptl->self_nodeinfo->qdir.qreqFifoShort));
|
|
Packit Service |
155747 |
ptl->reqH.end = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
(((uintptr_t)ptl->self_nodeinfo->qdir.qreqFifoShort) +
|
|
Packit Service |
155747 |
amsh_qcounts.qreqFifoShort * amsh_qelemsz.qreqFifoShort);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->repH.head = ptl->repH.base = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
(((uintptr_t)ptl->self_nodeinfo->qdir.qrepFifoShort));
|
|
Packit Service |
155747 |
ptl->repH.end = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
(((uintptr_t)ptl->self_nodeinfo->qdir.qrepFifoShort) +
|
|
Packit Service |
155747 |
amsh_qcounts.qrepFifoShort * amsh_qelemsz.qrepFifoShort);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
am_ctl_qhdr_init(&ptl->self_nodeinfo->qdir.qreqH->shortq,
|
|
Packit Service |
155747 |
amsh_qcounts.qreqFifoShort,
|
|
Packit Service |
155747 |
amsh_qelemsz.qreqFifoShort);
|
|
Packit Service |
155747 |
am_ctl_qhdr_init(&ptl->self_nodeinfo->qdir.qreqH->longbulkq,
|
|
Packit Service |
155747 |
amsh_qcounts.qreqFifoLong, amsh_qelemsz.qreqFifoLong);
|
|
Packit Service |
155747 |
am_ctl_qhdr_init(&ptl->self_nodeinfo->qdir.qrepH->shortq,
|
|
Packit Service |
155747 |
amsh_qcounts.qrepFifoShort,
|
|
Packit Service |
155747 |
amsh_qelemsz.qrepFifoShort);
|
|
Packit Service |
155747 |
am_ctl_qhdr_init(&ptl->self_nodeinfo->qdir.qrepH->longbulkq,
|
|
Packit Service |
155747 |
amsh_qcounts.qrepFifoLong, amsh_qelemsz.qrepFifoLong);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Set bulkidx in every bulk packet */
|
|
Packit Service |
155747 |
am_ctl_bulkpkt_init(ptl->self_nodeinfo->qdir.qreqFifoLong,
|
|
Packit Service |
155747 |
amsh_qelemsz.qreqFifoLong,
|
|
Packit Service |
155747 |
amsh_qcounts.qreqFifoLong);
|
|
Packit Service |
155747 |
am_ctl_bulkpkt_init(ptl->self_nodeinfo->qdir.qrepFifoLong,
|
|
Packit Service |
155747 |
amsh_qelemsz.qrepFifoLong,
|
|
Packit Service |
155747 |
amsh_qcounts.qrepFifoLong);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* install the old sighandler back */
|
|
Packit Service |
155747 |
sigaction(SIGSEGV, &action_stash.SIGSEGV_old_act, NULL);
|
|
Packit Service |
155747 |
sigaction(SIGBUS, &action_stash.SIGBUS_old_act, NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psm2_error_t psmi_shm_detach(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
uintptr_t shmbase;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ptl->self_nodeinfo == NULL)
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("unlinking shm file %s\n", ptl->amsh_keyname + 1);
|
|
Packit Service |
155747 |
shmbase = ptl->self_nodeinfo->amsh_shmbase;
|
|
Packit Service |
155747 |
shm_unlink(ptl->amsh_keyname);
|
|
Packit Service |
155747 |
psmi_free(ptl->amsh_keyname);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (munmap((void *)shmbase, am_ctl_sizeof_block())) {
|
|
Packit Service |
155747 |
err =
|
|
Packit Service |
155747 |
psmi_handle_error(NULL, PSM2_SHMEM_SEGMENT_ERR,
|
|
Packit Service |
155747 |
"Error with munmap of shared segment: %s",
|
|
Packit Service |
155747 |
strerror(errno));
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
ptl->self_nodeinfo = NULL;
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Update locally shared-pointer directory. The directory must be
|
|
Packit Service |
155747 |
* updated when a new epaddr is connected to or on every epaddr already
|
|
Packit Service |
155747 |
* connected to whenever the shared memory segment is relocated via mremap.
|
|
Packit Service |
155747 |
*
|
|
Packit Service |
155747 |
* @param epaddr Endpoint address for which to update local directory.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void am_update_directory(struct am_ctl_nodeinfo *nodeinfo)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
uintptr_t base_this;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
base_this = nodeinfo->amsh_shmbase +
|
|
Packit Service |
155747 |
AMSH_BLOCK_HEADER_SIZE;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Request queues */
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqH = (am_ctl_blockhdr_t *) base_this;
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqFifoShort = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
((uintptr_t) nodeinfo->qdir.qreqH +
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(sizeof(am_ctl_blockhdr_t), PSMI_PAGESIZE));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqFifoLong = (am_pkt_bulk_t *)
|
|
Packit Service |
155747 |
((uintptr_t) nodeinfo->qdir.qreqFifoShort +
|
|
Packit Service |
155747 |
nodeinfo->amsh_qsizes.qreqFifoShort);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Reply queues */
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepH = (am_ctl_blockhdr_t *)
|
|
Packit Service |
155747 |
((uintptr_t) nodeinfo->qdir.qreqFifoLong +
|
|
Packit Service |
155747 |
nodeinfo->amsh_qsizes.qreqFifoLong);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepFifoShort = (am_pkt_short_t *)
|
|
Packit Service |
155747 |
((uintptr_t) nodeinfo->qdir.qrepH +
|
|
Packit Service |
155747 |
PSMI_ALIGNUP(sizeof(am_ctl_blockhdr_t), PSMI_PAGESIZE));
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepFifoLong = (am_pkt_bulk_t *)
|
|
Packit Service |
155747 |
((uintptr_t) nodeinfo->qdir.qrepFifoShort +
|
|
Packit Service |
155747 |
nodeinfo->amsh_qsizes.qrepFifoShort);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("epaddr=%p Request Hdr=%p,Pkt=%p,Long=%p\n",
|
|
Packit Service |
155747 |
nodeinfo->epaddr,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqH,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqFifoShort,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qreqFifoLong);
|
|
Packit Service |
155747 |
_HFI_VDBG("epaddr=%p Reply Hdr=%p,Pkt=%p,Long=%p\n",
|
|
Packit Service |
155747 |
nodeinfo->epaddr,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepH,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepFifoShort,
|
|
Packit Service |
155747 |
nodeinfo->qdir.qrepFifoLong);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Sanity check */
|
|
Packit Service |
155747 |
uintptr_t base_next =
|
|
Packit Service |
155747 |
(uintptr_t) nodeinfo->qdir.qrepFifoLong +
|
|
Packit Service |
155747 |
nodeinfo->amsh_qsizes.qrepFifoLong;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert_always(base_next - base_this <= am_ctl_sizeof_block());
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* ep_epid_share_memory wrapper */
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
int amsh_epid_reachable(ptl_t *ptl_gen, psm2_epid_t epid)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
int result;
|
|
Packit Service |
155747 |
psm2_error_t err;
|
|
Packit Service |
155747 |
err = psm2_ep_epid_share_memory(ptl->ep, epid, &result);
|
|
Packit Service |
155747 |
psmi_assert_always(err == PSM2_OK);
|
|
Packit Service |
155747 |
return result;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_epaddr_add(ptl_t *ptl_gen, psm2_epid_t epid, uint16_t shmidx, psm2_epaddr_t *epaddr_o)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr;
|
|
Packit Service |
155747 |
am_epaddr_t *amaddr;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert(psmi_epid_lookup(ptl->ep, epid) == NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* The self PTL handles loopback communication. */
|
|
Packit Service |
155747 |
psmi_assert(epid != ptl->epid);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* note the size of the memory is am_epaddr_t */
|
|
Packit Service |
155747 |
epaddr = (psm2_epaddr_t) psmi_calloc(ptl->ep,
|
|
Packit Service |
155747 |
PER_PEER_ENDPOINT, 1,
|
|
Packit Service |
155747 |
sizeof(am_epaddr_t));
|
|
Packit Service |
155747 |
if (epaddr == NULL) {
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
psmi_assert_always(ptl->am_ep[shmidx].epaddr == NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if ((err = psmi_epid_set_hostname(psm2_epid_nid(epid),
|
|
Packit Service |
155747 |
psmi_gethostname(), 0)))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
epaddr->ptlctl = ptl->ctl;
|
|
Packit Service |
155747 |
epaddr->epid = epid;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* convert to am_epaddr_t */
|
|
Packit Service |
155747 |
amaddr = (am_epaddr_t *) epaddr;
|
|
Packit Service |
155747 |
/* tell the other endpoint their location in our directory */
|
|
Packit Service |
155747 |
amaddr->shmidx = shmidx;
|
|
Packit Service |
155747 |
/* we haven't connected yet, so we can't give them the same hint */
|
|
Packit Service |
155747 |
amaddr->return_shmidx = -1;
|
|
Packit Service |
155747 |
amaddr->cstate_outgoing = AMSH_CSTATE_OUTGOING_NONE;
|
|
Packit Service |
155747 |
amaddr->cstate_incoming = AMSH_CSTATE_INCOMING_NONE;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* other setup */
|
|
Packit Service |
155747 |
ptl->am_ep[shmidx].epaddr = epaddr;
|
|
Packit Service |
155747 |
am_update_directory(&ptl->am_ep[shmidx]);
|
|
Packit Service |
155747 |
/* Finally, add to table */
|
|
Packit Service |
155747 |
if ((err = psmi_epid_add(ptl->ep, epid, epaddr)))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
_HFI_VDBG("epaddr=%s added to ptl=%p\n",
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epid), ptl);
|
|
Packit Service |
155747 |
*epaddr_o = epaddr;
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
if (epaddr != ptl->epaddr)
|
|
Packit Service |
155747 |
psmi_free(epaddr);
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
amsh_epaddr_update(ptl_t *ptl_gen, psm2_epaddr_t epaddr)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
am_epaddr_t *amaddr;
|
|
Packit Service |
155747 |
uint16_t shmidx;
|
|
Packit Service |
155747 |
struct am_ctl_nodeinfo *nodeinfo;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
amaddr = (am_epaddr_t *) epaddr;
|
|
Packit Service |
155747 |
shmidx = amaddr->shmidx;
|
|
Packit Service |
155747 |
nodeinfo = (struct am_ctl_nodeinfo *) ptl->am_ep[shmidx].amsh_shmbase;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* restart the connection process */
|
|
Packit Service |
155747 |
amaddr->return_shmidx = -1;
|
|
Packit Service |
155747 |
amaddr->cstate_outgoing = AMSH_CSTATE_OUTGOING_NONE;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* wait for the other process to init again */
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
volatile uint16_t *is_init = &nodeinfo->is_init;
|
|
Packit Service |
155747 |
while (*is_init == 0)
|
|
Packit Service |
155747 |
usleep(1);
|
|
Packit Service |
155747 |
ips_sync_reads();
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* get the updated values from the new nodeinfo page */
|
|
Packit Service |
155747 |
ptl->am_ep[shmidx].psm_verno = nodeinfo->psm_verno;
|
|
Packit Service |
155747 |
ptl->am_ep[shmidx].pid = nodeinfo->pid;
|
|
Packit Service |
155747 |
ptl->am_ep[shmidx].amsh_qsizes = nodeinfo->amsh_qsizes;
|
|
Packit Service |
155747 |
am_update_directory(&ptl->am_ep[shmidx]);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
struct ptl_connection_req {
|
|
Packit Service |
155747 |
int isdone;
|
|
Packit Service |
155747 |
int op; /* connect or disconnect */
|
|
Packit Service |
155747 |
int numep;
|
|
Packit Service |
155747 |
int numep_left;
|
|
Packit Service |
155747 |
int phase;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int *epid_mask;
|
|
Packit Service |
155747 |
const psm2_epid_t *epids; /* input epid list */
|
|
Packit Service |
155747 |
psm2_epaddr_t *epaddr;
|
|
Packit Service |
155747 |
psm2_error_t *errors; /* inout errors */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Used for connect/disconnect */
|
|
Packit Service |
155747 |
psm2_amarg_t args[4];
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void amsh_free_epaddr(psm2_epaddr_t epaddr)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psmi_epid_remove(epaddr->ptlctl->ep, epaddr->epid);
|
|
Packit Service |
155747 |
psmi_free(epaddr);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define PTL_OP_CONNECT 0
|
|
Packit Service |
155747 |
#define PTL_OP_DISCONNECT 1
|
|
Packit Service |
155747 |
#define PTL_OP_ABORT 2
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_connreq_init(ptl_t *ptl_gen, int op, /* connect, disconnect or abort */
|
|
Packit Service |
155747 |
int numep, const psm2_epid_t *array_of_epid, /* non-NULL on connect */
|
|
Packit Service |
155747 |
const int array_of_epid_mask[],
|
|
Packit Service |
155747 |
psm2_error_t *array_of_errors,
|
|
Packit Service |
155747 |
psm2_epaddr_t *array_of_epaddr,
|
|
Packit Service |
155747 |
struct ptl_connection_req **req_o)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
int i, cstate;
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr;
|
|
Packit Service |
155747 |
psm2_epid_t epid;
|
|
Packit Service |
155747 |
struct ptl_connection_req *req = NULL;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
req = (struct ptl_connection_req *)
|
|
Packit Service |
155747 |
psmi_calloc(ptl->ep, PER_PEER_ENDPOINT, 1,
|
|
Packit Service |
155747 |
sizeof(struct ptl_connection_req));
|
|
Packit Service |
155747 |
if (req == NULL)
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
req->isdone = 0;
|
|
Packit Service |
155747 |
req->op = op;
|
|
Packit Service |
155747 |
req->numep = numep;
|
|
Packit Service |
155747 |
req->numep_left = 0;
|
|
Packit Service |
155747 |
req->phase = ptl->connect_phase;
|
|
Packit Service |
155747 |
req->epid_mask = (int *)
|
|
Packit Service |
155747 |
psmi_calloc(ptl->ep, PER_PEER_ENDPOINT, numep, sizeof(int));
|
|
Packit Service |
155747 |
if (req->epid_mask == NULL) {
|
|
Packit Service |
155747 |
psmi_free(req);
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
req->epaddr = array_of_epaddr;
|
|
Packit Service |
155747 |
req->epids = array_of_epid;
|
|
Packit Service |
155747 |
req->errors = array_of_errors;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* First check if there's really something to connect/disconnect
|
|
Packit Service |
155747 |
* for this PTL */
|
|
Packit Service |
155747 |
for (i = 0; i < numep; i++) {
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_NONE; /* no connect by default */
|
|
Packit Service |
155747 |
if (!array_of_epid_mask[i])
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
if (op == PTL_OP_CONNECT) {
|
|
Packit Service |
155747 |
epid = array_of_epid[i];
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Connect only to other processes reachable by shared memory.
|
|
Packit Service |
155747 |
The self PTL handles loopback communication, so explicitly
|
|
Packit Service |
155747 |
refuse to connect to self. */
|
|
Packit Service |
155747 |
if (!amsh_epid_reachable(ptl_gen, epid)
|
|
Packit Service |
155747 |
|| epid == ptl->epid) {
|
|
Packit Service |
155747 |
array_of_errors[i] = PSM2_EPID_UNREACHABLE;
|
|
Packit Service |
155747 |
array_of_epaddr[i] = NULL;
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("looking at epid %llx\n",
|
|
Packit Service |
155747 |
(unsigned long long)epid);
|
|
Packit Service |
155747 |
epaddr = psmi_epid_lookup(ptl->ep, epid);
|
|
Packit Service |
155747 |
if (epaddr != NULL) {
|
|
Packit Service |
155747 |
if (epaddr->ptlctl->ptl != ptl_gen) {
|
|
Packit Service |
155747 |
array_of_errors[i] =
|
|
Packit Service |
155747 |
PSM2_EPID_UNREACHABLE;
|
|
Packit Service |
155747 |
array_of_epaddr[i] = NULL;
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) epaddr)->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_ESTABLISHED) {
|
|
Packit Service |
155747 |
array_of_epaddr[i] = epaddr;
|
|
Packit Service |
155747 |
array_of_errors[i] = PSM2_OK;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_assert(cstate ==
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_NONE);
|
|
Packit Service |
155747 |
array_of_errors[i] = PSM2_TIMEOUT;
|
|
Packit Service |
155747 |
array_of_epaddr[i] = epaddr;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_PREREQ;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_PREREQ;
|
|
Packit Service |
155747 |
array_of_epaddr[i] = NULL;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else { /* disc or abort */
|
|
Packit Service |
155747 |
epaddr = array_of_epaddr[i];
|
|
Packit Service |
155747 |
if (epaddr->ptlctl->ptl != ptl_gen)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert(epaddr != NULL);
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) epaddr)->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_ESTABLISHED) {
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_PREREQ;
|
|
Packit Service |
155747 |
_HFI_VDBG
|
|
Packit Service |
155747 |
("Just set index %d to AMSH_CMASK_PREREQ\n",
|
|
Packit Service |
155747 |
i);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
/* XXX undef ? */
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (req->epid_mask[i] != AMSH_CMASK_NONE)
|
|
Packit Service |
155747 |
req->numep_left++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (req->numep_left == 0) { /* nothing to do */
|
|
Packit Service |
155747 |
psmi_free(req->epid_mask);
|
|
Packit Service |
155747 |
psmi_free(req);
|
|
Packit Service |
155747 |
_HFI_VDBG("Nothing to connect, bump up phase\n");
|
|
Packit Service |
155747 |
ptl->connect_phase++;
|
|
Packit Service |
155747 |
*req_o = NULL;
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
*req_o = req;
|
|
Packit Service |
155747 |
return PSM2_OK_NO_PROGRESS;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_connreq_poll(ptl_t *ptl_gen, struct ptl_connection_req *req)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
int i, j, cstate;
|
|
Packit Service |
155747 |
uint16_t shmidx = (uint16_t)-1;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
psm2_epid_t epid;
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (req == NULL || req->isdone)
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert_always(ptl->connect_phase == req->phase);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (req->op == PTL_OP_DISCONNECT || req->op == PTL_OP_ABORT) {
|
|
Packit Service |
155747 |
for (i = 0; i < req->numep; i++) {
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_NONE ||
|
|
Packit Service |
155747 |
req->epid_mask[i] == AMSH_CMASK_DONE)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
epaddr = req->epaddr[i];
|
|
Packit Service |
155747 |
psmi_assert(epaddr != NULL);
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_PREREQ) {
|
|
Packit Service |
155747 |
shmidx = ((am_epaddr_t *) epaddr)->shmidx;
|
|
Packit Service |
155747 |
/* Make sure the target of the disconnect is still there */
|
|
Packit Service |
155747 |
if (ptl->am_ep[shmidx].
|
|
Packit Service |
155747 |
epid != epaddr->epid) {
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_NONE;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_PREREQ) {
|
|
Packit Service |
155747 |
req->args[0].u16w0 = PSMI_AM_DISC_REQ;
|
|
Packit Service |
155747 |
req->args[0].u16w1 = shmidx;
|
|
Packit Service |
155747 |
req->args[0].u32w1 = ptl->connect_phase;
|
|
Packit Service |
155747 |
req->args[1].u64w0 = (uint64_t) ptl->epid;
|
|
Packit Service |
155747 |
psmi_assert(shmidx != (uint16_t)-1);
|
|
Packit Service |
155747 |
req->args[2].u32w0 = getpid();
|
|
Packit Service |
155747 |
req->args[2].u32w1 = PSM2_OK;
|
|
Packit Service |
155747 |
req->args[3].u64w0 =
|
|
Packit Service |
155747 |
(uint64_t) (uintptr_t) &req->errors[i];
|
|
Packit Service |
155747 |
psmi_amsh_short_request(ptl_gen, epaddr,
|
|
Packit Service |
155747 |
amsh_conn_handler_hidx,
|
|
Packit Service |
155747 |
req->args, 4, NULL, 0,
|
|
Packit Service |
155747 |
0);
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_DISC_REQUESTED;
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Only munmap if we have nothing more to
|
|
Packit Service |
155747 |
* communicate with the other node, i.e. we
|
|
Packit Service |
155747 |
* already recieved a disconnect req from the
|
|
Packit Service |
155747 |
* other node.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
if (((am_epaddr_t *) epaddr)->cstate_incoming ==
|
|
Packit Service |
155747 |
AMSH_CSTATE_INCOMING_DISC_REQUESTED)
|
|
Packit Service |
155747 |
err = psmi_do_unmap(ptl->am_ep[shmidx].amsh_shmbase);
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_POSTREQ;
|
|
Packit Service |
155747 |
} else if (req->epid_mask[i] == AMSH_CMASK_POSTREQ) {
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) epaddr)->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_DISC_REPLIED) {
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_NONE;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
/* First see if we've made progress on any postreqs */
|
|
Packit Service |
155747 |
int n_prereq = 0;
|
|
Packit Service |
155747 |
for (i = 0; i < req->numep; i++) {
|
|
Packit Service |
155747 |
int cstate;
|
|
Packit Service |
155747 |
if (req->epid_mask[i] != AMSH_CMASK_POSTREQ) {
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_PREREQ)
|
|
Packit Service |
155747 |
n_prereq++;
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
epaddr = req->epaddr[i];
|
|
Packit Service |
155747 |
psmi_assert(epaddr != NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* detect if a race has occurred on due to re-using an
|
|
Packit Service |
155747 |
* old shm file - if so, restart the connection */
|
|
Packit Service |
155747 |
shmidx = ((am_epaddr_t *) epaddr)->shmidx;
|
|
Packit Service |
155747 |
if (ptl->am_ep[shmidx].pid !=
|
|
Packit Service |
155747 |
((struct am_ctl_nodeinfo *) ptl->am_ep[shmidx].amsh_shmbase)->pid) {
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_PREREQ;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_NONE;
|
|
Packit Service |
155747 |
n_prereq++;
|
|
Packit Service |
155747 |
amsh_epaddr_update(ptl_gen, epaddr);
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) epaddr)->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_REPLIED) {
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_ESTABLISHED;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (n_prereq > 0) {
|
|
Packit Service |
155747 |
psmi_assert(req->numep_left > 0);
|
|
Packit Service |
155747 |
/* Go through the list of peers we need to connect to and find out
|
|
Packit Service |
155747 |
* if they each shared ep is mapped into shm */
|
|
Packit Service |
155747 |
for (i = 0; i < req->numep; i++) {
|
|
Packit Service |
155747 |
if (req->epid_mask[i] != AMSH_CMASK_PREREQ)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
epid = req->epids[i];
|
|
Packit Service |
155747 |
epaddr = req->epaddr[i];
|
|
Packit Service |
155747 |
/* Go through mapped epids and find the epid we're looking for */
|
|
Packit Service |
155747 |
for (shmidx = -1, j = 0;
|
|
Packit Service |
155747 |
j <= ptl->max_ep_idx; j++) {
|
|
Packit Service |
155747 |
/* epid is connected and ready to go */
|
|
Packit Service |
155747 |
if (ptl->am_ep[j].
|
|
Packit Service |
155747 |
epid == epid) {
|
|
Packit Service |
155747 |
shmidx = j;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (shmidx == (uint16_t)-1) {
|
|
Packit Service |
155747 |
/* Couldn't find peer's epid in dirpage.
|
|
Packit Service |
155747 |
Check shmdir to see if epid is up now. */
|
|
Packit Service |
155747 |
if ((err = psmi_shm_map_remote(ptl_gen, epid, &shmidx, 0))) {
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
/* Before we even send the request out, check to see if
|
|
Packit Service |
155747 |
* versions are interoperable */
|
|
Packit Service |
155747 |
if (!psmi_verno_isinteroperable
|
|
Packit Service |
155747 |
(ptl->am_ep[shmidx].
|
|
Packit Service |
155747 |
psm_verno)) {
|
|
Packit Service |
155747 |
char buf[32];
|
|
Packit Service |
155747 |
uint16_t their_verno =
|
|
Packit Service |
155747 |
ptl->am_ep[shmidx].
|
|
Packit Service |
155747 |
psm_verno;
|
|
Packit Service |
155747 |
snprintf(buf, sizeof(buf), "%d.%d",
|
|
Packit Service |
155747 |
PSMI_VERNO_GET_MAJOR
|
|
Packit Service |
155747 |
(their_verno),
|
|
Packit Service |
155747 |
PSMI_VERNO_GET_MINOR
|
|
Packit Service |
155747 |
(their_verno));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_INFO("Local endpoint id %" PRIx64
|
|
Packit Service |
155747 |
" has version %s "
|
|
Packit Service |
155747 |
"which is not supported by library version %d.%d",
|
|
Packit Service |
155747 |
epid, buf, PSM2_VERNO_MAJOR,
|
|
Packit Service |
155747 |
PSM2_VERNO_MINOR);
|
|
Packit Service |
155747 |
req->errors[i] =
|
|
Packit Service |
155747 |
PSM2_EPID_INVALID_VERSION;
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (epaddr != NULL) {
|
|
Packit Service |
155747 |
psmi_assert(((am_epaddr_t *) epaddr)->
|
|
Packit Service |
155747 |
shmidx == shmidx);
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
if ((epaddr =
|
|
Packit Service |
155747 |
psmi_epid_lookup(ptl->ep,
|
|
Packit Service |
155747 |
epid)) == NULL) {
|
|
Packit Service |
155747 |
if ((err =
|
|
Packit Service |
155747 |
amsh_epaddr_add(ptl_gen, epid, shmidx,
|
|
Packit Service |
155747 |
&epaddr))) {
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
/* Remote pid is unknown at the moment */
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->pid =
|
|
Packit Service |
155747 |
AMSH_PID_UNKNOWN;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
req->epaddr[i] = epaddr;
|
|
Packit Service |
155747 |
req->args[0].u16w0 = PSMI_AM_CONN_REQ;
|
|
Packit Service |
155747 |
/* tell the other process its shmidx here */
|
|
Packit Service |
155747 |
req->args[0].u16w1 = shmidx;
|
|
Packit Service |
155747 |
req->args[0].u32w1 = ptl->connect_phase;
|
|
Packit Service |
155747 |
req->args[1].u64w0 = (uint64_t) ptl->epid;
|
|
Packit Service |
155747 |
req->args[2].u32w0 = getpid();
|
|
Packit Service |
155747 |
req->args[2].u32w1 = PSM2_OK;
|
|
Packit Service |
155747 |
req->args[3].u64w0 =
|
|
Packit Service |
155747 |
(uint64_t) (uintptr_t) &req->errors[i];
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_POSTREQ;
|
|
Packit Service |
155747 |
psmi_amsh_short_request(ptl_gen, epaddr,
|
|
Packit Service |
155747 |
amsh_conn_handler_hidx,
|
|
Packit Service |
155747 |
req->args, 4, NULL, 0,
|
|
Packit Service |
155747 |
0);
|
|
Packit Service |
155747 |
_HFI_PRDBG("epaddr=%p, epid=%" PRIx64
|
|
Packit Service |
155747 |
" at shmidx=%d\n", epaddr, epid,
|
|
Packit Service |
155747 |
shmidx);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (req->numep_left == 0) { /* we're all done */
|
|
Packit Service |
155747 |
req->isdone = 1;
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
sched_yield();
|
|
Packit Service |
155747 |
return PSM2_OK_NO_PROGRESS;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_connreq_fini(ptl_t *ptl_gen, struct ptl_connection_req *req)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Wherever we are at in our connect process, we've been instructed to
|
|
Packit Service |
155747 |
* finish the connection process */
|
|
Packit Service |
155747 |
if (req == NULL)
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* This prevents future connect replies from referencing data structures
|
|
Packit Service |
155747 |
* that disappeared */
|
|
Packit Service |
155747 |
ptl->connect_phase++;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* First process any leftovers in postreq or prereq */
|
|
Packit Service |
155747 |
for (i = 0; i < req->numep; i++) {
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_NONE)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
else if (req->epid_mask[i] == AMSH_CMASK_POSTREQ) {
|
|
Packit Service |
155747 |
int cstate;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) req->epaddr[i])->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_REPLIED) {
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
((am_epaddr_t *) req->epaddr[i])->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_ESTABLISHED;
|
|
Packit Service |
155747 |
} else { /* never actually got reply */
|
|
Packit Service |
155747 |
req->errors[i] = PSM2_TIMEOUT;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
/* If we couldn't go from prereq to postreq, that means we couldn't
|
|
Packit Service |
155747 |
* find the shmidx for an epid in time. This can only be a case of
|
|
Packit Service |
155747 |
* time out */
|
|
Packit Service |
155747 |
else if (req->epid_mask[i] == AMSH_CMASK_PREREQ) {
|
|
Packit Service |
155747 |
req->errors[i] = PSM2_TIMEOUT;
|
|
Packit Service |
155747 |
req->numep_left--;
|
|
Packit Service |
155747 |
req->epid_mask[i] = AMSH_CMASK_DONE;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Whatever is left can only be in DONE or NONE state */
|
|
Packit Service |
155747 |
for (i = 0; i < req->numep; i++) {
|
|
Packit Service |
155747 |
if (req->epid_mask[i] == AMSH_CMASK_NONE)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
psmi_assert(req->epid_mask[i] == AMSH_CMASK_DONE);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
err = psmi_error_cmp(err, req->errors[i]);
|
|
Packit Service |
155747 |
/* XXX TODO: Report errors in connection. */
|
|
Packit Service |
155747 |
/* Only free epaddr if they have disconnected from us */
|
|
Packit Service |
155747 |
int cstate = ((am_epaddr_t *) req->epaddr[i])->cstate_incoming;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_INCOMING_DISC_REQUESTED) {
|
|
Packit Service |
155747 |
if (req->op == PTL_OP_DISCONNECT || req->op == PTL_OP_ABORT) {
|
|
Packit Service |
155747 |
psmi_assert(req->epaddr[i] != NULL);
|
|
Packit Service |
155747 |
amsh_free_epaddr(req->epaddr[i]);
|
|
Packit Service |
155747 |
req->epaddr[i] = NULL;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_free(req->epid_mask);
|
|
Packit Service |
155747 |
psmi_free(req);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Wrapper for 2.0's use of connect/disconnect. The plan is to move the
|
|
Packit Service |
155747 |
* init/poll/fini interface up to the PTL level for 2.2 */
|
|
Packit Service |
155747 |
#define CONNREQ_ZERO_POLLS_BEFORE_YIELD 20
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_connreq_wrap(ptl_t *ptl_gen, int op,
|
|
Packit Service |
155747 |
int numep,
|
|
Packit Service |
155747 |
const psm2_epid_t *array_of_epid,
|
|
Packit Service |
155747 |
const int array_of_epid_mask[],
|
|
Packit Service |
155747 |
psm2_error_t *array_of_errors,
|
|
Packit Service |
155747 |
psm2_epaddr_t *array_of_epaddr, uint64_t timeout_ns)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err;
|
|
Packit Service |
155747 |
uint64_t t_start;
|
|
Packit Service |
155747 |
struct ptl_connection_req *req;
|
|
Packit Service |
155747 |
int num_polls_noprogress = 0;
|
|
Packit Service |
155747 |
static int shm_polite_attach = -1;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (shm_polite_attach == -1) {
|
|
Packit Service |
155747 |
char *p = getenv("PSM2_SHM_POLITE_ATTACH");
|
|
Packit Service |
155747 |
if (p && *p && atoi(p) != 0) {
|
|
Packit Service |
155747 |
fprintf(stderr, "%s: Using Polite SHM segment attach\n",
|
|
Packit Service |
155747 |
psmi_gethostname());
|
|
Packit Service |
155747 |
shm_polite_attach = 1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
shm_polite_attach = 0;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Initialize */
|
|
Packit Service |
155747 |
err = amsh_ep_connreq_init(ptl_gen, op, numep,
|
|
Packit Service |
155747 |
array_of_epid, array_of_epid_mask,
|
|
Packit Service |
155747 |
array_of_errors, array_of_epaddr, &req;;
|
|
Packit Service |
155747 |
if (err != PSM2_OK_NO_PROGRESS) /* Either we're all done with connect or
|
|
Packit Service |
155747 |
* there was an error */
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Poll until either
|
|
Packit Service |
155747 |
* 1. We time out
|
|
Packit Service |
155747 |
* 2. We are done with connecting
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
t_start = get_cycles();
|
|
Packit Service |
155747 |
do {
|
|
Packit Service |
155747 |
psmi_poll_internal(ptl->ep, 1);
|
|
Packit Service |
155747 |
err = amsh_ep_connreq_poll(ptl_gen, req);
|
|
Packit Service |
155747 |
if (err == PSM2_OK)
|
|
Packit Service |
155747 |
break; /* Finished before timeout */
|
|
Packit Service |
155747 |
else if (err != PSM2_OK_NO_PROGRESS) {
|
|
Packit Service |
155747 |
psmi_free(req->epid_mask);
|
|
Packit Service |
155747 |
psmi_free(req);
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
} else if (shm_polite_attach &&
|
|
Packit Service |
155747 |
++num_polls_noprogress ==
|
|
Packit Service |
155747 |
CONNREQ_ZERO_POLLS_BEFORE_YIELD) {
|
|
Packit Service |
155747 |
num_polls_noprogress = 0;
|
|
Packit Service |
155747 |
PSMI_YIELD(ptl->ep->mq->progress_lock);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
while (psmi_cycles_left(t_start, timeout_ns));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
err = amsh_ep_connreq_fini(ptl_gen, req);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_connect(ptl_t *ptl,
|
|
Packit Service |
155747 |
int numep,
|
|
Packit Service |
155747 |
const psm2_epid_t *array_of_epid,
|
|
Packit Service |
155747 |
const int array_of_epid_mask[],
|
|
Packit Service |
155747 |
psm2_error_t *array_of_errors,
|
|
Packit Service |
155747 |
psm2_epaddr_t *array_of_epaddr, uint64_t timeout_ns)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return amsh_ep_connreq_wrap(ptl, PTL_OP_CONNECT, numep, array_of_epid,
|
|
Packit Service |
155747 |
array_of_epid_mask, array_of_errors,
|
|
Packit Service |
155747 |
array_of_epaddr, timeout_ns);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_ep_disconnect(ptl_t *ptl, int force, int numep,
|
|
Packit Service |
155747 |
psm2_epaddr_t array_of_epaddr[],
|
|
Packit Service |
155747 |
const int array_of_epaddr_mask[],
|
|
Packit Service |
155747 |
psm2_error_t array_of_errors[], uint64_t timeout_ns)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return amsh_ep_connreq_wrap(ptl,
|
|
Packit Service |
155747 |
force ? PTL_OP_ABORT : PTL_OP_DISCONNECT,
|
|
Packit Service |
155747 |
numep, NULL, array_of_epaddr_mask,
|
|
Packit Service |
155747 |
array_of_errors,
|
|
Packit Service |
155747 |
array_of_epaddr,
|
|
Packit Service |
155747 |
timeout_ns);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#undef CSWAP
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
int32_t
|
|
Packit Service |
155747 |
cswap(volatile int32_t *p, int32_t old_value, int32_t new_value))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
asm volatile ("lock cmpxchg %2, %0" :
|
|
Packit Service |
155747 |
"+m" (*p), "+a"(old_value) : "r"(new_value) : "memory");
|
|
Packit Service |
155747 |
return old_value;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
am_pkt_short_t *
|
|
Packit Service |
155747 |
am_ctl_getslot_pkt_inner(volatile am_ctl_qhdr_t *shq, am_pkt_short_t *pkt0))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
am_pkt_short_t *pkt;
|
|
Packit Service |
155747 |
uint32_t idx;
|
|
Packit Service |
155747 |
#ifndef CSWAP
|
|
Packit Service |
155747 |
pthread_spin_lock(&shq->lock);
|
|
Packit Service |
155747 |
idx = shq->tail;
|
|
Packit Service |
155747 |
pkt = (am_pkt_short_t *) ((uintptr_t) pkt0 + idx * shq->elem_sz);
|
|
Packit Service |
155747 |
if (pkt->flag == QFREE) {
|
|
Packit Service |
155747 |
ips_sync_reads();
|
|
Packit Service |
155747 |
pkt->flag = QUSED;
|
|
Packit Service |
155747 |
shq->tail += 1;
|
|
Packit Service |
155747 |
if (shq->tail == shq->elem_cnt)
|
|
Packit Service |
155747 |
shq->tail = 0;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
pkt = 0;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
pthread_spin_unlock(&shq->lock);
|
|
Packit Service |
155747 |
#else
|
|
Packit Service |
155747 |
uint32_t idx_next;
|
|
Packit Service |
155747 |
do {
|
|
Packit Service |
155747 |
idx = shq->tail;
|
|
Packit Service |
155747 |
idx_next = (idx + 1 == shq->elem_cnt) ? 0 : idx + 1;
|
|
Packit Service |
155747 |
} while (cswap(&shq->tail, idx, idx_next) != idx);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
pkt = (am_pkt_short_t *) ((uintptr_t) pkt0 + idx * shq->elem_sz);
|
|
Packit Service |
155747 |
while (cswap(&pkt->flag, QFREE, QUSED) != QFREE);
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
return pkt;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* This is safe because 'flag' is at the same offset on both pkt and bulkpkt */
|
|
Packit Service |
155747 |
#define am_ctl_getslot_bulkpkt_inner(shq, pkt0) ((am_pkt_bulk_t *) \
|
|
Packit Service |
155747 |
am_ctl_getslot_pkt_inner(shq, (am_pkt_short_t *)(pkt0)))
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
am_pkt_short_t *
|
|
Packit Service |
155747 |
am_ctl_getslot_pkt(ptl_t *ptl_gen, uint16_t shmidx, int is_reply))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
volatile am_ctl_qhdr_t *shq;
|
|
Packit Service |
155747 |
am_pkt_short_t *pkt0;
|
|
Packit Service |
155747 |
if (!is_reply) {
|
|
Packit Service |
155747 |
shq = &(ptl->am_ep[shmidx].qdir.qreqH->shortq);
|
|
Packit Service |
155747 |
pkt0 = ptl->am_ep[shmidx].qdir.qreqFifoShort;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
shq = &(ptl->am_ep[shmidx].qdir.qrepH->shortq);
|
|
Packit Service |
155747 |
pkt0 = ptl->am_ep[shmidx].qdir.qrepFifoShort;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return am_ctl_getslot_pkt_inner(shq, pkt0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
am_pkt_bulk_t *
|
|
Packit Service |
155747 |
am_ctl_getslot_long(ptl_t *ptl_gen, uint16_t shmidx, int is_reply))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
volatile am_ctl_qhdr_t *shq;
|
|
Packit Service |
155747 |
am_pkt_bulk_t *pkt0;
|
|
Packit Service |
155747 |
if (!is_reply) {
|
|
Packit Service |
155747 |
shq = &(ptl->am_ep[shmidx].qdir.qreqH->longbulkq);
|
|
Packit Service |
155747 |
pkt0 = ptl->am_ep[shmidx].qdir.qreqFifoLong;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
shq = &(ptl->am_ep[shmidx].qdir.qrepH->longbulkq);
|
|
Packit Service |
155747 |
pkt0 = ptl->am_ep[shmidx].qdir.qrepFifoLong;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return am_ctl_getslot_bulkpkt_inner(shq, pkt0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_handlertab_t psmi_allhandlers[] = {
|
|
Packit Service |
155747 |
{0}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{amsh_conn_handler}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{psmi_am_mq_handler}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{psmi_am_mq_handler_data}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{psmi_am_mq_handler_rtsmatch}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{psmi_am_mq_handler_rtsdone}
|
|
Packit Service |
155747 |
,
|
|
Packit Service |
155747 |
{psmi_am_handler}
|
|
Packit Service |
155747 |
};
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(void advance_head(volatile am_ctl_qshort_cache_t *hdr))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
QMARKFREE(hdr->head);
|
|
Packit Service |
155747 |
hdr->head++;
|
|
Packit Service |
155747 |
if (hdr->head == hdr->end)
|
|
Packit Service |
155747 |
hdr->head = hdr->base;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define AMSH_ZERO_POLLS_BEFORE_YIELD 64
|
|
Packit Service |
155747 |
#define AMSH_POLLS_BEFORE_PSM_POLL 16
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* XXX this can be made faster. Instead of checking the flag of the head, keep
|
|
Packit Service |
155747 |
* a cached copy of the integer value of the tail and compare it against the
|
|
Packit Service |
155747 |
* previous one we saw.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_poll_internal_inner(ptl_t *ptl_gen, int replyonly,
|
|
Packit Service |
155747 |
int is_internal))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK_NO_PROGRESS;
|
|
Packit Service |
155747 |
/* poll replies */
|
|
Packit Service |
155747 |
if (!QISEMPTY(ptl->repH.head->flag)) {
|
|
Packit Service |
155747 |
do {
|
|
Packit Service |
155747 |
ips_sync_reads();
|
|
Packit Service |
155747 |
process_packet(ptl_gen, (am_pkt_short_t *) ptl->repH.head,
|
|
Packit Service |
155747 |
0);
|
|
Packit Service |
155747 |
advance_head(&ptl->repH);
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
} while (!QISEMPTY(ptl->repH.head->flag));
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (!replyonly) {
|
|
Packit Service |
155747 |
/* Request queue not enable for 2.0, will be re-enabled to support long
|
|
Packit Service |
155747 |
* replies */
|
|
Packit Service |
155747 |
if (!is_internal && ptl->psmi_am_reqq_fifo.first != NULL) {
|
|
Packit Service |
155747 |
psmi_am_reqq_drain(ptl_gen);
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
if (!QISEMPTY(ptl->reqH.head->flag)) {
|
|
Packit Service |
155747 |
do {
|
|
Packit Service |
155747 |
ips_sync_reads();
|
|
Packit Service |
155747 |
process_packet(ptl_gen,
|
|
Packit Service |
155747 |
(am_pkt_short_t *) ptl->reqH.
|
|
Packit Service |
155747 |
head, 1);
|
|
Packit Service |
155747 |
advance_head(&ptl->reqH);
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
} while (!QISEMPTY(ptl->reqH.head->flag));
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (is_internal) {
|
|
Packit Service |
155747 |
if (err == PSM2_OK) /* some progress, no yields */
|
|
Packit Service |
155747 |
ptl->zero_polls = 0;
|
|
Packit Service |
155747 |
else if (++ptl->zero_polls == AMSH_ZERO_POLLS_BEFORE_YIELD) {
|
|
Packit Service |
155747 |
/* no progress for AMSH_ZERO_POLLS_BEFORE_YIELD */
|
|
Packit Service |
155747 |
sched_yield();
|
|
Packit Service |
155747 |
ptl->zero_polls = 0;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (++ptl->amsh_only_polls == AMSH_POLLS_BEFORE_PSM_POLL) {
|
|
Packit Service |
155747 |
psmi_poll_internal(ptl->ep, 0);
|
|
Packit Service |
155747 |
ptl->amsh_only_polls = 0;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return err; /* if we actually did something */
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* non-inlined version */
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_poll_internal(ptl_t *ptl, int replyonly)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return amsh_poll_internal_inner(ptl, replyonly, 1);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#ifdef PSM_PROFILE
|
|
Packit Service |
155747 |
#define AMSH_POLL_UNTIL(ptl, isreply, cond) \
|
|
Packit Service |
155747 |
do { \
|
|
Packit Service |
155747 |
PSMI_PROFILE_BLOCK(); \
|
|
Packit Service |
155747 |
while (!(cond)) { \
|
|
Packit Service |
155747 |
PSMI_PROFILE_REBLOCK( \
|
|
Packit Service |
155747 |
amsh_poll_internal(ptl, isreply) == \
|
|
Packit Service |
155747 |
PSM2_OK_NO_PROGRESS); \
|
|
Packit Service |
155747 |
} \
|
|
Packit Service |
155747 |
PSMI_PROFILE_UNBLOCK(); \
|
|
Packit Service |
155747 |
} while (0)
|
|
Packit Service |
155747 |
#else
|
|
Packit Service |
155747 |
#define AMSH_POLL_UNTIL(ptl, isreply, cond) \
|
|
Packit Service |
155747 |
do { \
|
|
Packit Service |
155747 |
while (!(cond)) { \
|
|
Packit Service |
155747 |
amsh_poll_internal(ptl, isreply); \
|
|
Packit Service |
155747 |
} \
|
|
Packit Service |
155747 |
} while (0)
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static psm2_error_t amsh_poll(ptl_t *ptl, int replyonly)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return amsh_poll_internal_inner(ptl, replyonly, 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
am_send_pkt_short(ptl_t *ptl, uint32_t destidx, uint32_t returnidx,
|
|
Packit Service |
155747 |
uint32_t bulkidx, uint16_t fmt, uint16_t nargs,
|
|
Packit Service |
155747 |
uint16_t handleridx, psm2_amarg_t *args,
|
|
Packit Service |
155747 |
const void *src, uint32_t len, int isreply))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
volatile am_pkt_short_t *pkt;
|
|
Packit Service |
155747 |
int copy_nargs;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
AMSH_POLL_UNTIL(ptl, isreply,
|
|
Packit Service |
155747 |
(pkt =
|
|
Packit Service |
155747 |
am_ctl_getslot_pkt(ptl, destidx, isreply)) != NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* got a free pkt... fill it in */
|
|
Packit Service |
155747 |
pkt->bulkidx = bulkidx;
|
|
Packit Service |
155747 |
pkt->shmidx = returnidx;
|
|
Packit Service |
155747 |
pkt->type = fmt;
|
|
Packit Service |
155747 |
pkt->nargs = nargs;
|
|
Packit Service |
155747 |
pkt->handleridx = handleridx;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Limit the number of args copied here to NSHORT_ARGS. Additional args
|
|
Packit Service |
155747 |
are carried in the bulkpkt. */
|
|
Packit Service |
155747 |
copy_nargs = nargs;
|
|
Packit Service |
155747 |
if (copy_nargs > NSHORT_ARGS) {
|
|
Packit Service |
155747 |
copy_nargs = NSHORT_ARGS;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i < copy_nargs; i++)
|
|
Packit Service |
155747 |
pkt->args[i] = args[i];
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (fmt == AMFMT_SHORT_INLINE)
|
|
Packit Service |
155747 |
mq_copy_tiny((uint32_t *) &pkt->args[nargs], (uint32_t *) src,
|
|
Packit Service |
155747 |
len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("pkt=%p fmt=%d bulkidx=%d,flag=%d,nargs=%d,"
|
|
Packit Service |
155747 |
"buf=%p,len=%d,hidx=%d,value=%d\n", pkt, (int)fmt, bulkidx,
|
|
Packit Service |
155747 |
pkt->flag, pkt->nargs, src, (int)len, (int)handleridx,
|
|
Packit Service |
155747 |
src != NULL ? *((uint32_t *) src) : 0);
|
|
Packit Service |
155747 |
QMARKREADY(pkt);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#define amsh_shm_copy_short psmi_mq_mtucpy
|
|
Packit Service |
155747 |
#define amsh_shm_copy_long psmi_mq_mtucpy
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
int
|
|
Packit Service |
155747 |
psmi_amsh_generic_inner(uint32_t amtype, ptl_t *ptl_gen, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, void *dst, int flags))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
#ifdef PSM_DEBUG
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
uint16_t type;
|
|
Packit Service |
155747 |
uint32_t bulkidx;
|
|
Packit Service |
155747 |
uint16_t hidx = (uint16_t) handler;
|
|
Packit Service |
155747 |
int destidx = ((am_epaddr_t *) epaddr)->shmidx;
|
|
Packit Service |
155747 |
int returnidx = ((am_epaddr_t *) epaddr)->return_shmidx;
|
|
Packit Service |
155747 |
int is_reply = AM_IS_REPLY(amtype);
|
|
Packit Service |
155747 |
volatile am_pkt_bulk_t *bulkpkt;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("%s epaddr=%s, shmidx=%d, type=%d\n",
|
|
Packit Service |
155747 |
is_reply ? "reply" : "request",
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->epid),
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->shmidx, amtype);
|
|
Packit Service |
155747 |
psmi_assert(epaddr != ptl->epaddr);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
switch (amtype) {
|
|
Packit Service |
155747 |
case AMREQUEST_SHORT:
|
|
Packit Service |
155747 |
case AMREPLY_SHORT:
|
|
Packit Service |
155747 |
if (len + (nargs << 3) <= (NSHORT_ARGS << 3)) {
|
|
Packit Service |
155747 |
/* Payload fits in args packet */
|
|
Packit Service |
155747 |
type = AMFMT_SHORT_INLINE;
|
|
Packit Service |
155747 |
bulkidx = len;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert(len < amsh_qelemsz.qreqFifoLong);
|
|
Packit Service |
155747 |
psmi_assert(src != NULL || nargs > NSHORT_ARGS);
|
|
Packit Service |
155747 |
type = AMFMT_SHORT;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
AMSH_POLL_UNTIL(ptl_gen, is_reply,
|
|
Packit Service |
155747 |
(bulkpkt =
|
|
Packit Service |
155747 |
am_ctl_getslot_long(ptl_gen, destidx,
|
|
Packit Service |
155747 |
is_reply)) !=
|
|
Packit Service |
155747 |
NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
bulkidx = bulkpkt->idx;
|
|
Packit Service |
155747 |
bulkpkt->len = len;
|
|
Packit Service |
155747 |
_HFI_VDBG("bulkpkt %p flag is %d from idx %d\n",
|
|
Packit Service |
155747 |
bulkpkt, bulkpkt->flag, destidx);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i < nargs - NSHORT_ARGS; i++) {
|
|
Packit Service |
155747 |
bulkpkt->args[i] = args[i + NSHORT_ARGS];
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
amsh_shm_copy_short((void *)bulkpkt->payload, src,
|
|
Packit Service |
155747 |
(uint32_t) len);
|
|
Packit Service |
155747 |
QMARKREADY(bulkpkt);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
am_send_pkt_short(ptl_gen, destidx, returnidx, bulkidx, type,
|
|
Packit Service |
155747 |
nargs, hidx, args, src, len, is_reply);
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
case AMREQUEST_LONG:
|
|
Packit Service |
155747 |
case AMREPLY_LONG:
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
uint32_t bytes_left = len;
|
|
Packit Service |
155747 |
uint8_t *src_this = (uint8_t *) src;
|
|
Packit Service |
155747 |
uint8_t *dst_this = (uint8_t *) dst;
|
|
Packit Service |
155747 |
uint32_t bytes_this;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
type = AMFMT_LONG;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("[long][%s] src=%p,dest=%p,len=%d,hidx=%d\n",
|
|
Packit Service |
155747 |
is_reply ? "rep" : "req", src, dst,
|
|
Packit Service |
155747 |
(uint32_t) len, hidx);
|
|
Packit Service |
155747 |
while (bytes_left) {
|
|
Packit Service |
155747 |
bytes_this = min(bytes_left, AMLONG_MTU);
|
|
Packit Service |
155747 |
AMSH_POLL_UNTIL(ptl_gen, is_reply,
|
|
Packit Service |
155747 |
(bulkpkt =
|
|
Packit Service |
155747 |
am_ctl_getslot_long(ptl_gen,
|
|
Packit Service |
155747 |
destidx,
|
|
Packit Service |
155747 |
is_reply))
|
|
Packit Service |
155747 |
!= NULL);
|
|
Packit Service |
155747 |
bytes_left -= bytes_this;
|
|
Packit Service |
155747 |
if (bytes_left == 0)
|
|
Packit Service |
155747 |
type = AMFMT_LONG_END;
|
|
Packit Service |
155747 |
bulkidx = bulkpkt->idx;
|
|
Packit Service |
155747 |
amsh_shm_copy_long((void *)bulkpkt->payload,
|
|
Packit Service |
155747 |
src_this, bytes_this);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
bulkpkt->dest = (uintptr_t) dst;
|
|
Packit Service |
155747 |
bulkpkt->dest_off =
|
|
Packit Service |
155747 |
(uint32_t) ((uintptr_t) dst_this -
|
|
Packit Service |
155747 |
(uintptr_t) dst);
|
|
Packit Service |
155747 |
bulkpkt->len = bytes_this;
|
|
Packit Service |
155747 |
QMARKREADY(bulkpkt);
|
|
Packit Service |
155747 |
am_send_pkt_short(ptl_gen, destidx, returnidx,
|
|
Packit Service |
155747 |
bulkidx, type, nargs, hidx,
|
|
Packit Service |
155747 |
args, NULL, 0, is_reply);
|
|
Packit Service |
155747 |
src_this += bytes_this;
|
|
Packit Service |
155747 |
dst_this += bytes_this;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
default:
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return 1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* A generic version that's not inlined */
|
|
Packit Service |
155747 |
int
|
|
Packit Service |
155747 |
psmi_amsh_generic(uint32_t amtype, ptl_t *ptl, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, void *dst, int flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return psmi_amsh_generic_inner(amtype, ptl, epaddr, handler, args,
|
|
Packit Service |
155747 |
nargs, src, len, dst, flags);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int
|
|
Packit Service |
155747 |
psmi_amsh_short_request(ptl_t *ptl, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, int flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return psmi_amsh_generic_inner(AMREQUEST_SHORT, ptl, epaddr, handler,
|
|
Packit Service |
155747 |
args, nargs, src, len, NULL, flags);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
int
|
|
Packit Service |
155747 |
psmi_amsh_long_request(ptl_t *ptl, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, void *dest, int flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return psmi_amsh_generic_inner(AMREQUEST_LONG, ptl, epaddr, handler,
|
|
Packit Service |
155747 |
args, nargs, src, len, dest, flags);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
psmi_amsh_short_reply(amsh_am_token_t *tok,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, int flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psmi_amsh_generic_inner(AMREPLY_SHORT, tok->ptl, tok->tok.epaddr_incoming,
|
|
Packit Service |
155747 |
handler, args, nargs, src, len, NULL, flags);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
psmi_amsh_long_reply(amsh_am_token_t *tok,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
const void *src, size_t len, void *dest, int flags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psmi_amsh_generic_inner(AMREPLY_LONG, tok->ptl, tok->tok.epaddr_incoming,
|
|
Packit Service |
155747 |
handler, args, nargs, src, len, dest, flags);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void psmi_am_reqq_init(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
ptl->psmi_am_reqq_fifo.first = NULL;
|
|
Packit Service |
155747 |
ptl->psmi_am_reqq_fifo.lastp = &ptl->psmi_am_reqq_fifo.first;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psm2_error_t psmi_am_reqq_drain(ptl_t *ptl_gen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
am_reqq_t *reqn = ptl->psmi_am_reqq_fifo.first;
|
|
Packit Service |
155747 |
am_reqq_t *req;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK_NO_PROGRESS;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* We're going to process the entire list, and running the generic handler
|
|
Packit Service |
155747 |
* below can cause other requests to be enqueued in the queue that we're
|
|
Packit Service |
155747 |
* processing. */
|
|
Packit Service |
155747 |
ptl->psmi_am_reqq_fifo.first = NULL;
|
|
Packit Service |
155747 |
ptl->psmi_am_reqq_fifo.lastp = &ptl->psmi_am_reqq_fifo.first;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
while ((req = reqn) != NULL) {
|
|
Packit Service |
155747 |
err = PSM2_OK;
|
|
Packit Service |
155747 |
reqn = req->next;
|
|
Packit Service |
155747 |
_HFI_VDBG
|
|
Packit Service |
155747 |
("push of reqq=%p epaddr=%s localreq=%p remotereq=%p\n",
|
|
Packit Service |
155747 |
req, psmi_epaddr_get_hostname(req->epaddr->epid),
|
|
Packit Service |
155747 |
(void *)(uintptr_t) req->args[1].u64w0,
|
|
Packit Service |
155747 |
(void *)(uintptr_t) req->args[0].u64w0);
|
|
Packit Service |
155747 |
psmi_amsh_generic(req->amtype, req->ptl, req->epaddr,
|
|
Packit Service |
155747 |
req->handler, req->args, req->nargs, req->src,
|
|
Packit Service |
155747 |
req->len, req->dest, req->amflags);
|
|
Packit Service |
155747 |
if (req->flags & AM_FLAG_SRC_TEMP)
|
|
Packit Service |
155747 |
psmi_free(req->src);
|
|
Packit Service |
155747 |
psmi_free(req);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
psmi_am_reqq_add(int amtype, ptl_t *ptl_gen, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
psm2_handler_t handler, psm2_amarg_t *args, int nargs,
|
|
Packit Service |
155747 |
void *src, size_t len, void *dest, int amflags)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
int flags = 0;
|
|
Packit Service |
155747 |
am_reqq_t *nreq =
|
|
Packit Service |
155747 |
(am_reqq_t *) psmi_malloc(ptl->ep, UNDEFINED, sizeof(am_reqq_t));
|
|
Packit Service |
155747 |
psmi_assert_always(nreq != NULL);
|
|
Packit Service |
155747 |
_HFI_VDBG("alloc of reqq=%p, to epaddr=%s, ptr=%p, len=%d, "
|
|
Packit Service |
155747 |
"localreq=%p, remotereq=%p\n", nreq,
|
|
Packit Service |
155747 |
psmi_epaddr_get_hostname(epaddr->epid), dest,
|
|
Packit Service |
155747 |
(int)len, (void *)(uintptr_t) args[1].u64w0,
|
|
Packit Service |
155747 |
(void *)(uintptr_t) args[0].u64w0);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert(nargs <= 8);
|
|
Packit Service |
155747 |
nreq->next = NULL;
|
|
Packit Service |
155747 |
nreq->amtype = amtype;
|
|
Packit Service |
155747 |
nreq->ptl = ptl_gen;
|
|
Packit Service |
155747 |
nreq->epaddr = epaddr;
|
|
Packit Service |
155747 |
nreq->handler = handler;
|
|
Packit Service |
155747 |
for (i = 0; i < nargs; i++)
|
|
Packit Service |
155747 |
nreq->args[i] = args[i];
|
|
Packit Service |
155747 |
nreq->nargs = nargs;
|
|
Packit Service |
155747 |
if (AM_IS_LONG(amtype) && src != NULL &&
|
|
Packit Service |
155747 |
len > 0 && !(amflags & AM_FLAG_SRC_ASYNC)) {
|
|
Packit Service |
155747 |
abort();
|
|
Packit Service |
155747 |
flags |= AM_FLAG_SRC_TEMP;
|
|
Packit Service |
155747 |
nreq->src = psmi_malloc(ptl->ep, UNDEFINED, len);
|
|
Packit Service |
155747 |
psmi_assert_always(nreq->src != NULL); /* XXX mem */
|
|
Packit Service |
155747 |
amsh_shm_copy_short(nreq->src, src, len);
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
nreq->src = src;
|
|
Packit Service |
155747 |
nreq->len = len;
|
|
Packit Service |
155747 |
nreq->dest = dest;
|
|
Packit Service |
155747 |
nreq->amflags = amflags;
|
|
Packit Service |
155747 |
nreq->flags = flags;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
nreq->next = NULL;
|
|
Packit Service |
155747 |
*(ptl->psmi_am_reqq_fifo.lastp) = nreq;
|
|
Packit Service |
155747 |
ptl->psmi_am_reqq_fifo.lastp = &nreq->next;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void process_packet(ptl_t *ptl_gen, am_pkt_short_t *pkt, int isreq)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
amsh_am_token_t tok;
|
|
Packit Service |
155747 |
psmi_handler_fn_t fn;
|
|
Packit Service |
155747 |
psm2_amarg_t *args = pkt->args;
|
|
Packit Service |
155747 |
uint16_t shmidx = pkt->shmidx;
|
|
Packit Service |
155747 |
int nargs = pkt->nargs;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
tok.tok.epaddr_incoming = ((shmidx != (uint16_t)-1) ? ptl->am_ep[shmidx].epaddr : 0);
|
|
Packit Service |
155747 |
tok.ptl = ptl_gen;
|
|
Packit Service |
155747 |
tok.mq = ptl->ep->mq;
|
|
Packit Service |
155747 |
tok.shmidx = shmidx;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
uint16_t hidx = (uint16_t) pkt->handleridx;
|
|
Packit Service |
155747 |
uint32_t bulkidx = pkt->bulkidx;
|
|
Packit Service |
155747 |
uintptr_t bulkptr;
|
|
Packit Service |
155747 |
am_pkt_bulk_t *bulkpkt;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fn = (psmi_handler_fn_t) psmi_allhandlers[hidx].fn;
|
|
Packit Service |
155747 |
psmi_assert(fn != NULL);
|
|
Packit Service |
155747 |
psmi_assert((uintptr_t) pkt > ptl->self_nodeinfo->amsh_shmbase);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (pkt->type == AMFMT_SHORT_INLINE) {
|
|
Packit Service |
155747 |
_HFI_VDBG
|
|
Packit Service |
155747 |
("%s inline flag=%d nargs=%d from_idx=%d pkt=%p hidx=%d\n",
|
|
Packit Service |
155747 |
isreq ? "request" : "reply", pkt->flag, nargs, shmidx, pkt,
|
|
Packit Service |
155747 |
hidx);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
fn(&tok, args, nargs, pkt->length > 0 ?
|
|
Packit Service |
155747 |
(void *)&args[nargs] : NULL, pkt->length);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
int isend = 0;
|
|
Packit Service |
155747 |
switch (pkt->type) {
|
|
Packit Service |
155747 |
case AMFMT_LONG_END:
|
|
Packit Service |
155747 |
isend = 1;
|
|
Packit Service |
155747 |
case AMFMT_LONG:
|
|
Packit Service |
155747 |
case AMFMT_SHORT:
|
|
Packit Service |
155747 |
if (isreq) {
|
|
Packit Service |
155747 |
bulkptr =
|
|
Packit Service |
155747 |
(uintptr_t) ptl->self_nodeinfo->qdir.
|
|
Packit Service |
155747 |
qreqFifoLong;
|
|
Packit Service |
155747 |
bulkptr += bulkidx * amsh_qelemsz.qreqFifoLong;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
bulkptr =
|
|
Packit Service |
155747 |
(uintptr_t) ptl->self_nodeinfo->qdir.
|
|
Packit Service |
155747 |
qrepFifoLong;
|
|
Packit Service |
155747 |
bulkptr += bulkidx * amsh_qelemsz.qrepFifoLong;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
default:
|
|
Packit Service |
155747 |
bulkptr = 0;
|
|
Packit Service |
155747 |
psmi_handle_error(PSMI_EP_NORETURN, PSM2_INTERNAL_ERR,
|
|
Packit Service |
155747 |
"Unknown/unhandled packet type 0x%x",
|
|
Packit Service |
155747 |
pkt->type);
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
bulkpkt = (am_pkt_bulk_t *) bulkptr;
|
|
Packit Service |
155747 |
_HFI_VDBG("ep=%p mq=%p type=%d bulkidx=%d flag=%d/%d nargs=%d "
|
|
Packit Service |
155747 |
"from_idx=%d pkt=%p/%p hidx=%d\n",
|
|
Packit Service |
155747 |
ptl->ep, ptl->ep->mq, pkt->type, bulkidx, pkt->flag,
|
|
Packit Service |
155747 |
bulkpkt->flag, nargs, shmidx, pkt, bulkpkt, hidx);
|
|
Packit Service |
155747 |
psmi_assert(bulkpkt->flag == QREADY);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (nargs > NSHORT_ARGS || isend == 1) {
|
|
Packit Service |
155747 |
/* Either there are more args in the bulkpkt, or this is the last
|
|
Packit Service |
155747 |
packet of a long payload. In either case, copy the args. */
|
|
Packit Service |
155747 |
int i;
|
|
Packit Service |
155747 |
args =
|
|
Packit Service |
155747 |
alloca((NSHORT_ARGS +
|
|
Packit Service |
155747 |
NBULK_ARGS) * sizeof(psm2_amarg_t));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (i = 0; i < NSHORT_ARGS; i++) {
|
|
Packit Service |
155747 |
args[i] = pkt->args[i];
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
for (; i < nargs; i++) {
|
|
Packit Service |
155747 |
args[i] = bulkpkt->args[i - NSHORT_ARGS];
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (pkt->type == AMFMT_SHORT) {
|
|
Packit Service |
155747 |
fn(&tok, args, nargs,
|
|
Packit Service |
155747 |
(void *)bulkpkt->payload, bulkpkt->len);
|
|
Packit Service |
155747 |
QMARKFREE(bulkpkt);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
amsh_shm_copy_long((void *)(bulkpkt->dest +
|
|
Packit Service |
155747 |
bulkpkt->dest_off),
|
|
Packit Service |
155747 |
bulkpkt->payload, bulkpkt->len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* If this is the last packet, copy args before running the
|
|
Packit Service |
155747 |
* handler */
|
|
Packit Service |
155747 |
if (isend) {
|
|
Packit Service |
155747 |
void *dest = (void *)bulkpkt->dest;
|
|
Packit Service |
155747 |
size_t len =
|
|
Packit Service |
155747 |
(size_t) (bulkpkt->dest_off + bulkpkt->len);
|
|
Packit Service |
155747 |
QMARKFREE(bulkpkt);
|
|
Packit Service |
155747 |
fn(&tok, args, nargs, dest, len);
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
QMARKFREE(bulkpkt);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_mq_rndv(ptl_t *ptl, psm2_mq_t mq, psm2_mq_req_t req,
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr, psm2_mq_tag_t *tag, const void *buf,
|
|
Packit Service |
155747 |
uint32_t len)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psm2_amarg_t args[5];
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
args[0].u32w0 = MQ_MSG_LONGRTS;
|
|
Packit Service |
155747 |
args[0].u32w1 = len;
|
|
Packit Service |
155747 |
args[1].u32w1 = tag->tag[0];
|
|
Packit Service |
155747 |
args[1].u32w0 = tag->tag[1];
|
|
Packit Service |
155747 |
args[2].u32w1 = tag->tag[2];
|
|
Packit Service |
155747 |
args[3].u64w0 = (uint64_t) (uintptr_t) req;
|
|
Packit Service |
155747 |
args[4].u64w0 = (uint64_t) (uintptr_t) buf;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_assert(req != NULL);
|
|
Packit Service |
155747 |
req->type = MQE_TYPE_SEND;
|
|
Packit Service |
155747 |
req->req_data.buf = (void *)buf;
|
|
Packit Service |
155747 |
req->req_data.buf_len = len;
|
|
Packit Service |
155747 |
req->req_data.send_msglen = len;
|
|
Packit Service |
155747 |
req->send_msgoff = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
/* If the send buffer is on gpu, we create a cuda IPC
|
|
Packit Service |
155747 |
* handle and send it as payload in the RTS
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
if (req->is_buf_gpu_mem) {
|
|
Packit Service |
155747 |
CUdeviceptr buf_base_ptr;
|
|
Packit Service |
155747 |
PSMI_CUDA_CALL(cuMemGetAddressRange, &buf_base_ptr, NULL, (CUdeviceptr)buf);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Offset in GPU buffer from which we copy data, we have to
|
|
Packit Service |
155747 |
* send it separetly because this offset is lost
|
|
Packit Service |
155747 |
* when cuIpcGetMemHandle is called */
|
|
Packit Service |
155747 |
req->cuda_ipc_offset = buf - (void*)buf_base_ptr;
|
|
Packit Service |
155747 |
args[2].u32w0 = (uint32_t)req->cuda_ipc_offset;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
PSMI_CUDA_CALL(cuIpcGetMemHandle,
|
|
Packit Service |
155747 |
&req->cuda_ipc_handle,
|
|
Packit Service |
155747 |
(CUdeviceptr) buf);
|
|
Packit Service |
155747 |
if (req->flags_internal & PSMI_REQ_FLAG_FASTPATH) {
|
|
Packit Service |
155747 |
psmi_am_reqq_add(AMREQUEST_SHORT, ptl,
|
|
Packit Service |
155747 |
epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 5, (void*)&req->cuda_ipc_handle,
|
|
Packit Service |
155747 |
sizeof(CUipcMemHandle), NULL, 0);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_amsh_short_request(ptl, epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 5, (void*)&req->cuda_ipc_handle,
|
|
Packit Service |
155747 |
sizeof(CUipcMemHandle), 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
req->cuda_ipc_handle_attached = 1;
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
if (req->flags_internal & PSMI_REQ_FLAG_FASTPATH) {
|
|
Packit Service |
155747 |
psmi_am_reqq_add(AMREQUEST_SHORT, ptl,
|
|
Packit Service |
155747 |
epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 5, NULL, 0, NULL, 0);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_amsh_short_request(ptl, epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 5, NULL, 0, 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/*
|
|
Packit Service |
155747 |
* All shared am mq sends, req can be NULL
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
PSMI_ALWAYS_INLINE(
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_mq_send_inner(psm2_mq_t mq, psm2_mq_req_t req, psm2_epaddr_t epaddr,
|
|
Packit Service |
155747 |
uint32_t flags_user, uint32_t flags_internal, psm2_mq_tag_t *tag,
|
|
Packit Service |
155747 |
const void *ubuf, uint32_t len))
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psm2_amarg_t args[3];
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
int is_blocking = (req == NULL);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
int gpu_mem;
|
|
Packit Service |
155747 |
/* All sends from a gpu buffer use the rendezvous protocol */
|
|
Packit Service |
155747 |
if (PSMI_IS_CUDA_ENABLED && PSMI_IS_CUDA_MEM((void*)ubuf)) {
|
|
Packit Service |
155747 |
if (!PSMI_IS_CUDA_ENABLED)
|
|
Packit Service |
155747 |
psmi_handle_error(PSMI_EP_NORETURN, PSM2_INTERNAL_ERR,
|
|
Packit Service |
155747 |
" Please enable PSM CUDA support when using GPU buffer \n");
|
|
Packit Service |
155747 |
gpu_mem = 1;
|
|
Packit Service |
155747 |
goto do_rendezvous;
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
gpu_mem = 0;
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (!flags_user && len <= AMLONG_MTU) {
|
|
Packit Service |
155747 |
if (len <= 32)
|
|
Packit Service |
155747 |
args[0].u32w0 = MQ_MSG_TINY;
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
args[0].u32w0 = MQ_MSG_SHORT;
|
|
Packit Service |
155747 |
args[1].u32w1 = tag->tag[0];
|
|
Packit Service |
155747 |
args[1].u32w0 = tag->tag[1];
|
|
Packit Service |
155747 |
args[2].u32w1 = tag->tag[2];
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (flags_internal & PSMI_REQ_FLAG_FASTPATH) {
|
|
Packit Service |
155747 |
psmi_am_reqq_add(AMREQUEST_SHORT, epaddr->ptlctl->ptl,
|
|
Packit Service |
155747 |
epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 3, (void *)ubuf, len, NULL, 0);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_amsh_short_request(epaddr->ptlctl->ptl, epaddr,
|
|
Packit Service |
155747 |
mq_handler_hidx, args, 3, ubuf, len, 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else if (flags_user & PSM2_MQ_FLAG_SENDSYNC)
|
|
Packit Service |
155747 |
goto do_rendezvous;
|
|
Packit Service |
155747 |
else if (len <= mq->shm_thresh_rv) {
|
|
Packit Service |
155747 |
uint32_t bytes_left = len;
|
|
Packit Service |
155747 |
uint32_t bytes_this = min(bytes_left, AMLONG_MTU);
|
|
Packit Service |
155747 |
uint8_t *buf = (uint8_t *) ubuf;
|
|
Packit Service |
155747 |
args[0].u32w0 = MQ_MSG_EAGER;
|
|
Packit Service |
155747 |
args[0].u32w1 = len;
|
|
Packit Service |
155747 |
args[1].u32w1 = tag->tag[0];
|
|
Packit Service |
155747 |
args[1].u32w0 = tag->tag[1];
|
|
Packit Service |
155747 |
args[2].u32w1 = tag->tag[2];
|
|
Packit Service |
155747 |
if (flags_internal & PSMI_REQ_FLAG_FASTPATH) {
|
|
Packit Service |
155747 |
psmi_am_reqq_add(AMREQUEST_SHORT, epaddr->ptlctl->ptl,
|
|
Packit Service |
155747 |
epaddr, mq_handler_hidx,
|
|
Packit Service |
155747 |
args, 3, buf, bytes_this, NULL, 0);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_amsh_short_request(epaddr->ptlctl->ptl, epaddr,
|
|
Packit Service |
155747 |
mq_handler_hidx, args, 3, buf,
|
|
Packit Service |
155747 |
bytes_this, 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
bytes_left -= bytes_this;
|
|
Packit Service |
155747 |
buf += bytes_this;
|
|
Packit Service |
155747 |
args[2].u32w0 = 0;
|
|
Packit Service |
155747 |
while (bytes_left) {
|
|
Packit Service |
155747 |
args[2].u32w0 += bytes_this;
|
|
Packit Service |
155747 |
bytes_this = min(bytes_left, AMLONG_MTU);
|
|
Packit Service |
155747 |
/* Here we kind of bend the rules, and assume that shared-memory
|
|
Packit Service |
155747 |
* active messages are delivered in order */
|
|
Packit Service |
155747 |
if (flags_internal & PSMI_REQ_FLAG_FASTPATH) {
|
|
Packit Service |
155747 |
psmi_am_reqq_add(AMREQUEST_SHORT, epaddr->ptlctl->ptl,
|
|
Packit Service |
155747 |
epaddr, mq_handler_data_hidx,
|
|
Packit Service |
155747 |
args, 3, buf, bytes_this, NULL, 0);
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_amsh_short_request(epaddr->ptlctl->ptl, epaddr,
|
|
Packit Service |
155747 |
mq_handler_data_hidx, args,
|
|
Packit Service |
155747 |
3, buf, bytes_this, 0);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
buf += bytes_this;
|
|
Packit Service |
155747 |
bytes_left -= bytes_this;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
do_rendezvous:
|
|
Packit Service |
155747 |
if (is_blocking) {
|
|
Packit Service |
155747 |
req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND);
|
|
Packit Service |
155747 |
if_pf(req == NULL)
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
req->req_data.send_msglen = len;
|
|
Packit Service |
155747 |
req->req_data.tag = *tag;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Since SEND command is blocking, this request is
|
|
Packit Service |
155747 |
* entirely internal and we will not be exposed to user.
|
|
Packit Service |
155747 |
* Setting as internal so it will not be added to
|
|
Packit Service |
155747 |
* mq->completed_q */
|
|
Packit Service |
155747 |
req->flags_internal |= (flags_internal | PSMI_REQ_FLAG_IS_INTERNAL);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
/* CUDA documentation dictates the use of SYNC_MEMOPS attribute
|
|
Packit Service |
155747 |
* when the buffer pointer received into PSM has been allocated
|
|
Packit Service |
155747 |
* by the application. This guarantees the all memory operations
|
|
Packit Service |
155747 |
* to this region of memory (used by multiple layers of the stack)
|
|
Packit Service |
155747 |
* always synchronize
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
if (gpu_mem) {
|
|
Packit Service |
155747 |
int trueflag = 1;
|
|
Packit Service |
155747 |
PSMI_CUDA_CALL(cuPointerSetAttribute, &trueflag,
|
|
Packit Service |
155747 |
CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,
|
|
Packit Service |
155747 |
(CUdeviceptr)ubuf);
|
|
Packit Service |
155747 |
req->is_buf_gpu_mem = 1;
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
req->is_buf_gpu_mem = 0;
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
err =
|
|
Packit Service |
155747 |
amsh_mq_rndv(epaddr->ptlctl->ptl, mq, req, epaddr, tag,
|
|
Packit Service |
155747 |
ubuf, len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (err == PSM2_OK && is_blocking) { /* wait... */
|
|
Packit Service |
155747 |
err = psmi_mq_wait_internal(&req;;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return err; /* skip eager accounting below */
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* All eager async sends are always "all done" */
|
|
Packit Service |
155747 |
if (req != NULL) {
|
|
Packit Service |
155747 |
req->state = MQ_STATE_COMPLETE;
|
|
Packit Service |
155747 |
mq_qq_append(&mq->completed_q, req);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
mq->stats.tx_num++;
|
|
Packit Service |
155747 |
mq->stats.tx_shm_num++;
|
|
Packit Service |
155747 |
mq->stats.tx_eager_num++;
|
|
Packit Service |
155747 |
mq->stats.tx_eager_bytes += len;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_mq_isend(psm2_mq_t mq, psm2_epaddr_t epaddr, uint32_t flags_user,
|
|
Packit Service |
155747 |
uint32_t flags_internal, psm2_mq_tag_t *tag, const void *ubuf,
|
|
Packit Service |
155747 |
uint32_t len, void *context, psm2_mq_req_t *req_o)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
psm2_mq_req_t req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND);
|
|
Packit Service |
155747 |
if_pf(req == NULL)
|
|
Packit Service |
155747 |
return PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
req->req_data.send_msglen = len;
|
|
Packit Service |
155747 |
req->req_data.tag = *tag;
|
|
Packit Service |
155747 |
req->req_data.context = context;
|
|
Packit Service |
155747 |
req->flags_user = flags_user;
|
|
Packit Service |
155747 |
req->flags_internal = flags_internal;
|
|
Packit Service |
155747 |
_HFI_VDBG("[ishrt][%s->%s][n=0][b=%p][l=%d][t=%08x.%08x.%08x]\n",
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->ptlctl->ep->epid),
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->epid), ubuf, len,
|
|
Packit Service |
155747 |
tag->tag[0], tag->tag[1], tag->tag[2]);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
amsh_mq_send_inner(mq, req, epaddr, flags_user, flags_internal, tag, ubuf, len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*req_o = req;
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_mq_send(psm2_mq_t mq, psm2_epaddr_t epaddr, uint32_t flags,
|
|
Packit Service |
155747 |
psm2_mq_tag_t *tag, const void *ubuf, uint32_t len)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
_HFI_VDBG("[shrt][%s->%s][n=0][b=%p][l=%d][t=%08x.%08x.%08x]\n",
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->ptlctl->ep->epid),
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->epid), ubuf, len,
|
|
Packit Service |
155747 |
tag->tag[0], tag->tag[1], tag->tag[2]);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
amsh_mq_send_inner(mq, NULL, epaddr, flags, PSMI_REQ_FLAG_NORMAL, tag, ubuf, len);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* kassist-related handling */
|
|
Packit Service |
155747 |
int psmi_epaddr_pid(psm2_epaddr_t epaddr)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
uint16_t shmidx = ((am_epaddr_t *) epaddr)->shmidx;
|
|
Packit Service |
155747 |
return ((struct ptl_am *)(epaddr->ptlctl->ptl))->am_ep[shmidx].pid;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
#if _HFI_DEBUGGING
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
const char *psmi_kassist_getmode(int mode)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
switch (mode) {
|
|
Packit Service |
155747 |
case PSMI_KASSIST_OFF:
|
|
Packit Service |
155747 |
return "kassist off";
|
|
Packit Service |
155747 |
case PSMI_KASSIST_CMA_GET:
|
|
Packit Service |
155747 |
return "cma get";
|
|
Packit Service |
155747 |
case PSMI_KASSIST_CMA_PUT:
|
|
Packit Service |
155747 |
return "cma put";
|
|
Packit Service |
155747 |
default:
|
|
Packit Service |
155747 |
return "unknown";
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
int psmi_get_kassist_mode()
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int mode = PSMI_KASSIST_MODE_DEFAULT;
|
|
Packit Service |
155747 |
/* Cuda PSM only supports KASSIST_CMA_GET */
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
mode = PSMI_KASSIST_CMA_GET;
|
|
Packit Service |
155747 |
#else
|
|
Packit Service |
155747 |
union psmi_envvar_val env_kassist;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (!psmi_getenv("PSM2_KASSIST_MODE",
|
|
Packit Service |
155747 |
"PSM Shared memory kernel assist mode "
|
|
Packit Service |
155747 |
"(cma-put, cma-get, none)",
|
|
Packit Service |
155747 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_STR,
|
|
Packit Service |
155747 |
(union psmi_envvar_val)
|
|
Packit Service |
155747 |
PSMI_KASSIST_MODE_DEFAULT_STRING, &env_kassist)) {
|
|
Packit Service |
155747 |
char *s = env_kassist.e_str;
|
|
Packit Service |
155747 |
if (strcasecmp(s, "cma-put") == 0)
|
|
Packit Service |
155747 |
mode = PSMI_KASSIST_CMA_PUT;
|
|
Packit Service |
155747 |
else if (strcasecmp(s, "cma-get") == 0)
|
|
Packit Service |
155747 |
mode = PSMI_KASSIST_CMA_GET;
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
mode = PSMI_KASSIST_OFF;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
/* cma-get is the fastest, so it's the default.
|
|
Packit Service |
155747 |
Availability of CMA is checked in psmi_shm_create();
|
|
Packit Service |
155747 |
if CMA is not available it falls back to 'none' there. */
|
|
Packit Service |
155747 |
mode = PSMI_KASSIST_CMA_GET;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
return mode;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Connection handling for shared memory AM.
|
|
Packit Service |
155747 |
*
|
|
Packit Service |
155747 |
* arg0 => conn_op, result (PSM error type)
|
|
Packit Service |
155747 |
* arg1 => epid (always)
|
|
Packit Service |
155747 |
* arg2 => pid, version.
|
|
Packit Service |
155747 |
* arg3 => pointer to error for replies.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
void
|
|
Packit Service |
155747 |
amsh_conn_handler(void *toki, psm2_amarg_t *args, int narg, void *buf,
|
|
Packit Service |
155747 |
size_t len)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
int op = args[0].u16w0;
|
|
Packit Service |
155747 |
int phase = args[0].u32w1;
|
|
Packit Service |
155747 |
psm2_epid_t epid = args[1].u64w0;
|
|
Packit Service |
155747 |
int16_t return_shmidx = args[0].u16w1;
|
|
Packit Service |
155747 |
psm2_error_t err = (psm2_error_t) args[2].u32w1;
|
|
Packit Service |
155747 |
psm2_error_t *perr = (psm2_error_t *) (uintptr_t) args[3].u64w0;
|
|
Packit Service |
155747 |
int pid = args[2].u32w0;
|
|
Packit Service |
155747 |
int force_remap = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr;
|
|
Packit Service |
155747 |
amsh_am_token_t *tok = (amsh_am_token_t *) toki;
|
|
Packit Service |
155747 |
uint16_t shmidx = tok->shmidx;
|
|
Packit Service |
155747 |
int is_valid;
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)(tok->ptl);
|
|
Packit Service |
155747 |
ptl_t *ptl_gen = tok->ptl;
|
|
Packit Service |
155747 |
int cstate;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* We do this because it's an assumption below */
|
|
Packit Service |
155747 |
psmi_assert_always(buf == NULL && len == 0);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
_HFI_VDBG("Conn op=%d, phase=%d, epid=%llx, err=%d\n",
|
|
Packit Service |
155747 |
op, phase, (unsigned long long)epid, err);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
switch (op) {
|
|
Packit Service |
155747 |
case PSMI_AM_CONN_REQ:
|
|
Packit Service |
155747 |
_HFI_VDBG("Connect from %d:%d\n",
|
|
Packit Service |
155747 |
(int)psm2_epid_nid(epid), (int)psm2_epid_context(epid));
|
|
Packit Service |
155747 |
epaddr = psmi_epid_lookup(ptl->ep, epid);
|
|
Packit Service |
155747 |
if (epaddr && ((am_epaddr_t *) epaddr)->pid != pid) {
|
|
Packit Service |
155747 |
/* If old pid is unknown consider new pid the correct one */
|
|
Packit Service |
155747 |
if (((am_epaddr_t *) epaddr)->pid == AMSH_PID_UNKNOWN) {
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->pid = pid;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_epid_remove(ptl->ep, epid);
|
|
Packit Service |
155747 |
epaddr = NULL;
|
|
Packit Service |
155747 |
force_remap = 1;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (shmidx == (uint16_t)-1) {
|
|
Packit Service |
155747 |
/* incoming packet will never be from our shmidx slot 0
|
|
Packit Service |
155747 |
thus the other process doesn't know our return info.
|
|
Packit Service |
155747 |
attach_to will lookup or create the proper shmidx */
|
|
Packit Service |
155747 |
if ((err = psmi_shm_map_remote(ptl_gen, epid, &shmidx, force_remap))) {
|
|
Packit Service |
155747 |
psmi_handle_error(PSMI_EP_NORETURN, err,
|
|
Packit Service |
155747 |
"Fatal error in "
|
|
Packit Service |
155747 |
"connecting to shm segment");
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
am_update_directory(&ptl->am_ep[shmidx]);
|
|
Packit Service |
155747 |
tok->shmidx = shmidx;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (epaddr == NULL) {
|
|
Packit Service |
155747 |
uintptr_t args_segoff =
|
|
Packit Service |
155747 |
(uintptr_t) args - ptl->self_nodeinfo->amsh_shmbase;
|
|
Packit Service |
155747 |
if ((err = amsh_epaddr_add(ptl_gen, epid, shmidx, &epaddr)))
|
|
Packit Service |
155747 |
/* Unfortunately, no way out of here yet */
|
|
Packit Service |
155747 |
psmi_handle_error(PSMI_EP_NORETURN, err,
|
|
Packit Service |
155747 |
"Fatal error "
|
|
Packit Service |
155747 |
"in connecting to shm segment");
|
|
Packit Service |
155747 |
args =
|
|
Packit Service |
155747 |
(psm2_amarg_t *) (ptl->self_nodeinfo->amsh_shmbase +
|
|
Packit Service |
155747 |
args_segoff);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->pid = pid;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Rewrite args */
|
|
Packit Service |
155747 |
ptl->connect_incoming++;
|
|
Packit Service |
155747 |
args[0].u16w0 = PSMI_AM_CONN_REP;
|
|
Packit Service |
155747 |
/* and return our shmidx for the connecting process */
|
|
Packit Service |
155747 |
args[0].u16w1 = shmidx;
|
|
Packit Service |
155747 |
args[1].u64w0 = (psm2_epid_t) ptl->epid;
|
|
Packit Service |
155747 |
args[2].u32w0 = getpid();
|
|
Packit Service |
155747 |
args[2].u32w1 = PSM2_OK;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_incoming =
|
|
Packit Service |
155747 |
AMSH_CSTATE_INCOMING_ESTABLISHED;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->return_shmidx = return_shmidx;
|
|
Packit Service |
155747 |
tok->tok.epaddr_incoming = epaddr; /* adjust token */
|
|
Packit Service |
155747 |
psmi_amsh_short_reply(tok, amsh_conn_handler_hidx,
|
|
Packit Service |
155747 |
args, narg, NULL, 0, 0);
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
case PSMI_AM_CONN_REP:
|
|
Packit Service |
155747 |
if (ptl->connect_phase != phase) {
|
|
Packit Service |
155747 |
_HFI_VDBG("Out of phase connect reply\n");
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
epaddr = ptl->am_ep[shmidx].epaddr;
|
|
Packit Service |
155747 |
/* check if a race has occurred on shm-file reuse.
|
|
Packit Service |
155747 |
* if so, don't transition to the next state.
|
|
Packit Service |
155747 |
* the next call to connreq_poll() will restart the
|
|
Packit Service |
155747 |
* connection.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
if (ptl->am_ep[shmidx].pid !=
|
|
Packit Service |
155747 |
((struct am_ctl_nodeinfo *) ptl->am_ep[shmidx].amsh_shmbase)->pid)
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
*perr = err;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing
|
|
Packit Service |
155747 |
= AMSH_CSTATE_OUTGOING_REPLIED;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->return_shmidx = return_shmidx;
|
|
Packit Service |
155747 |
ptl->connect_outgoing++;
|
|
Packit Service |
155747 |
_HFI_VDBG("CCC epaddr=%s connected to ptl=%p\n",
|
|
Packit Service |
155747 |
psmi_epaddr_get_name(epaddr->epid), ptl);
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
case PSMI_AM_DISC_REQ:
|
|
Packit Service |
155747 |
epaddr = psmi_epid_lookup(ptl->ep, epid);
|
|
Packit Service |
155747 |
if (!epaddr) {
|
|
Packit Service |
155747 |
_HFI_VDBG("Dropping disconnect request from an epid that we are not connected to\n");
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
args[0].u16w0 = PSMI_AM_DISC_REP;
|
|
Packit Service |
155747 |
args[2].u32w1 = PSM2_OK;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_incoming =
|
|
Packit Service |
155747 |
AMSH_CSTATE_INCOMING_DISC_REQUESTED;
|
|
Packit Service |
155747 |
ptl->connect_incoming--;
|
|
Packit Service |
155747 |
/* Before sending the reply, make sure the process
|
|
Packit Service |
155747 |
* is still connected */
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ptl->am_ep[shmidx].epid != epaddr->epid)
|
|
Packit Service |
155747 |
is_valid = 0;
|
|
Packit Service |
155747 |
else
|
|
Packit Service |
155747 |
is_valid = 1;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (is_valid) {
|
|
Packit Service |
155747 |
psmi_amsh_short_reply(tok, amsh_conn_handler_hidx,
|
|
Packit Service |
155747 |
args, narg, NULL, 0, 0);
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* Only munmap if we have nothing more to
|
|
Packit Service |
155747 |
* communicate with the other node, i.e. we are
|
|
Packit Service |
155747 |
* already disconnected with the other node
|
|
Packit Service |
155747 |
* or have sent a disconnect request.
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
cstate = ((am_epaddr_t *) epaddr)->cstate_outgoing;
|
|
Packit Service |
155747 |
if (cstate == AMSH_CSTATE_OUTGOING_DISC_REQUESTED) {
|
|
Packit Service |
155747 |
err = psmi_do_unmap(ptl->am_ep[shmidx].amsh_shmbase);
|
|
Packit Service |
155747 |
psmi_epid_remove(epaddr->ptlctl->ep, epaddr->epid);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
case PSMI_AM_DISC_REP:
|
|
Packit Service |
155747 |
if (ptl->connect_phase != phase) {
|
|
Packit Service |
155747 |
_HFI_VDBG("Out of phase disconnect reply\n");
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
*perr = err;
|
|
Packit Service |
155747 |
epaddr = tok->tok.epaddr_incoming;
|
|
Packit Service |
155747 |
((am_epaddr_t *) epaddr)->cstate_outgoing =
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_DISC_REPLIED;
|
|
Packit Service |
155747 |
ptl->connect_outgoing--;
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
default:
|
|
Packit Service |
155747 |
psmi_handle_error(PSMI_EP_NORETURN, PSM2_INTERNAL_ERR,
|
|
Packit Service |
155747 |
"Unknown/unhandled connect handler op=%d",
|
|
Packit Service |
155747 |
op);
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
return;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
size_t amsh_sizeof(void)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
return sizeof(struct ptl_am);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Fill in AM capabilities parameters */
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
psmi_amsh_am_get_parameters(psm2_ep_t ep, struct psm2_am_parameters *parameters)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
if (parameters == NULL) {
|
|
Packit Service |
155747 |
return PSM2_PARAM_ERR;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
parameters->max_handlers = PSMI_AM_NUM_HANDLERS;
|
|
Packit Service |
155747 |
parameters->max_nargs = PSMI_AM_MAX_ARGS;
|
|
Packit Service |
155747 |
parameters->max_request_short = AMLONG_MTU;
|
|
Packit Service |
155747 |
parameters->max_reply_short = AMLONG_MTU;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/**
|
|
Packit Service |
155747 |
* @param ep PSM Endpoint, guaranteed to have initialized epaddr and epid.
|
|
Packit Service |
155747 |
* @param ptl Pointer to caller-allocated space for PTL (fill in)
|
|
Packit Service |
155747 |
* @param ctl Pointer to caller-allocated space for PTL-control
|
|
Packit Service |
155747 |
* structure (fill in)
|
|
Packit Service |
155747 |
*/
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_init(psm2_ep_t ep, ptl_t *ptl_gen, ptl_ctl_t *ctl)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Preconditions */
|
|
Packit Service |
155747 |
psmi_assert_always(ep != NULL);
|
|
Packit Service |
155747 |
psmi_assert_always(ep->epaddr != NULL);
|
|
Packit Service |
155747 |
psmi_assert_always(ep->epid != 0);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->ep = ep; /* back pointer */
|
|
Packit Service |
155747 |
ptl->epid = ep->epid; /* cache epid */
|
|
Packit Service |
155747 |
ptl->epaddr = ep->epaddr; /* cache a copy */
|
|
Packit Service |
155747 |
ptl->ctl = ctl;
|
|
Packit Service |
155747 |
ptl->zero_polls = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->connect_phase = 0;
|
|
Packit Service |
155747 |
ptl->connect_incoming = 0;
|
|
Packit Service |
155747 |
ptl->connect_outgoing = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
memset(&ptl->amsh_empty_shortpkt, 0, sizeof(ptl->amsh_empty_shortpkt));
|
|
Packit Service |
155747 |
memset(&ptl->psmi_am_reqq_fifo, 0, sizeof(ptl->psmi_am_reqq_fifo));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->max_ep_idx = -1;
|
|
Packit Service |
155747 |
ptl->am_ep_size = AMSH_DIRBLOCK_SIZE;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->am_ep = (struct am_ctl_nodeinfo *)
|
|
Packit Service |
155747 |
psmi_memalign(ptl->ep, PER_PEER_ENDPOINT, 64,
|
|
Packit Service |
155747 |
ptl->am_ep_size * sizeof(struct am_ctl_nodeinfo));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ptl->am_ep == NULL) {
|
|
Packit Service |
155747 |
err = PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
memset(ptl->am_ep, 0, ptl->am_ep_size * sizeof(struct am_ctl_nodeinfo));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if ((err = amsh_init_segment(ptl_gen)))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->psm_verno = PSMI_VERNO;
|
|
Packit Service |
155747 |
if (ptl->psmi_kassist_mode != PSMI_KASSIST_OFF) {
|
|
Packit Service |
155747 |
if (cma_available()) {
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->amsh_features |=
|
|
Packit Service |
155747 |
AMSH_HAVE_CMA;
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_CMA;
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
ptl->psmi_kassist_mode =
|
|
Packit Service |
155747 |
PSMI_KASSIST_OFF;
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_NO_KASSIST;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else {
|
|
Packit Service |
155747 |
psmi_shm_mq_rv_thresh =
|
|
Packit Service |
155747 |
PSMI_MQ_RV_THRESH_NO_KASSIST;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->pid = getpid();
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->epid = ep->epid;
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->epaddr = ep->epaddr;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ips_mb();
|
|
Packit Service |
155747 |
ptl->self_nodeinfo->is_init = 1;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_am_reqq_init(ptl_gen);
|
|
Packit Service |
155747 |
memset(ctl, 0, sizeof(*ctl));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Fill in the control structure */
|
|
Packit Service |
155747 |
ctl->ep = ep;
|
|
Packit Service |
155747 |
ctl->ptl = ptl_gen;
|
|
Packit Service |
155747 |
ctl->ep_poll = amsh_poll;
|
|
Packit Service |
155747 |
ctl->ep_connect = amsh_ep_connect;
|
|
Packit Service |
155747 |
ctl->ep_disconnect = amsh_ep_disconnect;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ctl->mq_send = amsh_mq_send;
|
|
Packit Service |
155747 |
ctl->mq_isend = amsh_mq_isend;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
ctl->am_get_parameters = psmi_amsh_am_get_parameters;
|
|
Packit Service |
155747 |
ctl->am_short_request = psmi_amsh_am_short_request;
|
|
Packit Service |
155747 |
ctl->am_short_reply = psmi_amsh_am_short_reply;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* No stats in shm (for now...) */
|
|
Packit Service |
155747 |
ctl->epaddr_stats_num = NULL;
|
|
Packit Service |
155747 |
ctl->epaddr_stats_init = NULL;
|
|
Packit Service |
155747 |
ctl->epaddr_stats_get = NULL;
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
union psmi_envvar_val env_memcache_enabled;
|
|
Packit Service |
155747 |
psmi_getenv("PSM2_CUDA_MEMCACHE_ENABLED",
|
|
Packit Service |
155747 |
"PSM cuda ipc memhandle cache enabled (default is enabled)",
|
|
Packit Service |
155747 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT,
|
|
Packit Service |
155747 |
(union psmi_envvar_val)
|
|
Packit Service |
155747 |
1, &env_memcache_enabled);
|
|
Packit Service |
155747 |
if (PSMI_IS_CUDA_ENABLED && env_memcache_enabled.e_uint) {
|
|
Packit Service |
155747 |
union psmi_envvar_val env_memcache_size;
|
|
Packit Service |
155747 |
psmi_getenv("PSM2_CUDA_MEMCACHE_SIZE",
|
|
Packit Service |
155747 |
"Size of the cuda ipc memhandle cache ",
|
|
Packit Service |
155747 |
PSMI_ENVVAR_LEVEL_USER, PSMI_ENVVAR_TYPE_UINT,
|
|
Packit Service |
155747 |
(union psmi_envvar_val)
|
|
Packit Service |
155747 |
CUDA_MEMHANDLE_CACHE_SIZE, &env_memcache_size);
|
|
Packit Service |
155747 |
if ((err = am_cuda_memhandle_mpool_init(env_memcache_size.e_uint)
|
|
Packit Service |
155747 |
!= PSM2_OK))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
if ((err = am_cuda_memhandle_cache_map_init() != PSM2_OK))
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static psm2_error_t amsh_fini(ptl_t *ptl_gen, int force, uint64_t timeout_ns)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
struct ptl_am *ptl = (struct ptl_am *)ptl_gen;
|
|
Packit Service |
155747 |
struct psmi_eptab_iterator itor;
|
|
Packit Service |
155747 |
psm2_epaddr_t epaddr;
|
|
Packit Service |
155747 |
psm2_error_t err = PSM2_OK;
|
|
Packit Service |
155747 |
psm2_error_t err_seg;
|
|
Packit Service |
155747 |
uint64_t t_start = get_cycles();
|
|
Packit Service |
155747 |
int i = 0;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Close whatever has been left open -- this will be factored out for 2.1 */
|
|
Packit Service |
155747 |
if (ptl->connect_outgoing > 0) {
|
|
Packit Service |
155747 |
int num_disc = 0;
|
|
Packit Service |
155747 |
int *mask;
|
|
Packit Service |
155747 |
psm2_error_t *errs;
|
|
Packit Service |
155747 |
psm2_epaddr_t *epaddr_array;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
psmi_epid_itor_init(&itor, ptl->ep);
|
|
Packit Service |
155747 |
while ((epaddr = psmi_epid_itor_next(&itor))) {
|
|
Packit Service |
155747 |
if (epaddr->ptlctl->ptl != ptl_gen)
|
|
Packit Service |
155747 |
continue;
|
|
Packit Service |
155747 |
if (((am_epaddr_t *) epaddr)->cstate_outgoing ==
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_ESTABLISHED)
|
|
Packit Service |
155747 |
num_disc++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
psmi_epid_itor_fini(&itor);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
mask =
|
|
Packit Service |
155747 |
(int *)psmi_calloc(ptl->ep, UNDEFINED, num_disc,
|
|
Packit Service |
155747 |
sizeof(int));
|
|
Packit Service |
155747 |
errs = (psm2_error_t *)
|
|
Packit Service |
155747 |
psmi_calloc(ptl->ep, UNDEFINED, num_disc,
|
|
Packit Service |
155747 |
sizeof(psm2_error_t));
|
|
Packit Service |
155747 |
epaddr_array = (psm2_epaddr_t *)
|
|
Packit Service |
155747 |
psmi_calloc(ptl->ep, UNDEFINED, num_disc,
|
|
Packit Service |
155747 |
sizeof(psm2_epaddr_t));
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (errs == NULL || epaddr_array == NULL || mask == NULL) {
|
|
Packit Service |
155747 |
if (epaddr_array)
|
|
Packit Service |
155747 |
psmi_free(epaddr_array);
|
|
Packit Service |
155747 |
if (errs)
|
|
Packit Service |
155747 |
psmi_free(errs);
|
|
Packit Service |
155747 |
if (mask)
|
|
Packit Service |
155747 |
psmi_free(mask);
|
|
Packit Service |
155747 |
err = PSM2_NO_MEMORY;
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
psmi_epid_itor_init(&itor, ptl->ep);
|
|
Packit Service |
155747 |
while ((epaddr = psmi_epid_itor_next(&itor))) {
|
|
Packit Service |
155747 |
if (epaddr->ptlctl->ptl == ptl_gen) {
|
|
Packit Service |
155747 |
if (((am_epaddr_t *) epaddr)->cstate_outgoing ==
|
|
Packit Service |
155747 |
AMSH_CSTATE_OUTGOING_ESTABLISHED) {
|
|
Packit Service |
155747 |
mask[i] = 1;
|
|
Packit Service |
155747 |
epaddr_array[i] = epaddr;
|
|
Packit Service |
155747 |
i++;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
psmi_epid_itor_fini(&itor);
|
|
Packit Service |
155747 |
psmi_assert(i == num_disc && num_disc > 0);
|
|
Packit Service |
155747 |
err = amsh_ep_disconnect(ptl_gen, force, num_disc, epaddr_array,
|
|
Packit Service |
155747 |
mask, errs, timeout_ns);
|
|
Packit Service |
155747 |
psmi_free(mask);
|
|
Packit Service |
155747 |
psmi_free(errs);
|
|
Packit Service |
155747 |
psmi_free(epaddr_array);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if (ptl->connect_incoming > 0 || ptl->connect_outgoing > 0) {
|
|
Packit Service |
155747 |
while (ptl->connect_incoming > 0 || ptl->connect_outgoing > 0) {
|
|
Packit Service |
155747 |
if (!psmi_cycles_left(t_start, timeout_ns)) {
|
|
Packit Service |
155747 |
err = PSM2_TIMEOUT;
|
|
Packit Service |
155747 |
_HFI_VDBG("CCC timed out with from=%d,to=%d\n",
|
|
Packit Service |
155747 |
ptl->connect_incoming, ptl->connect_outgoing);
|
|
Packit Service |
155747 |
break;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
psmi_poll_internal(ptl->ep, 1);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
} else
|
|
Packit Service |
155747 |
_HFI_VDBG("CCC complete disconnect from=%d,to=%d\n",
|
|
Packit Service |
155747 |
ptl->connect_incoming, ptl->connect_outgoing);
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
if ((err_seg = psmi_shm_detach(ptl_gen))) {
|
|
Packit Service |
155747 |
err = err_seg;
|
|
Packit Service |
155747 |
goto fail;
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* This prevents poll calls between now and the point where the endpoint is
|
|
Packit Service |
155747 |
* deallocated to reference memory that disappeared */
|
|
Packit Service |
155747 |
ptl->repH.head = &ptl->amsh_empty_shortpkt;
|
|
Packit Service |
155747 |
ptl->reqH.head = &ptl->amsh_empty_shortpkt;
|
|
Packit Service |
155747 |
#ifdef PSM_CUDA
|
|
Packit Service |
155747 |
if (PSMI_IS_CUDA_ENABLED)
|
|
Packit Service |
155747 |
am_cuda_memhandle_cache_map_fini();
|
|
Packit Service |
155747 |
#endif
|
|
Packit Service |
155747 |
return PSM2_OK;
|
|
Packit Service |
155747 |
fail:
|
|
Packit Service |
155747 |
return err;
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_setopt(const void *component_obj, int optname,
|
|
Packit Service |
155747 |
const void *optval, uint64_t optlen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
/* No options for AM PTL at the moment */
|
|
Packit Service |
155747 |
return psmi_handle_error(NULL, PSM2_PARAM_ERR,
|
|
Packit Service |
155747 |
"Unknown AM ptl option %u.", optname);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
static
|
|
Packit Service |
155747 |
psm2_error_t
|
|
Packit Service |
155747 |
amsh_getopt(const void *component_obj, int optname,
|
|
Packit Service |
155747 |
void *optval, uint64_t *optlen)
|
|
Packit Service |
155747 |
{
|
|
Packit Service |
155747 |
/* No options for AM PTL at the moment */
|
|
Packit Service |
155747 |
return psmi_handle_error(NULL, PSM2_PARAM_ERR,
|
|
Packit Service |
155747 |
"Unknown AM ptl option %u.", optname);
|
|
Packit Service |
155747 |
}
|
|
Packit Service |
155747 |
|
|
Packit Service |
155747 |
/* Only symbol we expose out of here */
|
|
Packit Service |
155747 |
struct ptl_ctl_init
|
|
Packit Service |
155747 |
psmi_ptl_amsh = {
|
|
Packit Service |
155747 |
amsh_sizeof, amsh_init, amsh_fini, amsh_setopt, amsh_getopt
|
|
Packit Service |
155747 |
};
|