|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This file is provided under a dual BSD/GPLv2 license. When using or
|
|
Packit |
961e70 |
redistributing this file, you may do so under either license.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
GPL LICENSE SUMMARY
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This program is free software; you can redistribute it and/or modify
|
|
Packit |
961e70 |
it under the terms of version 2 of the GNU General Public License as
|
|
Packit |
961e70 |
published by the Free Software Foundation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
This program is distributed in the hope that it will be useful, but
|
|
Packit |
961e70 |
WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
Packit |
961e70 |
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
Packit |
961e70 |
General Public License for more details.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Contact Information:
|
|
Packit |
961e70 |
Intel Corporation, www.intel.com
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
BSD LICENSE
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Copyright(c) 2015 Intel Corporation.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
Redistribution and use in source and binary forms, with or without
|
|
Packit |
961e70 |
modification, are permitted provided that the following conditions
|
|
Packit |
961e70 |
are met:
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
* Redistributions of source code must retain the above copyright
|
|
Packit |
961e70 |
notice, this list of conditions and the following disclaimer.
|
|
Packit |
961e70 |
* Redistributions in binary form must reproduce the above copyright
|
|
Packit |
961e70 |
notice, this list of conditions and the following disclaimer in
|
|
Packit |
961e70 |
the documentation and/or other materials provided with the
|
|
Packit |
961e70 |
distribution.
|
|
Packit |
961e70 |
* Neither the name of Intel Corporation nor the names of its
|
|
Packit |
961e70 |
contributors may be used to endorse or promote products derived
|
|
Packit |
961e70 |
from this software without specific prior written permission.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
Packit |
961e70 |
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
Packit |
961e70 |
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
Packit |
961e70 |
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
Packit |
961e70 |
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
Packit |
961e70 |
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
961e70 |
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
Packit |
961e70 |
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
Packit |
961e70 |
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
Packit |
961e70 |
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
Packit |
961e70 |
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Copyright (c) 2003-2015 Intel Corporation. All rights reserved. */
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
#include "psm_user.h"
|
|
Packit |
961e70 |
#include "psm_mq_internal.h"
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
*
|
|
Packit |
961e70 |
* MQ request allocator
|
|
Packit |
961e70 |
*
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_mq_req_t MOCKABLE(psmi_mq_req_alloc)(psm2_mq_t mq, uint32_t type)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
psm2_mq_req_t req;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_assert(type == MQE_TYPE_RECV || type == MQE_TYPE_SEND);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (type == MQE_TYPE_SEND)
|
|
Packit |
961e70 |
req = psmi_mpool_get(mq->sreq_pool);
|
|
Packit |
961e70 |
else
|
|
Packit |
961e70 |
req = psmi_mpool_get(mq->rreq_pool);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if_pt(req != NULL) {
|
|
Packit |
961e70 |
/* A while ago there were issues about forgetting to zero-out parts of the
|
|
Packit |
961e70 |
* structure, I'm leaving this as a debug-time option */
|
|
Packit |
961e70 |
#ifdef PSM_DEBUG
|
|
Packit |
961e70 |
memset(req, 0, sizeof(struct psm2_mq_req));
|
|
Packit |
961e70 |
#endif
|
|
Packit |
961e70 |
req->type = type;
|
|
Packit |
961e70 |
req->state = MQ_STATE_FREE;
|
|
Packit |
961e70 |
memset(req->next, 0, NUM_MQ_SUBLISTS * sizeof(psm2_mq_req_t));
|
|
Packit |
961e70 |
memset(req->prev, 0, NUM_MQ_SUBLISTS * sizeof(psm2_mq_req_t));
|
|
Packit |
961e70 |
memset(req->q, 0, NUM_MQ_SUBLISTS * sizeof(struct mqq *));
|
|
Packit |
961e70 |
req->req_data.error_code = PSM2_OK;
|
|
Packit |
961e70 |
req->mq = mq;
|
|
Packit |
961e70 |
req->testwait_callback = NULL;
|
|
Packit |
961e70 |
req->rts_peer = NULL;
|
|
Packit |
961e70 |
req->req_data.peer = NULL;
|
|
Packit |
961e70 |
req->ptl_req_ptr = NULL;
|
|
Packit |
961e70 |
#ifdef PSM_CUDA
|
|
Packit |
961e70 |
req->is_buf_gpu_mem = 0;
|
|
Packit |
961e70 |
req->user_gpu_buffer = NULL;
|
|
Packit |
961e70 |
#endif
|
|
Packit |
961e70 |
req->flags_user = 0;
|
|
Packit |
961e70 |
req->flags_internal = 0;
|
|
Packit |
961e70 |
return req;
|
|
Packit |
961e70 |
} else { /* we're out of reqs */
|
|
Packit |
961e70 |
int issend = (type == MQE_TYPE_SEND);
|
|
Packit |
961e70 |
uint32_t reqmax, reqchunk;
|
|
Packit |
961e70 |
psmi_mpool_get_obj_info(issend ? mq->sreq_pool : mq->rreq_pool,
|
|
Packit |
961e70 |
&reqchunk, &reqmax);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psmi_handle_error(PSMI_EP_NORETURN, PSM2_PARAM_ERR,
|
|
Packit |
961e70 |
"Exhausted %d MQ %s request descriptors, which usually indicates "
|
|
Packit |
961e70 |
"a user program error or insufficient request descriptors (%s=%d)",
|
|
Packit |
961e70 |
reqmax, issend ? "isend" : "irecv",
|
|
Packit |
961e70 |
issend ? "PSM2_MQ_SENDREQS_MAX" :
|
|
Packit |
961e70 |
"PSM2_MQ_RECVREQS_MAX", reqmax);
|
|
Packit |
961e70 |
return NULL;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
MOCK_DEF_EPILOGUE(psmi_mq_req_alloc);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
#ifdef PSM_CUDA
|
|
Packit |
961e70 |
void psmi_cuda_recvreq_alloc_func(int is_alloc, void* context, void* obj) {
|
|
Packit |
961e70 |
psm2_mq_req_t recvreq = (psm2_mq_req_t)obj;
|
|
Packit |
961e70 |
if (PSMI_IS_CUDA_ENABLED) {
|
|
Packit |
961e70 |
if (is_alloc)
|
|
Packit |
961e70 |
PSMI_CUDA_CALL(cuEventCreate, &recvreq->cuda_ipc_event, CU_EVENT_DEFAULT);
|
|
Packit |
961e70 |
else
|
|
Packit |
961e70 |
PSMI_CUDA_CALL(cuEventDestroy, recvreq->cuda_ipc_event);
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
return;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
#endif
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t psmi_mq_req_init(psm2_mq_t mq)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
psm2_mq_req_t warmup_req;
|
|
Packit |
961e70 |
psm2_error_t err = PSM2_OK;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
_HFI_VDBG("mq element sizes are %d bytes\n",
|
|
Packit |
961e70 |
(int)sizeof(struct psm2_mq_req));
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
* Send MQ requests
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct psmi_rlimit_mpool rlim = MQ_SENDREQ_LIMITS;
|
|
Packit |
961e70 |
uint32_t maxsz, chunksz;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if ((err =
|
|
Packit |
961e70 |
psmi_parse_mpool_env(mq, 0, &rlim, &maxsz, &chunksz)))
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if ((mq->sreq_pool =
|
|
Packit |
961e70 |
psmi_mpool_create(sizeof(struct psm2_mq_req), chunksz,
|
|
Packit |
961e70 |
maxsz, 0, DESCRIPTORS, NULL,
|
|
Packit |
961e70 |
NULL)) == NULL) {
|
|
Packit |
961e70 |
err = PSM2_NO_MEMORY;
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
* Receive MQ requests
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
struct psmi_rlimit_mpool rlim = MQ_RECVREQ_LIMITS;
|
|
Packit |
961e70 |
uint32_t maxsz, chunksz;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if ((err =
|
|
Packit |
961e70 |
psmi_parse_mpool_env(mq, 0, &rlim, &maxsz, &chunksz)))
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
/* Have a callback function for receive req mpool which creates
|
|
Packit |
961e70 |
* and destroy events.
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
#ifdef PSM_CUDA
|
|
Packit |
961e70 |
if (PSMI_IS_CUDA_ENABLED) {
|
|
Packit |
961e70 |
if ((mq->rreq_pool =
|
|
Packit |
961e70 |
psmi_mpool_create_for_cuda(sizeof(struct psm2_mq_req), chunksz,
|
|
Packit |
961e70 |
maxsz, 0, DESCRIPTORS, NULL,
|
|
Packit |
961e70 |
NULL, psmi_cuda_recvreq_alloc_func, NULL)) == NULL) {
|
|
Packit |
961e70 |
err = PSM2_NO_MEMORY;
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
else {
|
|
Packit |
961e70 |
if ((mq->rreq_pool =
|
|
Packit |
961e70 |
psmi_mpool_create(sizeof(struct psm2_mq_req), chunksz,
|
|
Packit |
961e70 |
maxsz, 0, DESCRIPTORS, NULL,
|
|
Packit |
961e70 |
NULL)) == NULL) {
|
|
Packit |
961e70 |
err = PSM2_NO_MEMORY;
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
#else
|
|
Packit |
961e70 |
if ((mq->rreq_pool =
|
|
Packit |
961e70 |
psmi_mpool_create(sizeof(struct psm2_mq_req), chunksz,
|
|
Packit |
961e70 |
maxsz, 0, DESCRIPTORS, NULL,
|
|
Packit |
961e70 |
NULL)) == NULL) {
|
|
Packit |
961e70 |
err = PSM2_NO_MEMORY;
|
|
Packit |
961e70 |
goto fail;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
#endif
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/* Warm up the allocators */
|
|
Packit |
961e70 |
warmup_req = psmi_mq_req_alloc(mq, MQE_TYPE_RECV);
|
|
Packit |
961e70 |
psmi_assert_always(warmup_req != NULL);
|
|
Packit |
961e70 |
psmi_mq_req_free(warmup_req);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
warmup_req = psmi_mq_req_alloc(mq, MQE_TYPE_SEND);
|
|
Packit |
961e70 |
psmi_assert_always(warmup_req != NULL);
|
|
Packit |
961e70 |
psmi_mq_req_free(warmup_req);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
fail:
|
|
Packit |
961e70 |
return err;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_error_t psmi_mq_req_fini(psm2_mq_t mq)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
psmi_mpool_destroy(mq->rreq_pool);
|
|
Packit |
961e70 |
psmi_mpool_destroy(mq->sreq_pool);
|
|
Packit |
961e70 |
return PSM2_OK;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
* Hooks to plug into QLogic MPI stats
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
static
|
|
Packit |
961e70 |
void psmi_mq_stats_callback(struct mpspawn_stats_req_args *args)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
uint64_t *entry = args->stats;
|
|
Packit |
961e70 |
psm2_mq_t mq = (psm2_mq_t) args->context;
|
|
Packit |
961e70 |
psm2_mq_stats_t mqstats;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
psm2_mq_get_stats(mq, &mqstats);
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
if (args->num < 8)
|
|
Packit |
961e70 |
return;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
entry[0] = mqstats.tx_eager_num;
|
|
Packit |
961e70 |
entry[1] = mqstats.tx_eager_bytes;
|
|
Packit |
961e70 |
entry[2] = mqstats.tx_rndv_num;
|
|
Packit |
961e70 |
entry[3] = mqstats.tx_rndv_bytes;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
entry[4] = mqstats.rx_user_num;
|
|
Packit |
961e70 |
entry[5] = mqstats.rx_user_bytes;
|
|
Packit |
961e70 |
entry[6] = mqstats.rx_sys_num;
|
|
Packit |
961e70 |
entry[7] = mqstats.rx_sys_bytes;
|
|
Packit |
961e70 |
}
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
void psmi_mq_stats_register(psm2_mq_t mq, mpspawn_stats_add_fn add_fn)
|
|
Packit |
961e70 |
{
|
|
Packit |
961e70 |
char *desc[8];
|
|
Packit |
961e70 |
uint16_t flags[8];
|
|
Packit |
961e70 |
int i;
|
|
Packit |
961e70 |
struct mpspawn_stats_add_args mp_add;
|
|
Packit |
961e70 |
/*
|
|
Packit |
961e70 |
* Hardcode flags until we correctly move mpspawn to its own repo.
|
|
Packit |
961e70 |
* flags[i] = MPSPAWN_REDUCTION_MAX | MPSPAWN_REDUCTION_MIN;
|
|
Packit |
961e70 |
*/
|
|
Packit |
961e70 |
for (i = 0; i < 8; i++)
|
|
Packit |
961e70 |
flags[i] = MPSPAWN_STATS_REDUCTION_ALL;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
desc[0] = "Eager count sent";
|
|
Packit |
961e70 |
desc[1] = "Eager bytes sent";
|
|
Packit |
961e70 |
desc[2] = "Rendezvous count sent";
|
|
Packit |
961e70 |
desc[3] = "Rendezvous bytes sent";
|
|
Packit |
961e70 |
desc[4] = "Expected count received";
|
|
Packit |
961e70 |
desc[5] = "Expected bytes received";
|
|
Packit |
961e70 |
desc[6] = "Unexpect count received";
|
|
Packit |
961e70 |
desc[7] = "Unexpect bytes received";
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
mp_add.version = MPSPAWN_STATS_VERSION;
|
|
Packit |
961e70 |
mp_add.num = 8;
|
|
Packit |
961e70 |
mp_add.header = "MPI Statistics Summary (max,min @ rank)";
|
|
Packit |
961e70 |
mp_add.req_fn = psmi_mq_stats_callback;
|
|
Packit |
961e70 |
mp_add.desc = desc;
|
|
Packit |
961e70 |
mp_add.flags = flags;
|
|
Packit |
961e70 |
mp_add.context = mq;
|
|
Packit |
961e70 |
|
|
Packit |
961e70 |
add_fn(&mp_add);
|
|
Packit |
961e70 |
}
|