/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
*
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#include "mpiimpl.h"
static int unweighted_dummy = 0x46618;
static int weights_empty_dummy = 0x022284;
/* cannot ==NULL, would be ambiguous */
int *const MPI_UNWEIGHTED = &unweighted_dummy;
int *const MPI_WEIGHTS_EMPTY = &weights_empty_dummy;
/* Keyval for topology information */
static int MPIR_Topology_keyval = MPI_KEYVAL_INVALID;
/* Local functions */
static int MPIR_Topology_copy_fn(MPI_Comm, int, void *, void *, void *, int *);
static int MPIR_Topology_delete_fn(MPI_Comm, int, void *, void *);
static int MPIR_Topology_finalize(void *);
/*
Return a poiner to the topology structure on a communicator.
Returns null if no topology structure is defined
*/
MPIR_Topology *MPIR_Topology_get(MPIR_Comm * comm_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *topo_ptr;
int flag;
if (MPIR_Topology_keyval == MPI_KEYVAL_INVALID) {
return 0;
}
mpi_errno = MPII_Comm_get_attr(comm_ptr->handle, MPIR_Topology_keyval,
&topo_ptr, &flag, MPIR_ATTR_PTR);
if (mpi_errno)
return NULL;
if (flag)
return topo_ptr;
return NULL;
}
#undef FUNCNAME
#define FUNCNAME MPIR_Topology_put
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPIR_Topology_put(MPIR_Comm * comm_ptr, MPIR_Topology * topo_ptr)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Assert(comm_ptr != NULL);
if (MPIR_Topology_keyval == MPI_KEYVAL_INVALID) {
/* Create a new keyval */
/* FIXME - thread safe code needs a thread lock here, followed
* by another test on the keyval to see if a different thread
* got there first */
mpi_errno = MPIR_Comm_create_keyval_impl(MPIR_Topology_copy_fn,
MPIR_Topology_delete_fn, &MPIR_Topology_keyval, 0);
/* Register the finalize handler */
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_Add_finalize(MPIR_Topology_finalize, (void *) 0, MPIR_FINALIZE_CALLBACK_PRIO - 1);
}
mpi_errno = MPIR_Comm_set_attr_impl(comm_ptr, MPIR_Topology_keyval, topo_ptr, MPIR_ATTR_PTR);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
/* Ignore p */
static int MPIR_Topology_finalize(void *p ATTRIBUTE((unused)))
{
MPL_UNREFERENCED_ARG(p);
if (MPIR_Topology_keyval != MPI_KEYVAL_INVALID) {
/* Just in case */
MPIR_Comm_free_keyval_impl(MPIR_Topology_keyval);
MPIR_Topology_keyval = MPI_KEYVAL_INVALID;
}
return 0;
}
static int *MPIR_Copy_array(int n, const int a[], int *err)
{
int *new_p;
/* the copy of NULL is NULL */
if (a == NULL) {
MPIR_Assert(n == 0);
return NULL;
}
new_p = (int *) MPL_malloc(n * sizeof(int), MPL_MEM_OTHER);
/* --BEGIN ERROR HANDLING-- */
if (!new_p) {
*err = MPI_ERR_OTHER;
return 0;
}
/* --END ERROR HANDLING-- */
MPIR_Memcpy(new_p, a, n * sizeof(int));
return new_p;
}
/* The keyval copy and delete functions must handle copying and deleting
the associated topology structures
We can reduce the number of allocations by making a single allocation
of enough integers for all fields (including the ones in the structure)
and freeing the single object later.
*/
#undef FUNCNAME
#define FUNCNAME MPIR_Topology_copy_fn
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static int MPIR_Topology_copy_fn(MPI_Comm comm ATTRIBUTE((unused)),
int keyval ATTRIBUTE((unused)),
void *extra_data ATTRIBUTE((unused)),
void *attr_in, void *attr_out, int *flag)
{
MPIR_Topology *old_topology = (MPIR_Topology *) attr_in;
MPIR_Topology *copy_topology = NULL;
MPIR_CHKPMEM_DECL(5);
int mpi_errno = 0;
MPL_UNREFERENCED_ARG(comm);
MPL_UNREFERENCED_ARG(keyval);
MPL_UNREFERENCED_ARG(extra_data);
*flag = 0;
*(void **) attr_out = NULL;
MPIR_CHKPMEM_MALLOC(copy_topology, MPIR_Topology *, sizeof(MPIR_Topology), mpi_errno,
"copy_topology", MPL_MEM_OTHER);
MPL_VG_MEM_INIT(copy_topology, sizeof(MPIR_Topology));
/* simplify copying and error handling */
#define MPIR_ARRAY_COPY_HELPER(kind_,array_field_,count_field_) \
do { \
copy_topology->topo.kind_.array_field_ = \
MPIR_Copy_array(old_topology->topo.kind_.count_field_, \
old_topology->topo.kind_.array_field_, \
&mpi_errno); \
if (mpi_errno) MPIR_ERR_POP(mpi_errno); \
MPIR_CHKPMEM_REGISTER(copy_topology->topo.kind_.array_field_); \
} while (0)
copy_topology->kind = old_topology->kind;
if (old_topology->kind == MPI_CART) {
copy_topology->topo.cart.ndims = old_topology->topo.cart.ndims;
copy_topology->topo.cart.nnodes = old_topology->topo.cart.nnodes;
MPIR_ARRAY_COPY_HELPER(cart, dims, ndims);
MPIR_ARRAY_COPY_HELPER(cart, periodic, ndims);
MPIR_ARRAY_COPY_HELPER(cart, position, ndims);
} else if (old_topology->kind == MPI_GRAPH) {
copy_topology->topo.graph.nnodes = old_topology->topo.graph.nnodes;
copy_topology->topo.graph.nedges = old_topology->topo.graph.nedges;
MPIR_ARRAY_COPY_HELPER(graph, index, nnodes);
MPIR_ARRAY_COPY_HELPER(graph, edges, nedges);
} else if (old_topology->kind == MPI_DIST_GRAPH) {
copy_topology->topo.dist_graph.indegree = old_topology->topo.dist_graph.indegree;
copy_topology->topo.dist_graph.outdegree = old_topology->topo.dist_graph.outdegree;
MPIR_ARRAY_COPY_HELPER(dist_graph, in, indegree);
MPIR_ARRAY_COPY_HELPER(dist_graph, in_weights, indegree);
MPIR_ARRAY_COPY_HELPER(dist_graph, out, outdegree);
MPIR_ARRAY_COPY_HELPER(dist_graph, out_weights, outdegree);
}
/* --BEGIN ERROR HANDLING-- */
else {
/* Unknown topology */
return MPI_ERR_TOPOLOGY;
}
/* --END ERROR HANDLING-- */
#undef MPIR_ARRAY_COPY_HELPER
*(void **) attr_out = (void *) copy_topology;
*flag = 1;
MPIR_CHKPMEM_COMMIT();
fn_exit:
/* Return mpi_errno in case one of the copy array functions failed */
return mpi_errno;
fn_fail:
/* --BEGIN ERROR HANDLING-- */
MPIR_CHKPMEM_REAP();
goto fn_exit;
/* --END ERROR HANDLING-- */
}
#undef FUNCNAME
#define FUNCNAME MPIR_Topology_delete_fn
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
static int MPIR_Topology_delete_fn(MPI_Comm comm ATTRIBUTE((unused)),
int keyval ATTRIBUTE((unused)),
void *attr_val, void *extra_data ATTRIBUTE((unused)))
{
MPIR_Topology *topology = (MPIR_Topology *) attr_val;
MPL_UNREFERENCED_ARG(comm);
MPL_UNREFERENCED_ARG(keyval);
MPL_UNREFERENCED_ARG(extra_data);
/* FIXME - free the attribute data structure */
if (topology->kind == MPI_CART) {
MPL_free(topology->topo.cart.dims);
MPL_free(topology->topo.cart.periodic);
MPL_free(topology->topo.cart.position);
MPL_free(topology);
} else if (topology->kind == MPI_GRAPH) {
MPL_free(topology->topo.graph.index);
MPL_free(topology->topo.graph.edges);
MPL_free(topology);
} else if (topology->kind == MPI_DIST_GRAPH) {
MPL_free(topology->topo.dist_graph.in);
MPL_free(topology->topo.dist_graph.out);
if (topology->topo.dist_graph.in_weights)
MPL_free(topology->topo.dist_graph.in_weights);
if (topology->topo.dist_graph.out_weights)
MPL_free(topology->topo.dist_graph.out_weights);
MPL_free(topology);
}
/* --BEGIN ERROR HANDLING-- */
else {
return MPI_ERR_TOPOLOGY;
}
/* --END ERROR HANDLING-- */
return MPI_SUCCESS;
}
/* the next two routines implement the following behavior (quoted from Section
* 7.6 of the MPI-3.0 standard):
*
* For a distributed graph topology, created with MPI_DIST_GRAPH_CREATE, the
* sequence of neighbors in the send and receive buffers at each process
* is defined as the sequence returned by MPI_DIST_GRAPH_NEIGHBORS for
* destinations and sources, respectively. For a general graph topology,
* created with MPI_GRAPH_CREATE, the order of neighbors in the send and
* receive buffers is defined as the sequence of neighbors as returned by
* MPI_GRAPH_NEIGHBORS. Note that general graph topologies should generally
* be replaced by the distributed graph topologies.
*
* For a Cartesian topology, created with MPI_CART_CREATE, the sequence of
* neighbors in the send and receive buffers at each process is defined by
* order of the dimensions, first the neighbor in the negative direction and
* then in the positive direction with displacement 1. The numbers of
* sources and destinations in the communication routines are 2*ndims with
* ndims defined in MPI_CART_CREATE. If a neighbor does not exist, i.e., at
* the border of a Cartesian topology in the case of a non-periodic virtual
* grid dimension (i.e., periods[...]==false), then this neighbor is defined
* to be MPI_PROC_NULL.
*/
#undef FUNCNAME
#define FUNCNAME MPIR_Topo_canon_nhb_count
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPIR_Topo_canon_nhb_count(MPIR_Comm * comm_ptr, int *indegree, int *outdegree, int *weighted)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *topo_ptr;
topo_ptr = MPIR_Topology_get(comm_ptr);
MPIR_ERR_CHKANDJUMP(!topo_ptr, mpi_errno, MPI_ERR_TOPOLOGY, "**notopology");
/* TODO consider dispatching via a vtable instead of doing if/else */
if (topo_ptr->kind == MPI_DIST_GRAPH) {
mpi_errno = MPIR_Dist_graph_neighbors_count_impl(comm_ptr, indegree, outdegree, weighted);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
} else if (topo_ptr->kind == MPI_GRAPH) {
int nneighbors = 0;
mpi_errno = MPIR_Graph_neighbors_count_impl(comm_ptr, comm_ptr->rank, &nneighbors);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
*indegree = *outdegree = nneighbors;
*weighted = FALSE;
} else if (topo_ptr->kind == MPI_CART) {
*indegree = 2 * topo_ptr->topo.cart.ndims;
*outdegree = 2 * topo_ptr->topo.cart.ndims;
*weighted = FALSE;
} else {
MPIR_Assert(FALSE);
}
fn_exit:
return mpi_errno;
fn_fail:
goto fn_exit;
}
#undef FUNCNAME
#define FUNCNAME MPIR_Topo_canon_nhb
#undef FCNAME
#define FCNAME MPL_QUOTE(FUNCNAME)
int MPIR_Topo_canon_nhb(MPIR_Comm * comm_ptr,
int indegree, int sources[], int inweights[],
int outdegree, int dests[], int outweights[])
{
int mpi_errno = MPI_SUCCESS;
MPIR_Topology *topo_ptr;
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPIR_TOPO_CANON_NHB);
MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPIR_TOPO_CANON_NHB);
topo_ptr = MPIR_Topology_get(comm_ptr);
MPIR_ERR_CHKANDJUMP(!topo_ptr, mpi_errno, MPI_ERR_TOPOLOGY, "**notopology");
/* TODO consider dispatching via a vtable instead of doing if/else */
if (topo_ptr->kind == MPI_DIST_GRAPH) {
mpi_errno = MPIR_Dist_graph_neighbors_impl(comm_ptr,
indegree, sources, inweights,
outdegree, dests, outweights);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
} else if (topo_ptr->kind == MPI_GRAPH) {
MPIR_Assert(indegree == outdegree);
mpi_errno = MPIR_Graph_neighbors_impl(comm_ptr, comm_ptr->rank, indegree, sources);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
MPIR_Memcpy(dests, sources, outdegree * sizeof(*dests));
/* ignore inweights/outweights */
} else if (topo_ptr->kind == MPI_CART) {
int d;
MPIR_Assert(indegree == outdegree);
MPIR_Assert(indegree == 2 * topo_ptr->topo.cart.ndims);
for (d = 0; d < topo_ptr->topo.cart.ndims; ++d) {
mpi_errno = MPIR_Cart_shift_impl(comm_ptr, d, 1, &sources[2 * d], &sources[2 * d + 1]);
if (mpi_errno)
MPIR_ERR_POP(mpi_errno);
dests[2 * d] = sources[2 * d];
dests[2 * d + 1] = sources[2 * d + 1];
}
/* ignore inweights/outweights */
} else {
MPIR_Assert(FALSE);
}
#ifdef MPL_USE_DBG_LOGGING
{
int i;
MPL_DBG_MSG_FMT(MPIR_DBG_PT2PT, VERBOSE,
(MPL_DBG_FDEST, "canonical neighbors for comm=0x%x comm_ptr=%p",
comm_ptr->handle, comm_ptr));
for (i = 0; i < outdegree; ++i) {
MPL_DBG_MSG_FMT(MPIR_DBG_PT2PT, VERBOSE,
(MPL_DBG_FDEST, "%d/%d: to %d", i, outdegree, dests[i]));
}
for (i = 0; i < indegree; ++i) {
MPL_DBG_MSG_FMT(MPIR_DBG_PT2PT, VERBOSE,
(MPL_DBG_FDEST, "%d/%d: from %d", i, indegree, sources[i]));
}
}
#endif
fn_exit:
MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPIR_TOPO_CANON_NHB);
return mpi_errno;
fn_fail:
goto fn_exit;
}