|
Packit Service |
c5cf8c |
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
|
|
Packit Service |
c5cf8c |
/*
|
|
Packit Service |
c5cf8c |
*
|
|
Packit Service |
c5cf8c |
* (C) 2001 by Argonne National Laboratory.
|
|
Packit Service |
c5cf8c |
* See COPYRIGHT in top-level directory.
|
|
Packit Service |
c5cf8c |
*/
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#include "mpiimpl.h"
|
|
Packit Service |
c5cf8c |
#include "mpicomm.h"
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_HWLOC
|
|
Packit Service |
c5cf8c |
#include "hwloc.h"
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_NETLOC
|
|
Packit Service |
c5cf8c |
#include "netloc_util.h"
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* -- Begin Profiling Symbol Block for routine MPI_Comm_split_type */
|
|
Packit Service |
c5cf8c |
#if defined(HAVE_PRAGMA_WEAK)
|
|
Packit Service |
c5cf8c |
#pragma weak MPI_Comm_split_type = PMPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
|
|
Packit Service |
c5cf8c |
#pragma _HP_SECONDARY_DEF PMPI_Comm_split_type MPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
#elif defined(HAVE_PRAGMA_CRI_DUP)
|
|
Packit Service |
c5cf8c |
#pragma _CRI duplicate MPI_Comm_split_type as PMPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
#elif defined(HAVE_WEAK_ATTRIBUTE)
|
|
Packit Service |
c5cf8c |
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key, MPI_Info info, MPI_Comm * newcomm)
|
|
Packit Service |
c5cf8c |
__attribute__ ((weak, alias("PMPI_Comm_split_type")));
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
/* -- End Profiling Symbol Block */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Define MPICH_MPI_FROM_PMPI if weak symbols are not supported to build
|
|
Packit Service |
c5cf8c |
the MPI routines */
|
|
Packit Service |
c5cf8c |
#ifndef MPICH_MPI_FROM_PMPI
|
|
Packit Service |
c5cf8c |
#undef MPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
#define MPI_Comm_split_type PMPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_self
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_self(MPIR_Comm * user_comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_self_ptr;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* split out the undefined processes */
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(user_comm_ptr, split_type == MPI_UNDEFINED ? MPI_UNDEFINED : 0,
|
|
Packit Service |
c5cf8c |
key, &comm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (split_type == MPI_UNDEFINED) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPIR_Comm_get_ptr(MPI_COMM_SELF, comm_self_ptr);
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_dup_impl(comm_self_ptr, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
if (comm_ptr)
|
|
Packit Service |
c5cf8c |
MPIR_Comm_free_impl(comm_ptr);
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_node
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_node(MPIR_Comm * user_comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* split out the undefined processes */
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(user_comm_ptr, split_type == MPI_UNDEFINED ? MPI_UNDEFINED : 0,
|
|
Packit Service |
c5cf8c |
key, &comm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (split_type == MPI_UNDEFINED) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPID_Get_node_id(comm_ptr, comm_ptr->rank, &color;;
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
if (comm_ptr)
|
|
Packit Service |
c5cf8c |
MPIR_Comm_free_impl(comm_ptr);
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_HWLOC
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
struct shmem_processor_info_table {
|
|
Packit Service |
c5cf8c |
const char *val;
|
|
Packit Service |
c5cf8c |
hwloc_obj_type_t obj_type;
|
|
Packit Service |
c5cf8c |
};
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* hwloc processor object table */
|
|
Packit Service |
c5cf8c |
static struct shmem_processor_info_table shmem_processor_info[] = {
|
|
Packit Service |
c5cf8c |
{"machine", HWLOC_OBJ_MACHINE},
|
|
Packit Service |
c5cf8c |
{"socket", HWLOC_OBJ_PACKAGE},
|
|
Packit Service |
c5cf8c |
{"package", HWLOC_OBJ_PACKAGE},
|
|
Packit Service |
c5cf8c |
{"numa", HWLOC_OBJ_NUMANODE},
|
|
Packit Service |
c5cf8c |
{"core", HWLOC_OBJ_CORE},
|
|
Packit Service |
c5cf8c |
{"hwthread", HWLOC_OBJ_PU},
|
|
Packit Service |
c5cf8c |
{"pu", HWLOC_OBJ_PU},
|
|
Packit Service |
c5cf8c |
{"l1dcache", HWLOC_OBJ_L1CACHE},
|
|
Packit Service |
c5cf8c |
{"l1ucache", HWLOC_OBJ_L1CACHE},
|
|
Packit Service |
c5cf8c |
{"l1icache", HWLOC_OBJ_L1ICACHE},
|
|
Packit Service |
c5cf8c |
{"l1cache", HWLOC_OBJ_L1CACHE},
|
|
Packit Service |
c5cf8c |
{"l2dcache", HWLOC_OBJ_L2CACHE},
|
|
Packit Service |
c5cf8c |
{"l2ucache", HWLOC_OBJ_L2CACHE},
|
|
Packit Service |
c5cf8c |
{"l2icache", HWLOC_OBJ_L2ICACHE},
|
|
Packit Service |
c5cf8c |
{"l2cache", HWLOC_OBJ_L2CACHE},
|
|
Packit Service |
c5cf8c |
{"l3dcache", HWLOC_OBJ_L3CACHE},
|
|
Packit Service |
c5cf8c |
{"l3ucache", HWLOC_OBJ_L3CACHE},
|
|
Packit Service |
c5cf8c |
{"l3icache", HWLOC_OBJ_L3ICACHE},
|
|
Packit Service |
c5cf8c |
{"l3cache", HWLOC_OBJ_L3CACHE},
|
|
Packit Service |
c5cf8c |
{"l4dcache", HWLOC_OBJ_L4CACHE},
|
|
Packit Service |
c5cf8c |
{"l4ucache", HWLOC_OBJ_L4CACHE},
|
|
Packit Service |
c5cf8c |
{"l4cache", HWLOC_OBJ_L4CACHE},
|
|
Packit Service |
c5cf8c |
{"l5dcache", HWLOC_OBJ_L5CACHE},
|
|
Packit Service |
c5cf8c |
{"l5ucache", HWLOC_OBJ_L5CACHE},
|
|
Packit Service |
c5cf8c |
{"l5cache", HWLOC_OBJ_L5CACHE},
|
|
Packit Service |
c5cf8c |
{NULL, HWLOC_OBJ_TYPE_MAX}
|
|
Packit Service |
c5cf8c |
};
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int node_split_processor(MPIR_Comm * comm_ptr, int key, const char *hintval,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
hwloc_obj_t obj_containing_cpuset;
|
|
Packit Service |
c5cf8c |
hwloc_obj_type_t query_obj_type = HWLOC_OBJ_TYPE_MAX;
|
|
Packit Service |
c5cf8c |
int i, mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* assign the node id as the color, initially */
|
|
Packit Service |
c5cf8c |
MPID_Get_node_id(comm_ptr, comm_ptr->rank, &color;;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* try to find the info value in the processor object
|
|
Packit Service |
c5cf8c |
* table */
|
|
Packit Service |
c5cf8c |
for (i = 0; shmem_processor_info[i].val; i++) {
|
|
Packit Service |
c5cf8c |
if (!strcmp(shmem_processor_info[i].val, hintval)) {
|
|
Packit Service |
c5cf8c |
query_obj_type = shmem_processor_info[i].obj_type;
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (query_obj_type == HWLOC_OBJ_TYPE_MAX)
|
|
Packit Service |
c5cf8c |
goto split_id;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
obj_containing_cpuset =
|
|
Packit Service |
c5cf8c |
hwloc_get_obj_covering_cpuset(MPIR_Process.hwloc_topology, MPIR_Process.bindset);
|
|
Packit Service |
c5cf8c |
MPIR_Assert(obj_containing_cpuset != NULL);
|
|
Packit Service |
c5cf8c |
if (obj_containing_cpuset->type == query_obj_type) {
|
|
Packit Service |
c5cf8c |
color = obj_containing_cpuset->logical_index;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
hwloc_obj_t hobj = NULL;
|
|
Packit Service |
c5cf8c |
hwloc_obj_t tmp = NULL;
|
|
Packit Service |
c5cf8c |
/* hwloc_get_ancestor_of_type call cannot be used here because HWLOC version 2.0 and above do not
|
|
Packit Service |
c5cf8c |
* treat memory objects (NUMA) as objects in topology tree (Details can be found in
|
|
Packit Service |
c5cf8c |
* https://www.open-mpi.org/projects/hwloc/doc/v2.0.1/a00327.php#upgrade_to_api_2x_memory_find)
|
|
Packit Service |
c5cf8c |
*/
|
|
Packit Service |
c5cf8c |
while ((tmp =
|
|
Packit Service |
c5cf8c |
hwloc_get_next_obj_by_type(MPIR_Process.hwloc_topology, query_obj_type,
|
|
Packit Service |
c5cf8c |
tmp)) != NULL) {
|
|
Packit Service |
c5cf8c |
if (hwloc_bitmap_isincluded(obj_containing_cpuset->cpuset, tmp->cpuset) ||
|
|
Packit Service |
c5cf8c |
hwloc_bitmap_isequal(tmp->cpuset, obj_containing_cpuset->cpuset)) {
|
|
Packit Service |
c5cf8c |
hobj = tmp;
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (hobj)
|
|
Packit Service |
c5cf8c |
color = hobj->logical_index;
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
split_id:
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int node_split_pci_device(MPIR_Comm * comm_ptr, int key,
|
|
Packit Service |
c5cf8c |
const char *hintval, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
hwloc_obj_t obj_containing_cpuset, io_device = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
obj_containing_cpuset =
|
|
Packit Service |
c5cf8c |
hwloc_get_obj_covering_cpuset(MPIR_Process.hwloc_topology, MPIR_Process.bindset);
|
|
Packit Service |
c5cf8c |
MPIR_Assert(obj_containing_cpuset != NULL);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
io_device =
|
|
Packit Service |
c5cf8c |
hwloc_get_pcidev_by_busidstring(MPIR_Process.hwloc_topology, hintval + strlen("pci:"));
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (io_device != NULL) {
|
|
Packit Service |
c5cf8c |
hwloc_obj_t non_io_ancestor =
|
|
Packit Service |
c5cf8c |
hwloc_get_non_io_ancestor_obj(MPIR_Process.hwloc_topology, io_device);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* An io object will never be the root of the topology and is
|
|
Packit Service |
c5cf8c |
* hence guaranteed to have a non io ancestor */
|
|
Packit Service |
c5cf8c |
MPIR_Assert(non_io_ancestor);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (hwloc_obj_is_in_subtree
|
|
Packit Service |
c5cf8c |
(MPIR_Process.hwloc_topology, obj_containing_cpuset, non_io_ancestor)) {
|
|
Packit Service |
c5cf8c |
color = non_io_ancestor->logical_index;
|
|
Packit Service |
c5cf8c |
} else
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
} else
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int io_device_found(const char *resource, const char *devname, hwloc_obj_t io_device,
|
|
Packit Service |
c5cf8c |
hwloc_obj_osdev_type_t obj_type)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
if (!strncmp(resource, devname, strlen(devname))) {
|
|
Packit Service |
c5cf8c |
/* device type does not match */
|
|
Packit Service |
c5cf8c |
if (io_device->attr->osdev.type != obj_type)
|
|
Packit Service |
c5cf8c |
return 0;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* device prefix does not match */
|
|
Packit Service |
c5cf8c |
if (strncmp(io_device->name, devname, strlen(devname)))
|
|
Packit Service |
c5cf8c |
return 0;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* specific device is supplied, but does not match */
|
|
Packit Service |
c5cf8c |
if (strlen(resource) != strlen(devname) && strcmp(io_device->name, resource))
|
|
Packit Service |
c5cf8c |
return 0;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
return 1;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int node_split_network_device(MPIR_Comm * comm_ptr, int key,
|
|
Packit Service |
c5cf8c |
const char *hintval, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
hwloc_obj_t obj_containing_cpuset, io_device = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* assign the node id as the color, initially */
|
|
Packit Service |
c5cf8c |
MPID_Get_node_id(comm_ptr, comm_ptr->rank, &color;;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
obj_containing_cpuset =
|
|
Packit Service |
c5cf8c |
hwloc_get_obj_covering_cpuset(MPIR_Process.hwloc_topology, MPIR_Process.bindset);
|
|
Packit Service |
c5cf8c |
MPIR_Assert(obj_containing_cpuset != NULL);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
while ((io_device = hwloc_get_next_osdev(MPIR_Process.hwloc_topology, io_device))) {
|
|
Packit Service |
c5cf8c |
hwloc_obj_t non_io_ancestor;
|
|
Packit Service |
c5cf8c |
uint32_t depth;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (!io_device_found(hintval, "hfi", io_device, HWLOC_OBJ_OSDEV_OPENFABRICS))
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
if (!io_device_found(hintval, "ib", io_device, HWLOC_OBJ_OSDEV_NETWORK))
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
if (!io_device_found(hintval, "eth", io_device, HWLOC_OBJ_OSDEV_NETWORK) &&
|
|
Packit Service |
c5cf8c |
!io_device_found(hintval, "en", io_device, HWLOC_OBJ_OSDEV_NETWORK))
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
non_io_ancestor = hwloc_get_non_io_ancestor_obj(MPIR_Process.hwloc_topology, io_device);
|
|
Packit Service |
c5cf8c |
while (!hwloc_obj_type_is_normal(non_io_ancestor->type))
|
|
Packit Service |
c5cf8c |
non_io_ancestor = non_io_ancestor->parent;
|
|
Packit Service |
c5cf8c |
MPIR_Assert(non_io_ancestor && non_io_ancestor->depth >= 0);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (!hwloc_obj_is_in_subtree
|
|
Packit Service |
c5cf8c |
(MPIR_Process.hwloc_topology, obj_containing_cpuset, non_io_ancestor))
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Get a unique ID for the non-IO object. Use fixed width
|
|
Packit Service |
c5cf8c |
* unsigned integers, so bit shift operations are well
|
|
Packit Service |
c5cf8c |
* defined */
|
|
Packit Service |
c5cf8c |
depth = (uint32_t) non_io_ancestor->depth;
|
|
Packit Service |
c5cf8c |
color = (int) ((depth << 16) + non_io_ancestor->logical_index);
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int node_split_gpu_device(MPIR_Comm * comm_ptr, int key,
|
|
Packit Service |
c5cf8c |
const char *hintval, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
hwloc_obj_t obj_containing_cpuset, io_device = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
obj_containing_cpuset =
|
|
Packit Service |
c5cf8c |
hwloc_get_obj_covering_cpuset(MPIR_Process.hwloc_topology, MPIR_Process.bindset);
|
|
Packit Service |
c5cf8c |
MPIR_Assert(obj_containing_cpuset != NULL);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
while ((io_device = hwloc_get_next_osdev(MPIR_Process.hwloc_topology, io_device))
|
|
Packit Service |
c5cf8c |
!= NULL) {
|
|
Packit Service |
c5cf8c |
if (io_device->attr->osdev.type == HWLOC_OBJ_OSDEV_GPU) {
|
|
Packit Service |
c5cf8c |
if ((*(hintval + strlen("gpu")) != '\0') &&
|
|
Packit Service |
c5cf8c |
atoi(hintval + strlen("gpu")) != io_device->logical_index)
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
hwloc_obj_t non_io_ancestor =
|
|
Packit Service |
c5cf8c |
hwloc_get_non_io_ancestor_obj(MPIR_Process.hwloc_topology, io_device);
|
|
Packit Service |
c5cf8c |
MPIR_Assert(non_io_ancestor);
|
|
Packit Service |
c5cf8c |
if (hwloc_obj_is_in_subtree
|
|
Packit Service |
c5cf8c |
(MPIR_Process.hwloc_topology, obj_containing_cpuset, non_io_ancestor)) {
|
|
Packit Service |
c5cf8c |
color =
|
|
Packit Service |
c5cf8c |
(non_io_ancestor->type << (sizeof(int) * 4)) + non_io_ancestor->logical_index;
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
#endif /* HAVE_HWLOC */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_NETLOC
|
|
Packit Service |
c5cf8c |
static int network_split_switch_level(MPIR_Comm * comm_ptr, int key,
|
|
Packit Service |
c5cf8c |
int switch_level, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int i, color;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
netloc_node_t *network_node;
|
|
Packit Service |
c5cf8c |
netloc_node_t **traversal_stack;
|
|
Packit Service |
c5cf8c |
int traversal_begin, traversal_end;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__FAT_TREE ||
|
|
Packit Service |
c5cf8c |
MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__CLOS_NETWORK) {
|
|
Packit Service |
c5cf8c |
netloc_node_t **switches_at_level;
|
|
Packit Service |
c5cf8c |
int switch_count;
|
|
Packit Service |
c5cf8c |
traversal_stack =
|
|
Packit Service |
c5cf8c |
(netloc_node_t **) MPL_malloc(sizeof(netloc_node_t *) *
|
|
Packit Service |
c5cf8c |
MPIR_Process.netloc_topology->num_nodes, MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
network_node = MPIR_Process.network_attr.network_endpoint;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
traversal_begin = 0;
|
|
Packit Service |
c5cf8c |
traversal_end = 0;
|
|
Packit Service |
c5cf8c |
MPIR_Netloc_get_switches_at_level(MPIR_Process.netloc_topology, MPIR_Process.network_attr,
|
|
Packit Service |
c5cf8c |
switch_level, &switches_at_level, &switch_count);
|
|
Packit Service |
c5cf8c |
/* Find the switch `switch_level` steps away */
|
|
Packit Service |
c5cf8c |
MPIR_Assert(traversal_end < MPIR_Process.netloc_topology->num_nodes);
|
|
Packit Service |
c5cf8c |
traversal_stack[traversal_end++] = network_node;
|
|
Packit Service |
c5cf8c |
color = 0;
|
|
Packit Service |
c5cf8c |
while (traversal_end > traversal_begin) {
|
|
Packit Service |
c5cf8c |
netloc_node_t *current_node = traversal_stack[traversal_begin++];
|
|
Packit Service |
c5cf8c |
int num_edges;
|
|
Packit Service |
c5cf8c |
netloc_edge_t **edges;
|
|
Packit Service |
c5cf8c |
if (current_node->node_type == NETLOC_NODE_TYPE_SWITCH
|
|
Packit Service |
c5cf8c |
&& MPIR_Process.network_attr.u.tree.node_levels[current_node->__uid__]
|
|
Packit Service |
c5cf8c |
== switch_level) {
|
|
Packit Service |
c5cf8c |
for (i = 0; i < switch_count; i++) {
|
|
Packit Service |
c5cf8c |
if (switches_at_level[i] == current_node) {
|
|
Packit Service |
c5cf8c |
color = color & (1 < i);
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
/*find all nodes not visited with an edge from the current node */
|
|
Packit Service |
c5cf8c |
netloc_get_all_edges(MPIR_Process.netloc_topology, network_node, &num_edges, &edges);
|
|
Packit Service |
c5cf8c |
for (i = 0; i < num_edges; i++) {
|
|
Packit Service |
c5cf8c |
MPIR_Assert(traversal_end < MPIR_Process.netloc_topology->num_nodes);
|
|
Packit Service |
c5cf8c |
traversal_stack[traversal_end++] = edges[i]->dest_node;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (color == 0) {
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPL_free(traversal_stack);
|
|
Packit Service |
c5cf8c |
MPL_free(switches_at_level);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
color = MPI_UNDEFINED;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int get_color_from_subset_bitmap(int node_index, int *bitmap, int bitmap_size, int min_size)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int color;
|
|
Packit Service |
c5cf8c |
int subset_size;
|
|
Packit Service |
c5cf8c |
int current_comm_color;
|
|
Packit Service |
c5cf8c |
int prev_comm_color;
|
|
Packit Service |
c5cf8c |
int i;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
subset_size = 0;
|
|
Packit Service |
c5cf8c |
current_comm_color = 0;
|
|
Packit Service |
c5cf8c |
prev_comm_color = -1;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
for (i = 0; i < bitmap_size; i++) {
|
|
Packit Service |
c5cf8c |
if (subset_size >= min_size) {
|
|
Packit Service |
c5cf8c |
subset_size = 0;
|
|
Packit Service |
c5cf8c |
prev_comm_color = current_comm_color;
|
|
Packit Service |
c5cf8c |
current_comm_color = i;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
subset_size += bitmap[i];
|
|
Packit Service |
c5cf8c |
if (i == node_index) {
|
|
Packit Service |
c5cf8c |
color = current_comm_color;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (subset_size < min_size && i == bitmap_size)
|
|
Packit Service |
c5cf8c |
color = prev_comm_color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int network_split_by_minsize(MPIR_Comm * comm_ptr, int key, int subcomm_min_size,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int i, color;
|
|
Packit Service |
c5cf8c |
int comm_size = MPIR_Comm_size(comm_ptr);
|
|
Packit Service |
c5cf8c |
netloc_node_t *network_node;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (subcomm_min_size == 0 || comm_size < subcomm_min_size ||
|
|
Packit Service |
c5cf8c |
MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__INVALID) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
int node_index, num_nodes, i;
|
|
Packit Service |
c5cf8c |
int *num_processes_at_node = NULL;
|
|
Packit Service |
c5cf8c |
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
|
|
Packit Service |
c5cf8c |
int subset_size;
|
|
Packit Service |
c5cf8c |
int current_comm_color;
|
|
Packit Service |
c5cf8c |
int prev_comm_color;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
network_node = MPIR_Process.network_attr.network_endpoint;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__FAT_TREE ||
|
|
Packit Service |
c5cf8c |
MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__CLOS_NETWORK) {
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPIR_Netloc_get_hostnode_index_in_tree(MPIR_Process.network_attr,
|
|
Packit Service |
c5cf8c |
MPIR_Process.netloc_topology, network_node,
|
|
Packit Service |
c5cf8c |
&node_index, &num_nodes);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
num_processes_at_node = (int *) MPL_calloc(1, sizeof(int) * num_nodes, MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
num_processes_at_node[node_index] = 1;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
} else if (MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__TORUS) {
|
|
Packit Service |
c5cf8c |
num_processes_at_node =
|
|
Packit Service |
c5cf8c |
(int *) MPL_calloc(1, sizeof(int) * MPIR_Process.netloc_topology->num_nodes,
|
|
Packit Service |
c5cf8c |
MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
num_processes_at_node[MPIR_Process.network_attr.u.torus.node_idx] = 1;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPIR_Assert(num_processes_at_node != NULL);
|
|
Packit Service |
c5cf8c |
/* Send the count to processes */
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(MPI_IN_PLACE, num_processes_at_node, num_nodes, MPI_INT,
|
|
Packit Service |
c5cf8c |
MPI_SUM, comm_ptr, &errflag);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__FAT_TREE ||
|
|
Packit Service |
c5cf8c |
MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__CLOS_NETWORK) {
|
|
Packit Service |
c5cf8c |
color =
|
|
Packit Service |
c5cf8c |
get_color_from_subset_bitmap(node_index, num_processes_at_node, num_nodes,
|
|
Packit Service |
c5cf8c |
subcomm_min_size);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
int *offset_along_dimension =
|
|
Packit Service |
c5cf8c |
(int *) MPL_calloc(MPIR_Process.network_attr.u.torus.dimension, sizeof(int),
|
|
Packit Service |
c5cf8c |
MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
int *partition =
|
|
Packit Service |
c5cf8c |
(int *) MPL_calloc(MPIR_Process.network_attr.u.torus.dimension, sizeof(int),
|
|
Packit Service |
c5cf8c |
MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
int start_index = offset_along_dimension[0];
|
|
Packit Service |
c5cf8c |
int num_processes = 0, total_num_processes = 0;
|
|
Packit Service |
c5cf8c |
int j, size;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
for (i = 0; i < MPIR_Process.network_attr.u.torus.dimension; i++) {
|
|
Packit Service |
c5cf8c |
partition[i] = 1;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
while (1) {
|
|
Packit Service |
c5cf8c |
int node_covered = 0;
|
|
Packit Service |
c5cf8c |
color = total_num_processes;
|
|
Packit Service |
c5cf8c |
for (i = 0; i < MPIR_Process.network_attr.u.torus.dimension;
|
|
Packit Service |
c5cf8c |
i = (i + 1) % MPIR_Process.network_attr.u.torus.dimension) {
|
|
Packit Service |
c5cf8c |
int cube_size;
|
|
Packit Service |
c5cf8c |
if (partition[i] - 1 + offset_along_dimension[i] ==
|
|
Packit Service |
c5cf8c |
MPIR_Process.network_attr.u.torus.geometry[i]) {
|
|
Packit Service |
c5cf8c |
if (i == MPIR_Process.network_attr.u.torus.dimension - 1) {
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
continue;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
partition[i]++;
|
|
Packit Service |
c5cf8c |
cube_size = 0;
|
|
Packit Service |
c5cf8c |
for (j = 0; j < MPIR_Process.network_attr.u.torus.dimension; j++) {
|
|
Packit Service |
c5cf8c |
if (partition[j] != 0) {
|
|
Packit Service |
c5cf8c |
cube_size = cube_size * partition[j];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
num_processes = 0;
|
|
Packit Service |
c5cf8c |
for (j = 0; j < cube_size; j++) {
|
|
Packit Service |
c5cf8c |
int *coordinate =
|
|
Packit Service |
c5cf8c |
(int *) MPL_calloc(MPIR_Process.network_attr.u.torus.dimension,
|
|
Packit Service |
c5cf8c |
sizeof(int), MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
int index = j;
|
|
Packit Service |
c5cf8c |
int k;
|
|
Packit Service |
c5cf8c |
int current_dim = 0;
|
|
Packit Service |
c5cf8c |
while (current_dim < MPIR_Process.network_attr.u.torus.dimension) {
|
|
Packit Service |
c5cf8c |
coordinate[current_dim++] = index % partition[j];
|
|
Packit Service |
c5cf8c |
index = index / partition[j];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
index = 0;
|
|
Packit Service |
c5cf8c |
for (k = 0; k < MPIR_Process.network_attr.u.torus.dimension; k++) {
|
|
Packit Service |
c5cf8c |
index =
|
|
Packit Service |
c5cf8c |
index * (partition[j] + offset_along_dimension[i]) + coordinate[k];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (index == MPIR_Process.network_attr.u.torus.node_idx) {
|
|
Packit Service |
c5cf8c |
node_covered = 1;
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
num_processes += num_processes_at_node[index];
|
|
Packit Service |
c5cf8c |
MPL_free(coordinate);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (num_processes >= subcomm_min_size) {
|
|
Packit Service |
c5cf8c |
total_num_processes += num_processes;
|
|
Packit Service |
c5cf8c |
num_processes = 0;
|
|
Packit Service |
c5cf8c |
for (j = 0; j < MPIR_Process.network_attr.u.torus.dimension; j++) {
|
|
Packit Service |
c5cf8c |
offset_along_dimension[i] += partition[j] + 1;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (total_num_processes == MPIR_Process.netloc_topology->num_nodes || node_covered) {
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPL_free(offset_along_dimension);
|
|
Packit Service |
c5cf8c |
MPL_free(partition);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* There are more processes in the subset than requested within the node.
|
|
Packit Service |
c5cf8c |
* Split further inside each node */
|
|
Packit Service |
c5cf8c |
if (num_processes_at_node[node_index] > subcomm_min_size && node_index == color &&
|
|
Packit Service |
c5cf8c |
((node_index < (node_index - 1) ||
|
|
Packit Service |
c5cf8c |
num_processes_at_node[node_index] < subcomm_min_size))) {
|
|
Packit Service |
c5cf8c |
MPIR_Comm *node_comm;
|
|
Packit Service |
c5cf8c |
int subcomm_rank;
|
|
Packit Service |
c5cf8c |
int min_tree_depth;
|
|
Packit Service |
c5cf8c |
hwloc_cpuset_t *node_comm_bindset;
|
|
Packit Service |
c5cf8c |
int num_procs;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
num_procs = num_processes_at_node[node_index];
|
|
Packit Service |
c5cf8c |
node_comm = *newcomm_ptr;
|
|
Packit Service |
c5cf8c |
subcomm_rank = MPIR_Comm_rank(node_comm);
|
|
Packit Service |
c5cf8c |
node_comm_bindset =
|
|
Packit Service |
c5cf8c |
(hwloc_cpuset_t *) MPL_calloc(num_procs, sizeof(hwloc_cpuset_t), MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
node_comm_bindset[subcomm_rank] = MPIR_Process.bindset;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Send the bindset to processes in node communicator */
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(MPI_IN_PLACE, node_comm_bindset,
|
|
Packit Service |
c5cf8c |
num_procs * sizeof(hwloc_cpuset_t), MPI_BYTE,
|
|
Packit Service |
c5cf8c |
MPI_NO_OP, node_comm, &errflag);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
min_tree_depth = -1;
|
|
Packit Service |
c5cf8c |
for (i = 0; i < num_procs; i++) {
|
|
Packit Service |
c5cf8c |
hwloc_obj_t obj_containing_cpuset =
|
|
Packit Service |
c5cf8c |
hwloc_get_obj_covering_cpuset(MPIR_Process.hwloc_topology,
|
|
Packit Service |
c5cf8c |
node_comm_bindset[i]);
|
|
Packit Service |
c5cf8c |
if (obj_containing_cpuset->depth < min_tree_depth || min_tree_depth == -1) {
|
|
Packit Service |
c5cf8c |
min_tree_depth = obj_containing_cpuset->depth;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (min_tree_depth) {
|
|
Packit Service |
c5cf8c |
int num_hwloc_objs_at_depth =
|
|
Packit Service |
c5cf8c |
hwloc_get_nbobjs_by_depth(MPIR_Process.hwloc_topology, min_tree_depth);
|
|
Packit Service |
c5cf8c |
int *processes_cpuset =
|
|
Packit Service |
c5cf8c |
(int *) MPL_calloc(num_hwloc_objs_at_depth, sizeof(int), MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
hwloc_obj_t parent_obj;
|
|
Packit Service |
c5cf8c |
int hw_obj_index;
|
|
Packit Service |
c5cf8c |
int current_proc_index = -1;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
parent_obj = NULL;
|
|
Packit Service |
c5cf8c |
hw_obj_index = 0;
|
|
Packit Service |
c5cf8c |
while ((parent_obj =
|
|
Packit Service |
c5cf8c |
hwloc_get_next_obj_by_depth(MPIR_Process.hwloc_topology, min_tree_depth,
|
|
Packit Service |
c5cf8c |
parent_obj)) != NULL) {
|
|
Packit Service |
c5cf8c |
for (i = 0; i < num_procs; i++) {
|
|
Packit Service |
c5cf8c |
if (hwloc_bitmap_isincluded(parent_obj->cpuset, node_comm_bindset[i]) ||
|
|
Packit Service |
c5cf8c |
hwloc_bitmap_isequal(parent_obj->cpuset, node_comm_bindset[i])) {
|
|
Packit Service |
c5cf8c |
processes_cpuset[hw_obj_index] = 1;
|
|
Packit Service |
c5cf8c |
if (i == subcomm_rank) {
|
|
Packit Service |
c5cf8c |
current_proc_index = hw_obj_index;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
break;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
hw_obj_index++;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
color =
|
|
Packit Service |
c5cf8c |
get_color_from_subset_bitmap(current_proc_index, processes_cpuset,
|
|
Packit Service |
c5cf8c |
num_hwloc_objs_at_depth, subcomm_min_size);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(node_comm, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
MPL_free(processes_cpuset);
|
|
Packit Service |
c5cf8c |
MPIR_Comm_free_impl(node_comm);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPL_free(node_comm_bindset);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPL_free(num_processes_at_node);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int network_split_by_min_memsize(MPIR_Comm * comm_ptr, int key, long min_mem_size,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int i, color;
|
|
Packit Service |
c5cf8c |
netloc_node_t *network_node;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Get available memory in the node */
|
|
Packit Service |
c5cf8c |
hwloc_obj_t memory_obj = NULL;
|
|
Packit Service |
c5cf8c |
long total_memory_size = 0;
|
|
Packit Service |
c5cf8c |
int memory_per_process;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
while ((memory_obj =
|
|
Packit Service |
c5cf8c |
hwloc_get_next_obj_by_type(MPIR_Process.hwloc_topology, HWLOC_OBJ_NUMANODE,
|
|
Packit Service |
c5cf8c |
memory_obj)) != NULL) {
|
|
Packit Service |
c5cf8c |
/* Memory size is in bytes here */
|
|
Packit Service |
c5cf8c |
total_memory_size += memory_obj->total_memory;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (min_mem_size == 0 || MPIR_Process.network_attr.type == MPIR_NETLOC_NETWORK_TYPE__INVALID) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
int num_ranks_node;
|
|
Packit Service |
c5cf8c |
if (MPIR_Process.comm_world->node_comm != NULL) {
|
|
Packit Service |
c5cf8c |
num_ranks_node = MPIR_Comm_size(MPIR_Process.comm_world->node_comm);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
num_ranks_node = 1;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
memory_per_process = total_memory_size / num_ranks_node;
|
|
Packit Service |
c5cf8c |
mpi_errno = network_split_by_minsize(comm_ptr, key, min_mem_size / memory_per_process,
|
|
Packit Service |
c5cf8c |
newcomm_ptr);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int network_split_by_torus_dimension(MPIR_Comm * comm_ptr, int key,
|
|
Packit Service |
c5cf8c |
int dimension, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int i, color;
|
|
Packit Service |
c5cf8c |
int comm_size = MPIR_Comm_size(comm_ptr);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Dimension is assumed to be indexed from 0 */
|
|
Packit Service |
c5cf8c |
if (MPIR_Process.network_attr.type != MPIR_NETLOC_NETWORK_TYPE__TORUS ||
|
|
Packit Service |
c5cf8c |
dimension >= MPIR_Process.network_attr.u.torus.dimension) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
int node_coordinates = MPIR_Process.network_attr.u.torus.node_idx;
|
|
Packit Service |
c5cf8c |
int *node_dimensions = MPIR_Process.network_attr.u.torus.geometry;
|
|
Packit Service |
c5cf8c |
color = 0;
|
|
Packit Service |
c5cf8c |
for (i = 0; i < MPIR_Process.network_attr.u.torus.dimension; i++) {
|
|
Packit Service |
c5cf8c |
int coordinate_along_dim;
|
|
Packit Service |
c5cf8c |
if (i == dimension) {
|
|
Packit Service |
c5cf8c |
coordinate_along_dim = 0;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
coordinate_along_dim = node_coordinates % node_dimensions[i];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (i == 0) {
|
|
Packit Service |
c5cf8c |
color = coordinate_along_dim;
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
color = color + coordinate_along_dim * node_dimensions[i - 1];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
node_coordinates = node_coordinates / node_dimensions[i];
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(comm_ptr, color, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static const char *SHMEM_INFO_KEY = "shmem_topo";
|
|
Packit Service |
c5cf8c |
static const char *NETWORK_INFO_KEY = "network_topo";
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
static int compare_info_hint(const char *hintval, MPIR_Comm * comm_ptr, int *info_args_are_equal)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int hintval_size = strlen(hintval);
|
|
Packit Service |
c5cf8c |
int hintval_size_max;
|
|
Packit Service |
c5cf8c |
int hintval_equal;
|
|
Packit Service |
c5cf8c |
int hintval_equal_global = 0;
|
|
Packit Service |
c5cf8c |
char *hintval_global = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
MPIR_Errflag_t errflag = MPIR_ERR_NONE;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Find the maximum hintval size. Each process locally compares
|
|
Packit Service |
c5cf8c |
* its hintval size to the global max, and makes sure that this
|
|
Packit Service |
c5cf8c |
* comparison is successful on all processes. */
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(&hintval_size, &hintval_size_max, 1, MPI_INT, MPI_MAX, comm_ptr, &errflag);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
hintval_equal = (hintval_size == hintval_size_max);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(&hintval_equal, &hintval_equal_global, 1, MPI_INT, MPI_LAND,
|
|
Packit Service |
c5cf8c |
comm_ptr, &errflag);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (!hintval_equal_global)
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Now that the sizes of the hintvals match, check to make sure
|
|
Packit Service |
c5cf8c |
* the actual hintvals themselves are the equal */
|
|
Packit Service |
c5cf8c |
hintval_global = (char *) MPL_malloc(strlen(hintval), MPL_MEM_OTHER);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(hintval, hintval_global, strlen(hintval), MPI_CHAR,
|
|
Packit Service |
c5cf8c |
MPI_MAX, comm_ptr, &errflag);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
hintval_equal = !memcmp(hintval, hintval_global, strlen(hintval));
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPID_Allreduce(&hintval_equal, &hintval_equal_global, 1, MPI_INT, MPI_LAND,
|
|
Packit Service |
c5cf8c |
comm_ptr, &errflag);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
if (hintval_global != NULL)
|
|
Packit Service |
c5cf8c |
MPL_free(hintval_global);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
*info_args_are_equal = hintval_equal_global;
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_node_topo
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_node_topo(MPIR_Comm * user_comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Info * info_ptr, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_ptr;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int flag = 0;
|
|
Packit Service |
c5cf8c |
char hintval[MPI_MAX_INFO_VAL + 1];
|
|
Packit Service |
c5cf8c |
int info_args_are_equal;
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_type_node(user_comm_ptr, split_type, key, &comm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (comm_ptr == NULL) {
|
|
Packit Service |
c5cf8c |
MPIR_Assert(split_type == MPI_UNDEFINED);
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (info_ptr) {
|
|
Packit Service |
c5cf8c |
MPIR_Info_get_impl(info_ptr, SHMEM_INFO_KEY, MPI_MAX_INFO_VAL, hintval, &flag;;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (!flag) {
|
|
Packit Service |
c5cf8c |
hintval[0] = '\0';
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = compare_info_hint(hintval, comm_ptr, &info_args_are_equal);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* if all processes do not have the same hintval, skip
|
|
Packit Service |
c5cf8c |
* topology-aware comm split */
|
|
Packit Service |
c5cf8c |
if (!info_args_are_equal)
|
|
Packit Service |
c5cf8c |
goto use_node_comm;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* if no info key is given, skip topology-aware comm split */
|
|
Packit Service |
c5cf8c |
if (!info_ptr)
|
|
Packit Service |
c5cf8c |
goto use_node_comm;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_HWLOC
|
|
Packit Service |
c5cf8c |
/* if our bindset is not valid, skip topology-aware comm split */
|
|
Packit Service |
c5cf8c |
if (!MPIR_Process.bindset_is_valid)
|
|
Packit Service |
c5cf8c |
goto use_node_comm;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (flag) {
|
|
Packit Service |
c5cf8c |
if (!strncmp(hintval, "pci:", strlen("pci:")))
|
|
Packit Service |
c5cf8c |
mpi_errno = node_split_pci_device(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
else if (!strncmp(hintval, "ib", strlen("ib")) ||
|
|
Packit Service |
c5cf8c |
!strncmp(hintval, "en", strlen("en")) ||
|
|
Packit Service |
c5cf8c |
!strncmp(hintval, "eth", strlen("eth")) || !strncmp(hintval, "hfi", strlen("hfi")))
|
|
Packit Service |
c5cf8c |
mpi_errno = node_split_network_device(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
else if (!strncmp(hintval, "gpu", strlen("gpu")))
|
|
Packit Service |
c5cf8c |
mpi_errno = node_split_gpu_device(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
mpi_errno = node_split_processor(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPIR_Comm_free_impl(comm_ptr);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
#endif /* HAVE_HWLOC */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
use_node_comm:
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = comm_ptr;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_network_topo
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_network_topo(MPIR_Comm * comm_ptr, int key, const char *hintval,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_NETLOC
|
|
Packit Service |
c5cf8c |
if (!strncmp(hintval, ("switch_level:"), strlen("switch_level:"))
|
|
Packit Service |
c5cf8c |
&& *(hintval + strlen("switch_level:")) != '\0') {
|
|
Packit Service |
c5cf8c |
int switch_level = atoi(hintval + strlen("switch_level:"));
|
|
Packit Service |
c5cf8c |
mpi_errno = network_split_switch_level(comm_ptr, key, switch_level, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
} else if (!strncmp(hintval, ("subcomm_min_size:"), strlen("subcomm_min_size:"))
|
|
Packit Service |
c5cf8c |
&& *(hintval + strlen("subcomm_min_size:")) != '\0') {
|
|
Packit Service |
c5cf8c |
int subcomm_min_size = atoi(hintval + strlen("subcomm_min_size:"));
|
|
Packit Service |
c5cf8c |
mpi_errno = network_split_by_minsize(comm_ptr, key, subcomm_min_size, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
} else if (!strncmp(hintval, ("min_mem_size:"), strlen("min_mem_size:"))
|
|
Packit Service |
c5cf8c |
&& *(hintval + strlen("min_mem_size:")) != '\0') {
|
|
Packit Service |
c5cf8c |
long min_mem_size = atol(hintval + strlen("min_mem_size:"));
|
|
Packit Service |
c5cf8c |
/* Split by minimum memory size per subcommunicator in bytes */
|
|
Packit Service |
c5cf8c |
mpi_errno = network_split_by_min_memsize(comm_ptr, key, min_mem_size, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
} else if (!strncmp(hintval, ("torus_dimension:"), strlen("torus_dimension:"))
|
|
Packit Service |
c5cf8c |
&& *(hintval + strlen("torus_dimension:")) != '\0') {
|
|
Packit Service |
c5cf8c |
int dimension = atol(hintval + strlen("torus_dimension:"));
|
|
Packit Service |
c5cf8c |
mpi_errno = network_split_by_torus_dimension(comm_ptr, key, dimension, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type(MPIR_Comm * user_comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Info * info_ptr, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* split out the undefined processes */
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_impl(user_comm_ptr, split_type == MPI_UNDEFINED ? MPI_UNDEFINED : 0,
|
|
Packit Service |
c5cf8c |
key, &comm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (split_type == MPI_UNDEFINED) {
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (split_type == MPI_COMM_TYPE_SHARED) {
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_type_self(comm_ptr, split_type, key, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
} else if (split_type == MPIX_COMM_TYPE_NEIGHBORHOOD) {
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPIR_Comm_split_type_neighborhood(comm_ptr, split_type, key, info_ptr, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
MPIR_ERR_SETANDJUMP(mpi_errno, MPI_ERR_ARG, "**arg");
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
if (comm_ptr)
|
|
Packit Service |
c5cf8c |
MPIR_Comm_free_impl(comm_ptr);
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_nbhd_common_dir
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_nbhd_common_dir(MPIR_Comm * user_comm_ptr, int key, const char *hintval,
|
|
Packit Service |
c5cf8c |
MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_ROMIO
|
|
Packit Service |
c5cf8c |
MPI_Comm dummycomm;
|
|
Packit Service |
c5cf8c |
MPIR_Comm *dummycomm_ptr;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_filesystem(user_comm_ptr->handle, key, hintval, &dummycomm);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPIR_Comm_get_ptr(dummycomm, dummycomm_ptr);
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = dummycomm_ptr;
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_neighborhood
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_neighborhood(MPIR_Comm * comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Info * info_ptr, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
int flag = 0;
|
|
Packit Service |
c5cf8c |
char hintval[MPI_MAX_INFO_VAL + 1];
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
int info_args_are_equal;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (info_ptr) {
|
|
Packit Service |
c5cf8c |
MPIR_Info_get_impl(info_ptr, "nbhd_common_dirname", MPI_MAX_INFO_VAL, hintval, &flag;;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (!flag) {
|
|
Packit Service |
c5cf8c |
hintval[0] = '\0';
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
*newcomm_ptr = NULL;
|
|
Packit Service |
c5cf8c |
mpi_errno = compare_info_hint(hintval, comm_ptr, &info_args_are_equal);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (info_args_are_equal && flag) {
|
|
Packit Service |
c5cf8c |
MPIR_Comm_split_type_nbhd_common_dir(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
/* Check if the info hint is a network topology hint */
|
|
Packit Service |
c5cf8c |
if (info_ptr) {
|
|
Packit Service |
c5cf8c |
MPIR_Info_get_impl(info_ptr, NETWORK_INFO_KEY, MPI_MAX_INFO_VAL, hintval, &flag;;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (!flag) {
|
|
Packit Service |
c5cf8c |
hintval[0] = '\0';
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = compare_info_hint(hintval, comm_ptr, &info_args_are_equal);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* if all processes have the same hintval, perform
|
|
Packit Service |
c5cf8c |
* topology-aware comm split */
|
|
Packit Service |
c5cf8c |
if (info_args_are_equal) {
|
|
Packit Service |
c5cf8c |
MPIR_Comm_split_type_network_topo(comm_ptr, key, hintval, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPIR_Comm_split_type_impl
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
int MPIR_Comm_split_type_impl(MPIR_Comm * comm_ptr, int split_type, int key,
|
|
Packit Service |
c5cf8c |
MPIR_Info * info_ptr, MPIR_Comm ** newcomm_ptr)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Only MPI_COMM_TYPE_SHARED, MPI_UNDEFINED, and
|
|
Packit Service |
c5cf8c |
* NEIGHBORHOOD are supported */
|
|
Packit Service |
c5cf8c |
MPIR_Assert(split_type == MPI_COMM_TYPE_SHARED ||
|
|
Packit Service |
c5cf8c |
split_type == MPI_UNDEFINED || split_type == MPIX_COMM_TYPE_NEIGHBORHOOD);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
if (MPIR_Comm_fns != NULL && MPIR_Comm_fns->split_type != NULL) {
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_fns->split_type(comm_ptr, split_type, key, info_ptr, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
} else {
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_type(comm_ptr, split_type, key, info_ptr, newcomm_ptr);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#endif /* MPICH_MPI_FROM_PMPI */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#undef FUNCNAME
|
|
Packit Service |
c5cf8c |
#define FUNCNAME MPI_Comm_split_type
|
|
Packit Service |
c5cf8c |
#undef FCNAME
|
|
Packit Service |
c5cf8c |
#define FCNAME MPL_QUOTE(FUNCNAME)
|
|
Packit Service |
c5cf8c |
/*@
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPI_Comm_split_type - Creates new communicators based on split types and keys
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Input Parameters:
|
|
Packit Service |
c5cf8c |
+ comm - communicator (handle)
|
|
Packit Service |
c5cf8c |
. split_type - type of processes to be grouped together (nonnegative integer).
|
|
Packit Service |
c5cf8c |
. key - control of rank assignment (integer)
|
|
Packit Service |
c5cf8c |
- info - hints to improve communicator creation (handle)
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Output Parameters:
|
|
Packit Service |
c5cf8c |
. newcomm - new communicator (handle)
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
Notes:
|
|
Packit Service |
c5cf8c |
The 'split_type' must be non-negative or 'MPI_UNDEFINED'.
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
.N ThreadSafe
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
.N Fortran
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
.N Errors
|
|
Packit Service |
c5cf8c |
.N MPI_SUCCESS
|
|
Packit Service |
c5cf8c |
.N MPI_ERR_COMM
|
|
Packit Service |
c5cf8c |
.N MPI_ERR_EXHAUSTED
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
.seealso: MPI_Comm_free
|
|
Packit Service |
c5cf8c |
@*/
|
|
Packit Service |
c5cf8c |
int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key, MPI_Info info, MPI_Comm * newcomm)
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
int mpi_errno = MPI_SUCCESS;
|
|
Packit Service |
c5cf8c |
MPIR_Comm *comm_ptr = NULL, *newcomm_ptr;
|
|
Packit Service |
c5cf8c |
MPIR_Info *info_ptr = NULL;
|
|
Packit Service |
c5cf8c |
MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_COMM_SPLIT_TYPE);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPIR_ERRTEST_INITIALIZED_ORDIE();
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
|
|
Packit Service |
c5cf8c |
MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPI_COMM_SPLIT_TYPE);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Validate parameters, especially handles needing to be converted */
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_ERROR_CHECKING
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPID_BEGIN_ERROR_CHECKS;
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPIR_ERRTEST_COMM(comm, mpi_errno);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPID_END_ERROR_CHECKS;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
#endif /* HAVE_ERROR_CHECKING */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Get handles to MPI objects. */
|
|
Packit Service |
c5cf8c |
MPIR_Comm_get_ptr(comm, comm_ptr);
|
|
Packit Service |
c5cf8c |
MPIR_Info_get_ptr(info, info_ptr);
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* Validate parameters and objects (post conversion) */
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_ERROR_CHECKING
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
MPID_BEGIN_ERROR_CHECKS;
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
/* Validate comm_ptr */
|
|
Packit Service |
c5cf8c |
MPIR_Comm_valid_ptr(comm_ptr, mpi_errno, FALSE);
|
|
Packit Service |
c5cf8c |
/* If comm_ptr is not valid, it will be reset to null */
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
goto fn_fail;
|
|
Packit Service |
c5cf8c |
MPIR_ERRTEST_ARGNULL(newcomm, "newcomm", mpi_errno);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
MPID_END_ERROR_CHECKS;
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
#endif /* HAVE_ERROR_CHECKING */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* ... body of routine ... */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Comm_split_type_impl(comm_ptr, split_type, key, info_ptr, &newcomm_ptr);
|
|
Packit Service |
c5cf8c |
if (mpi_errno)
|
|
Packit Service |
c5cf8c |
MPIR_ERR_POP(mpi_errno);
|
|
Packit Service |
c5cf8c |
if (newcomm_ptr)
|
|
Packit Service |
c5cf8c |
MPIR_OBJ_PUBLISH_HANDLE(*newcomm, newcomm_ptr->handle);
|
|
Packit Service |
c5cf8c |
else
|
|
Packit Service |
c5cf8c |
*newcomm = MPI_COMM_NULL;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
/* ... end of body of routine ... */
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_exit:
|
|
Packit Service |
c5cf8c |
MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPI_COMM_SPLIT_TYPE);
|
|
Packit Service |
c5cf8c |
MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);
|
|
Packit Service |
c5cf8c |
return mpi_errno;
|
|
Packit Service |
c5cf8c |
|
|
Packit Service |
c5cf8c |
fn_fail:
|
|
Packit Service |
c5cf8c |
/* --BEGIN ERROR HANDLING-- */
|
|
Packit Service |
c5cf8c |
#ifdef HAVE_ERROR_CHECKING
|
|
Packit Service |
c5cf8c |
{
|
|
Packit Service |
c5cf8c |
/* FIXME this error code is wrong, it's the error code for
|
|
Packit Service |
c5cf8c |
* regular MPI_Comm_split */
|
|
Packit Service |
c5cf8c |
mpi_errno =
|
|
Packit Service |
c5cf8c |
MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__,
|
|
Packit Service |
c5cf8c |
MPI_ERR_OTHER, "**mpi_comm_split",
|
|
Packit Service |
c5cf8c |
"**mpi_comm_split %C %d %d %p", comm, split_type, key, newcomm);
|
|
Packit Service |
c5cf8c |
}
|
|
Packit Service |
c5cf8c |
#endif
|
|
Packit Service |
c5cf8c |
mpi_errno = MPIR_Err_return_comm(comm_ptr, FCNAME, mpi_errno);
|
|
Packit Service |
c5cf8c |
goto fn_exit;
|
|
Packit Service |
c5cf8c |
/* --END ERROR HANDLING-- */
|
|
Packit Service |
c5cf8c |
}
|