/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */

/*

 *

 *  (C) 2001 by Argonne National Laboratory.

 *      See COPYRIGHT in top-level directory.

 */

#include "mpiimpl.h"

#include "mpicomm.h"

/* -- Begin Profiling Symbol Block for routine MPI_Intercomm_create */

#if defined(HAVE_PRAGMA_WEAK)

#pragma weak MPI_Intercomm_create = PMPI_Intercomm_create

#elif defined(HAVE_PRAGMA_HP_SEC_DEF)

#pragma _HP_SECONDARY_DEF PMPI_Intercomm_create  MPI_Intercomm_create

#elif defined(HAVE_PRAGMA_CRI_DUP)

#pragma _CRI duplicate MPI_Intercomm_create as PMPI_Intercomm_create

#elif defined(HAVE_WEAK_ATTRIBUTE)

int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader, MPI_Comm peer_comm,

                         int remote_leader, int tag, MPI_Comm * newintercomm)

    __attribute__ ((weak, alias("PMPI_Intercomm_create")));

#endif

/* -- End Profiling Symbol Block */

/* Define MPICH_MPI_FROM_PMPI if weak symbols are not supported to build

   the MPI routines */

#ifndef MPICH_MPI_FROM_PMPI

#undef MPI_Intercomm_create

#define MPI_Intercomm_create PMPI_Intercomm_create

#undef FUNCNAME

#define FUNCNAME MPIR_Intercomm_create_impl

#undef FCNAME

#define FCNAME MPL_QUOTE(FUNCNAME)

int MPIR_Intercomm_create_impl(MPIR_Comm * local_comm_ptr, int local_leader,

                               MPIR_Comm * peer_comm_ptr, int remote_leader, int tag,

                               MPIR_Comm ** new_intercomm_ptr)

{

    int mpi_errno = MPI_SUCCESS;

    MPIR_Context_id_t final_context_id, recvcontext_id;

    int remote_size = 0, *remote_lpids = NULL;

    int comm_info[3];

    int is_low_group = 0;

    MPIR_Errflag_t errflag = MPIR_ERR_NONE;

    MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPIR_COMM_KIND__INTERCOMM_CREATE_IMPL);

    MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPIR_COMM_KIND__INTERCOMM_CREATE_IMPL);

    /* Shift tag into the tagged coll space */

    tag |= MPIR_TAG_COLL_BIT;

    mpi_errno = MPID_Intercomm_exchange_map(local_comm_ptr, local_leader,

                                            peer_comm_ptr, remote_leader,

                                            &remote_size, &remote_lpids, &is_low_group);

    if (mpi_errno)

        MPIR_ERR_POP(mpi_errno);

    /*

     * Create the contexts.  Each group will have a context for sending

     * to the other group. All processes must be involved.  Because

     * we know that the local and remote groups are disjoint, this

     * step will complete

     */

    MPL_DBG_MSG_FMT(MPIR_DBG_COMM, VERBOSE,

                    (MPL_DBG_FDEST, "About to get contextid (local_size=%d) on rank %d",

                     local_comm_ptr->local_size, local_comm_ptr->rank));

    /* In the multi-threaded case, MPIR_Get_contextid_sparse assumes that the

     * calling routine already holds the single criticial section */

    /* TODO: Make sure this is tag-safe */

    mpi_errno = MPIR_Get_contextid_sparse(local_comm_ptr, &recvcontext_id, FALSE);

    if (mpi_errno)

        MPIR_ERR_POP(mpi_errno);

    MPIR_Assert(recvcontext_id != 0);

    MPL_DBG_MSG_FMT(MPIR_DBG_COMM, VERBOSE, (MPL_DBG_FDEST, "Got contextid=%d", recvcontext_id));

    /* Leaders can now swap context ids and then broadcast the value

     * to the local group of processes */

    if (local_comm_ptr->rank == local_leader) {

        MPIR_Context_id_t remote_context_id;

        mpi_errno =

            MPIC_Sendrecv(&recvcontext_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, remote_leader, tag,

                          &remote_context_id, 1, MPIR_CONTEXT_ID_T_DATATYPE, remote_leader, tag,

                          peer_comm_ptr, MPI_STATUS_IGNORE, &errflag);

        if (mpi_errno)

            MPIR_ERR_POP(mpi_errno);

        final_context_id = remote_context_id;

        /* Now, send all of our local processes the remote_lpids,

         * along with the final context id */

        comm_info[0] = final_context_id;

        MPL_DBG_MSG(MPIR_DBG_COMM, VERBOSE, "About to bcast on local_comm");

        mpi_errno = MPIR_Bcast(comm_info, 1, MPI_INT, local_leader, local_comm_ptr, &errflag);

        if (mpi_errno)

            MPIR_ERR_POP(mpi_errno);

        MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");

        MPL_DBG_MSG_D(MPIR_DBG_COMM, VERBOSE, "end of bcast on local_comm of size %d",

                      local_comm_ptr->local_size);

    } else {

        /* we're the other processes */

        MPL_DBG_MSG(MPIR_DBG_COMM, VERBOSE, "About to receive bcast on local_comm");

        mpi_errno = MPIR_Bcast(comm_info, 1, MPI_INT, local_leader, local_comm_ptr, &errflag);

        if (mpi_errno)

            MPIR_ERR_POP(mpi_errno);

        MPIR_ERR_CHKANDJUMP(errflag, mpi_errno, MPI_ERR_OTHER, "**coll_fail");

        /* Extract the context and group sign informatin */

        final_context_id = comm_info[0];

    }

    /* At last, we now have the information that we need to build the

     * intercommunicator */

    /* All processes in the local_comm now build the communicator */

    mpi_errno = MPIR_Comm_create(new_intercomm_ptr);

    if (mpi_errno)

        goto fn_fail;

    (*new_intercomm_ptr)->context_id = final_context_id;

    (*new_intercomm_ptr)->recvcontext_id = recvcontext_id;

    (*new_intercomm_ptr)->remote_size = remote_size;

    (*new_intercomm_ptr)->local_size = local_comm_ptr->local_size;

    (*new_intercomm_ptr)->pof2 = local_comm_ptr->pof2;

    (*new_intercomm_ptr)->rank = local_comm_ptr->rank;

    (*new_intercomm_ptr)->comm_kind = MPIR_COMM_KIND__INTERCOMM;

    (*new_intercomm_ptr)->local_comm = 0;

    (*new_intercomm_ptr)->is_low_group = is_low_group;

    mpi_errno = MPID_Create_intercomm_from_lpids(*new_intercomm_ptr, remote_size, remote_lpids);

    if (mpi_errno)

        goto fn_fail;

    MPIR_Comm_map_dup(*new_intercomm_ptr, local_comm_ptr, MPIR_COMM_MAP_DIR__L2L);

    /* Inherit the error handler (if any) */

    MPID_THREAD_CS_ENTER(POBJ, MPIR_THREAD_POBJ_COMM_MUTEX(local_comm_ptr));

    (*new_intercomm_ptr)->errhandler = local_comm_ptr->errhandler;

    if (local_comm_ptr->errhandler) {

        MPIR_Errhandler_add_ref(local_comm_ptr->errhandler);

    }

    MPID_THREAD_CS_EXIT(POBJ, MPIR_THREAD_POBJ_COMM_MUTEX(local_comm_ptr));

    mpi_errno = MPIR_Comm_commit(*new_intercomm_ptr);

    if (mpi_errno)

        MPIR_ERR_POP(mpi_errno);

  fn_exit:

    if (remote_lpids) {

        MPL_free(remote_lpids);

        remote_lpids = NULL;

    }

    MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPIR_COMM_KIND__INTERCOMM_CREATE_IMPL);

    return mpi_errno;

  fn_fail:

    goto fn_exit;

}

#endif /* MPICH_MPI_FROM_PMPI */

#undef FUNCNAME

#define FUNCNAME MPI_Intercomm_create

#undef FCNAME

#define FCNAME MPL_QUOTE(FUNCNAME)

/*@

MPI_Intercomm_create - Creates an intercommuncator from two intracommunicators

Input Parameters:

+ local_comm - Local (intra)communicator

. local_leader - Rank in local_comm of leader (often 0)

. peer_comm - Communicator used to communicate between a

              designated process in the other communicator.

              Significant only at the process in 'local_comm' with

              rank 'local_leader'.

. remote_leader - Rank in peer_comm of remote leader (often 0)

- tag - Message tag to use in constructing intercommunicator; if multiple

  'MPI_Intercomm_creates' are being made, they should use different tags (more

  precisely, ensure that the local and remote leaders are using different

  tags for each 'MPI_intercomm_create').

Output Parameters:

. newintercomm - Created intercommunicator

Notes:

   'peer_comm' is significant only for the process designated the

   'local_leader' in the 'local_comm'.

  The MPI 1.1 Standard contains two mutually exclusive comments on the

  input intercommunicators.  One says that their repective groups must be

  disjoint; the other that the leaders can be the same process.  After

  some discussion by the MPI Forum, it has been decided that the groups must

  be disjoint.  Note that the `reason` given for this in the standard is

  `not` the reason for this choice; rather, the `other` operations on

  intercommunicators (like 'MPI_Intercomm_merge') do not make sense if the

  groups are not disjoint.

.N ThreadSafe

.N Fortran

.N Errors

.N MPI_SUCCESS

.N MPI_ERR_COMM

.N MPI_ERR_TAG

.N MPI_ERR_EXHAUSTED

.N MPI_ERR_RANK

.seealso: MPI_Intercomm_merge, MPI_Comm_free, MPI_Comm_remote_group,

          MPI_Comm_remote_size

@*/

int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,

                         MPI_Comm peer_comm, int remote_leader, int tag, MPI_Comm * newintercomm)

{

    int mpi_errno = MPI_SUCCESS;

    MPIR_Comm *local_comm_ptr = NULL;

    MPIR_Comm *peer_comm_ptr = NULL;

    MPIR_Comm *new_intercomm_ptr;

    MPIR_FUNC_TERSE_STATE_DECL(MPID_STATE_MPI_INTERCOMM_CREATE);

    MPIR_ERRTEST_INITIALIZED_ORDIE();

    MPID_THREAD_CS_ENTER(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);

    MPIR_FUNC_TERSE_ENTER(MPID_STATE_MPI_INTERCOMM_CREATE);

    /* Validate parameters, especially handles needing to be converted */

#ifdef HAVE_ERROR_CHECKING

    {

        MPID_BEGIN_ERROR_CHECKS;

        {

            MPIR_ERRTEST_COMM_TAG(tag, mpi_errno);

            MPIR_ERRTEST_COMM(local_comm, mpi_errno);

        }

        MPID_END_ERROR_CHECKS;

    }

#endif /* HAVE_ERROR_CHECKING */

    /* Convert MPI object handles to object pointers */

    MPIR_Comm_get_ptr(local_comm, local_comm_ptr);

    /* Validate parameters and objects (post conversion) */

#ifdef HAVE_ERROR_CHECKING

    {

        MPID_BEGIN_ERROR_CHECKS;

        {

            /* Validate local_comm_ptr */

            MPIR_Comm_valid_ptr(local_comm_ptr, mpi_errno, FALSE);

            if (local_comm_ptr) {

                /*  Only check if local_comm_ptr valid */

                MPIR_ERRTEST_COMM_INTRA(local_comm_ptr, mpi_errno);

                if ((local_leader < 0 || local_leader >= local_comm_ptr->local_size)) {

                    MPIR_ERR_SET2(mpi_errno, MPI_ERR_RANK,

                                  "**ranklocal", "**ranklocal %d %d",

                                  local_leader, local_comm_ptr->local_size - 1);

                    /* If local_comm_ptr is not valid, it will be reset to null */

                    if (mpi_errno)

                        goto fn_fail;

                }

                if (local_comm_ptr->rank == local_leader) {

                    MPIR_ERRTEST_COMM(peer_comm, mpi_errno);

                }

            }

            MPIR_ERRTEST_ARGNULL(newintercomm, "newintercomm", mpi_errno);

        }

        MPID_END_ERROR_CHECKS;

    }

#endif /* HAVE_ERROR_CHECKING */

    if (local_comm_ptr->rank == local_leader) {

        MPIR_Comm_get_ptr(peer_comm, peer_comm_ptr);

#ifdef HAVE_ERROR_CHECKING

        {

            MPID_BEGIN_ERROR_CHECKS;

            {

                MPIR_Comm_valid_ptr(peer_comm_ptr, mpi_errno, FALSE);

                /* Note: In MPI 1.0, peer_comm was restricted to

                 * intracommunicators.  In 1.1, it may be any communicator */

                /* In checking the rank of the remote leader,

                 * allow the peer_comm to be in intercommunicator

                 * by checking against the remote size */

                if (!mpi_errno && peer_comm_ptr &&

                    (remote_leader < 0 || remote_leader >= peer_comm_ptr->remote_size)) {

                    MPIR_ERR_SET2(mpi_errno, MPI_ERR_RANK,

                                  "**rankremote", "**rankremote %d %d",

                                  remote_leader, peer_comm_ptr->remote_size - 1);

                }

                /* Check that the local leader and the remote leader are

                 * different processes.  This test requires looking at

                 * the lpid for the two ranks in their respective

                 * communicators.  However, an easy test is for

                 * the same ranks in an intracommunicator; we only

                 * need the lpid comparison for intercommunicators */

                /* Here is the test.  We restrict this test to the

                 * process that is the local leader (local_comm_ptr->rank ==

                 * local_leader because we can then use peer_comm_ptr->rank

                 * to get the rank in peer_comm of the local leader. */

                if (peer_comm_ptr->comm_kind == MPIR_COMM_KIND__INTRACOMM &&

                    local_comm_ptr->rank == local_leader && peer_comm_ptr->rank == remote_leader) {

                    MPIR_ERR_SET(mpi_errno, MPI_ERR_RANK, "**ranksdistinct");

                }

                if (mpi_errno)

                    goto fn_fail;

                MPIR_ERRTEST_ARGNULL(newintercomm, "newintercomm", mpi_errno);

            }

            MPID_END_ERROR_CHECKS;

        }

#endif /* HAVE_ERROR_CHECKING */

    }

    /* ... body of routine ... */

    mpi_errno = MPIR_Intercomm_create_impl(local_comm_ptr, local_leader, peer_comm_ptr,

                                           remote_leader, tag, &new_intercomm_ptr);

    if (mpi_errno)

        goto fn_fail;

    MPIR_OBJ_PUBLISH_HANDLE(*newintercomm, new_intercomm_ptr->handle);

    /* ... end of body of routine ... */

  fn_exit:

    MPIR_FUNC_TERSE_EXIT(MPID_STATE_MPI_INTERCOMM_CREATE);

    MPID_THREAD_CS_EXIT(GLOBAL, MPIR_THREAD_GLOBAL_ALLFUNC_MUTEX);

    return mpi_errno;

  fn_fail:

    /* --BEGIN ERROR HANDLING-- */

#ifdef HAVE_ERROR_CHECKING

    {

        mpi_errno =

            MPIR_Err_create_code(mpi_errno, MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_OTHER,

                                 "**mpi_intercomm_create",

                                 "**mpi_intercomm_create %C %d %C %d %d %p", local_comm,

                                 local_leader, peer_comm, remote_leader, tag, newintercomm);

    }

#endif /* HAVE_ERROR_CHECKING */

    mpi_errno = MPIR_Err_return_comm(local_comm_ptr, FCNAME, mpi_errno);

    goto fn_exit;

    /* --END ERROR HANDLING-- */

}