/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
/*
 * This (is a placeholder for a) test that creates 4 threads, each of which
 * does a concurrent spawn of 4 more processes, for a total of 17 MPI processes
 * The resulting intercomms are tested for consistency (to ensure that the
 * spawns didn't get confused among the threads).
 *
 * As an option, it will time the Spawn calls.  If the spawn calls block the
 * calling thread, this may show up in the timing of the calls
 */

#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include "mpitest.h"
#include "mpithreadtest.h"

/*
static char MTEST_Descrip[] = "Spawn jobs from multiple threads";
*/

#define NTHREADS 4
#define NPERTHREAD 4
int activeThreads = 0;

MPI_Comm intercomms[NTHREADS];

MTEST_THREAD_RETURN_TYPE spawnProcess(void *p);

MTEST_THREAD_RETURN_TYPE spawnProcess(void *p)
{
    int rank, i;
    int errcodes[NPERTHREAD];

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);

    /* The thread number is passed into this routine through the value of the
     * argument */
    i = (int) (long) p;

    /* Synchronize */
    MTest_thread_barrier(NTHREADS);

    /* Spawn */
    MPI_Comm_spawn((char *) "./multispawn", MPI_ARGV_NULL, NPERTHREAD,
                   MPI_INFO_NULL, 0, MPI_COMM_SELF, &intercomms[i], errcodes);

    MPI_Bcast(&i, 1, MPI_INT, MPI_ROOT, intercomms[i]);

    return MTEST_THREAD_RETVAL_IGN;
}

int main(int argc, char *argv[])
{
    int rank, size, i, wasParent = 0;
    int provided;
    int err;
    MPI_Comm parentcomm;

    MPI_Init_thread(&argc, &argv, MPI_THREAD_MULTIPLE, &provided);

    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    if (provided != MPI_THREAD_MULTIPLE) {
        if (rank == 0) {
            printf
                ("MPI_Init_thread must return MPI_THREAD_MULTIPLE in order for this test to run.\n");
            fflush(stdout);
        }
        MPI_Finalize();
        return -1;
    }

    MPI_Comm_get_parent(&parentcomm);
    if (parentcomm == MPI_COMM_NULL) {
        wasParent = 1;

        err = MTest_thread_barrier_init();
        if (err) {
            printf("barrier_init failed\n");
            fflush(stdout);
            MPI_Finalize();
            return 1;
        }

        for (i = 0; i < NTHREADS - 1; i++) {
            MTest_Start_thread(spawnProcess, (void *) (long) i);
        }

        /* spawn the processes */
        spawnProcess((void *) (NTHREADS - 1));

        /* Exit the threads (but the spawned processes remain) */
        MTest_Join_threads();

        err = MTest_thread_barrier_free();
        if (err) {
            printf("barrier_free failed\n");
            fflush(stdout);
            MPI_Finalize();
            return 1;
        }

        /* The master thread (this thread) checks the created communicators */
        for (i = 0; i < NTHREADS; i++) {
            MPI_Bcast(&i, 1, MPI_INT, MPI_ROOT, intercomms[i]);
        }

        /* Free the created processes */
        for (i = 0; i < NTHREADS; i++) {
            MPI_Comm_disconnect(&intercomms[i]);
        }
    }
    else {
        int num, threadnum;

        /* I'm the created process */
        MPI_Bcast(&threadnum, 1, MPI_INT, 0, parentcomm);

        /* Form an intra comm with my parent */

        /* receive from my parent */
        MPI_Bcast(&num, 1, MPI_INT, 0, parentcomm);

        if (num != threadnum) {
            fprintf(stderr, "Unexpected thread num (%d != %d)\n", threadnum, num);
        }

        /* Let the parent free the intercomms */
        MPI_Comm_disconnect(&parentcomm);

    }

    if (wasParent)
        MTest_Finalize(0);

    MPI_Finalize();
    return 0;
}