/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *
 *  (C) 2009 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include "mpitest.h"

/*
static char MTEST_Descrip[] = "Test error reporting from faults with point to point communication";
*/

int ReportErr(int errcode, const char name[]);

int main(int argc, char *argv[])
{
    int wrank, wsize, rank, size, color;
    int j, tmp;
    int err, toterrs, errs = 0;
    MPI_Comm newcomm;

    MPI_Init(&argc, &argv);

    MPI_Comm_size(MPI_COMM_WORLD, &wsize);
    MPI_Comm_rank(MPI_COMM_WORLD, &wrank);

    /* Color is 0 or 1; 1 will be the processes that "fault" */
    /* process 0 and wsize/2+1...wsize-1 are in non-faulting group */
    color = (wrank > 0) && (wrank <= wsize / 2);
    MPI_Comm_split(MPI_COMM_WORLD, color, wrank, &newcomm);

    MPI_Comm_size(newcomm, &size);
    MPI_Comm_rank(newcomm, &rank);

    /* Set errors return on COMM_WORLD and the new comm */
    MPI_Comm_set_errhandler(MPI_ERRORS_RETURN, MPI_COMM_WORLD);
    MPI_Comm_set_errhandler(MPI_ERRORS_RETURN, newcomm);

    err = MPI_Barrier(MPI_COMM_WORLD);
    if (err)
        errs += ReportErr(err, "Barrier");
    if (color) {
        /* Simulate a fault on some processes */
        exit(1);
    }
    else {
        /* To improve the chance that the "faulted" processes will have
         * exited, wait for 1 second */
        MTestSleep(1);
    }

    /* Can we still use newcomm? */
    for (j = 0; j < rank; j++) {
        err = MPI_Recv(&tmp, 1, MPI_INT, j, 0, newcomm, MPI_STATUS_IGNORE);
        if (err)
            errs += ReportErr(err, "Recv");
    }
    for (j = rank + 1; j < size; j++) {
        err = MPI_Send(&rank, 1, MPI_INT, j, 0, newcomm);
        if (err)
            errs += ReportErr(err, "Recv");
    }

    /* Now, try sending in MPI_COMM_WORLD on dead processes */
    /* There is a race condition here - we don't know for sure that the faulted
     * processes have exited.  However, we can ensure a failure by using
     * synchronous sends - the sender will wait until the reciever handles
     * receives the message, which will not happen (the process will exit
     * without matching the message, even if it has not yet exited). */
    for (j = 1; j <= wsize / 2; j++) {
        err = MPI_Ssend(&rank, 1, MPI_INT, j, 0, MPI_COMM_WORLD);
        if (!err) {
            errs++;
            fprintf(stderr, "Ssend succeeded to dead process %d\n", j);
        }
    }

    err = MPI_Allreduce(&errs, &toterrs, 1, MPI_INT, MPI_SUM, newcomm);
    if (err)
        errs += ReportErr(err, "Allreduce");
    MPI_Comm_free(&newcomm);

    MPI_Finalize();

    if (wrank == 0) {
        if (toterrs > 0) {
            printf(" Found %d errors\n", toterrs);
        }
        else {
            printf(" No Errors\n");
        }
    }

    return 0;
}

int ReportErr(int errcode, const char name[])
{
    int errclass, errlen;
    char errmsg[MPI_MAX_ERROR_STRING];
    MPI_Error_class(errcode, &errclass);
    MPI_Error_string(errcode, errmsg, &errlen);
    fprintf(stderr, "In %s, error code %d(class %d) = %s\n", name, errcode, errclass, errmsg);
    return 1;
}