/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */

/*

 *  (C) 2011 by Argonne National Laboratory.

 *      See COPYRIGHT in top-level directory.

 */

/* This test attempts to execute multiple simultaneous nonblocking collective

 * (NBC) MPI routines at the same time, and manages their completion with a

 * variety of routines (MPI_{Wait,Test}{,_all,_any,_some}).  It also throws a

 * few point-to-point operations into the mix.

 *

 * Possible improvements:

 * - post operations on multiple comms from multiple threads

 */

#include "mpi.h"

#include <stdlib.h>

#include <stdio.h>

#include <string.h>

#include <assert.h>

#ifdef HAVE_UNISTD_H

#include <unistd.h>

#endif

static int errs = 0;

/* Constants that control the high level test harness behavior. */

/* MAIN_ITERATIONS is how many NBC ops the test will attempt to issue. */

#define MAIN_ITERATIONS (100000)

/* WINDOW is the maximum number of outstanding NBC requests at any given time */

#define WINDOW (20)

/* we sleep with probability 1/CHANCE_OF_SLEEP */

#define CHANCE_OF_SLEEP (1000)

/* JITTER_DELAY is denominated in microseconds (us) */

#define JITTER_DELAY (50000)    /* 0.05 seconds */

/* NUM_COMMS is the number of communicators on which ops will be posted */

#define NUM_COMMS (4)

/* Constants that control behavior of the individual testing operations.

 * Altering these can help to explore the testing space, but increasing them too

 * much can consume too much memory (often O(n^2) usage). */

/* FIXME is COUNT==10 too limiting? should we try a larger count too (~500)? */

#define COUNT (10)

#define PRIME (17)

#define my_assert(cond_)                                                                 \

    do {                                                                                 \

        if (!(cond_)) {                                                                  \

            ++errs;                                                                      \

            if (errs < 10) {                                                             \

                fprintf(stderr, "assertion (%s) failed on line %d\n", #cond_, __LINE__); \

            }                                                                            \

        }                                                                                \

    } while (0)

/* Intended to act like "rand_r", but we can be sure that it will exist and be

 * consistent across all of comm world.  Returns a number in the range

 * [0,GEN_PRN_MAX] */

#define GEN_PRN_MAX (4294967291-1)

static unsigned int gen_prn(unsigned int x)

{

    /* a simple "multiplicative congruential method" PRNG, with parameters:

     *   m=4294967291, largest 32-bit prime

     *   a=279470273, good primitive root of m from "TABLES OF LINEAR

     *                CONGRUENTIAL GENERATORS OF DIFFERENT SIZES AND GOOD

     *                LATTICE STRUCTURE", by Pierre L’Ecuyer */

    return (279470273UL * (unsigned long) x) % 4294967291UL;

}

/* given a random unsigned int value "rndval_" from gen_prn, this evaluates to a

 * value in the range [min_,max_) */

#define rand_range(rndval_,min_,max_) \

    ((unsigned int)((min_) + ((rndval_) * (1.0 / (GEN_PRN_MAX+1.0)) * ((max_) - (min_)))))

static void sum_fn(void *invec, void *inoutvec, int *len, MPI_Datatype * datatype)

{

    int i;

    int *in = invec;

    int *inout = inoutvec;

    for (i = 0; i < *len; ++i) {

        inout[i] = in[i] + inout[i];

    }

}

/* used to keep track of buffers that should be freed after the corresponding

 * operation has completed */

struct laundry {

    int case_num;               /* which test case initiated this req/laundry */

    MPI_Comm comm;

    int *buf;

    int *recvbuf;

    int *sendcounts;

    int *recvcounts;

    int *sdispls;

    int *rdispls;

    int *sendtypes;

    int *recvtypes;

};

static void cleanup_laundry(struct laundry *l)

{

    l->case_num = -1;

    l->comm = MPI_COMM_NULL;

    if (l->buf)

        free(l->buf);

    if (l->recvbuf)

        free(l->recvbuf);

    if (l->sendcounts)

        free(l->sendcounts);

    if (l->recvcounts)

        free(l->recvcounts);

    if (l->sdispls)

        free(l->sdispls);

    if (l->rdispls)

        free(l->rdispls);

    if (l->sendtypes)

        free(l->sendtypes);

    if (l->recvtypes)

        free(l->recvtypes);

}

/* Starts a "random" operation on "comm" corresponding to "rndnum" and returns

 * in (*req) a request handle corresonding to that operation.  This call should

 * be considered collective over comm (with a consistent value for "rndnum"),

 * even though the operation may only be a point-to-point request. */

static void start_random_nonblocking(MPI_Comm comm, unsigned int rndnum, MPI_Request * req,

                                     struct laundry *l)

{

    int i, j;

    int rank, size;

    int *buf = NULL;

    int *recvbuf = NULL;

    int *sendcounts = NULL;

    int *recvcounts = NULL;

    int *sdispls = NULL;

    int *rdispls = NULL;

    int *sendtypes = NULL;

    int *recvtypes = NULL;

    signed char *buf_alias = NULL;

    MPI_Comm_rank(comm, &rank;;

    MPI_Comm_size(comm, &size);

    *req = MPI_REQUEST_NULL;

    l->case_num = -1;

    l->comm = comm;

    l->buf = buf = malloc(COUNT * size * sizeof(int));

    l->recvbuf = recvbuf = malloc(COUNT * size * sizeof(int));

    l->sendcounts = sendcounts = malloc(size * sizeof(int));

    l->recvcounts = recvcounts = malloc(size * sizeof(int));

    l->sdispls = sdispls = malloc(size * sizeof(int));

    l->rdispls = rdispls = malloc(size * sizeof(int));

    l->sendtypes = sendtypes = malloc(size * sizeof(MPI_Datatype));

    l->recvtypes = recvtypes = malloc(size * sizeof(MPI_Datatype));

#define NUM_CASES (21)

    l->case_num = rand_range(rndnum, 0, NUM_CASES);

    switch (l->case_num) {

    case 0:    /* MPI_Ibcast */

        for (i = 0; i < COUNT; ++i) {

            if (rank == 0) {

                buf[i] = i;

            }

            else {

                buf[i] = 0xdeadbeef;

            }

        }

        MPI_Ibcast(buf, COUNT, MPI_INT, 0, comm, req);

        break;

    case 1:    /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */

        /* FIXME fiddle with PRIME and buffer allocation s.t. PRIME is much larger (1021?) */

        buf_alias = (signed char *) buf;

        my_assert(COUNT * size * sizeof(int) > PRIME);  /* sanity */

        for (i = 0; i < PRIME; ++i) {

            if (rank == 0)

                buf_alias[i] = i;

            else

                buf_alias[i] = 0xdb;

        }

        for (i = PRIME; i < COUNT * size * sizeof(int); ++i) {

            buf_alias[i] = 0xbf;

        }

        MPI_Ibcast(buf_alias, PRIME, MPI_SIGNED_CHAR, 0, comm, req);

        break;

    case 2:    /* MPI_Ibarrier */

        MPI_Ibarrier(comm, req);

        break;

    case 3:    /* MPI_Ireduce */

        for (i = 0; i < COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, 0, comm, req);

        break;

    case 4:    /* same again, use a user op and free it before the wait */

        {

            MPI_Op op = MPI_OP_NULL;

            MPI_Op_create(sum_fn, /*commute= */ 1, &op);

            for (i = 0; i < COUNT; ++i) {

                buf[i] = rank + i;

                recvbuf[i] = 0xdeadbeef;

            }

            MPI_Ireduce(buf, recvbuf, COUNT, MPI_INT, op, 0, comm, req);

            MPI_Op_free(&op);

        }

        break;

    case 5:    /* MPI_Iallreduce */

        for (i = 0; i < COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Iallreduce(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);

        break;

    case 6:    /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */

        for (i = 0; i < size; ++i) {

            sendcounts[i] = COUNT;

            recvcounts[i] = COUNT;

            sdispls[i] = COUNT * i;

            rdispls[i] = COUNT * i;

            for (j = 0; j < COUNT; ++j) {

                buf[i * COUNT + j] = rank + (i * j);

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Ialltoallv(buf, sendcounts, sdispls, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT,

                       comm, req);

        break;

    case 7:    /* MPI_Igather */

        for (i = 0; i < size * COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);

        break;

    case 8:    /* same test again, just use a dup'ed datatype and free it before the wait */

        {

            MPI_Datatype type = MPI_DATATYPE_NULL;

            MPI_Type_dup(MPI_INT, &type);

            for (i = 0; i < size * COUNT; ++i) {

                buf[i] = rank + i;

                recvbuf[i] = 0xdeadbeef;

            }

            MPI_Igather(buf, COUNT, MPI_INT, recvbuf, COUNT, type, 0, comm, req);

            MPI_Type_free(&type);       /* should cause implementations that don't refcount

                                         * correctly to blow up or hang in the wait */

        }

        break;

    case 9:    /* MPI_Iscatter */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                if (rank == 0)

                    buf[i * COUNT + j] = i + j;

                else

                    buf[i * COUNT + j] = 0xdeadbeef;

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Iscatter(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);

        break;

    case 10:   /* MPI_Iscatterv */

        for (i = 0; i < size; ++i) {

            /* weak test, just test the regular case where all counts are equal */

            sendcounts[i] = COUNT;

            sdispls[i] = i * COUNT;

            for (j = 0; j < COUNT; ++j) {

                if (rank == 0)

                    buf[i * COUNT + j] = i + j;

                else

                    buf[i * COUNT + j] = 0xdeadbeef;

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Iscatterv(buf, sendcounts, sdispls, MPI_INT, recvbuf, COUNT, MPI_INT, 0, comm, req);

        break;

    case 11:   /* MPI_Ireduce_scatter */

        for (i = 0; i < size; ++i) {

            recvcounts[i] = COUNT;

            for (j = 0; j < COUNT; ++j) {

                buf[i * COUNT + j] = rank + i;

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Ireduce_scatter(buf, recvbuf, recvcounts, MPI_INT, MPI_SUM, comm, req);

        break;

    case 12:   /* MPI_Ireduce_scatter_block */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                buf[i * COUNT + j] = rank + i;

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Ireduce_scatter_block(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);

        break;

    case 13:   /* MPI_Igatherv */

        for (i = 0; i < size * COUNT; ++i) {

            buf[i] = 0xdeadbeef;

            recvbuf[i] = 0xdeadbeef;

        }

        for (i = 0; i < COUNT; ++i) {

            buf[i] = rank + i;

        }

        for (i = 0; i < size; ++i) {

            recvcounts[i] = COUNT;

            rdispls[i] = i * COUNT;

        }

        MPI_Igatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, 0, comm, req);

        break;

    case 14:   /* MPI_Ialltoall */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                buf[i * COUNT + j] = rank + (i * j);

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Ialltoall(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);

        break;

    case 15:   /* MPI_Iallgather */

        for (i = 0; i < size * COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Iallgather(buf, COUNT, MPI_INT, recvbuf, COUNT, MPI_INT, comm, req);

        break;

    case 16:   /* MPI_Iallgatherv */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

            recvcounts[i] = COUNT;

            rdispls[i] = i * COUNT;

        }

        for (i = 0; i < COUNT; ++i)

            buf[i] = rank + i;

        MPI_Iallgatherv(buf, COUNT, MPI_INT, recvbuf, recvcounts, rdispls, MPI_INT, comm, req);

        break;

    case 17:   /* MPI_Iscan */

        for (i = 0; i < COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Iscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);

        break;

    case 18:   /* MPI_Iexscan */

        for (i = 0; i < COUNT; ++i) {

            buf[i] = rank + i;

            recvbuf[i] = 0xdeadbeef;

        }

        MPI_Iexscan(buf, recvbuf, COUNT, MPI_INT, MPI_SUM, comm, req);

        break;

    case 19:   /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */

        for (i = 0; i < size; ++i) {

            sendcounts[i] = COUNT;

            recvcounts[i] = COUNT;

            sdispls[i] = COUNT * i * sizeof(int);

            rdispls[i] = COUNT * i * sizeof(int);

            sendtypes[i] = MPI_INT;

            recvtypes[i] = MPI_INT;

            for (j = 0; j < COUNT; ++j) {

                buf[i * COUNT + j] = rank + (i * j);

                recvbuf[i * COUNT + j] = 0xdeadbeef;

            }

        }

        MPI_Ialltoallw(buf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes,

                       comm, req);

        break;

    case 20:   /* basic pt2pt MPI_Isend/MPI_Irecv pairing */

        /* even ranks send to odd ranks, but only if we have a full pair */

        if ((rank % 2 != 0) || (rank != size - 1)) {

            for (j = 0; j < COUNT; ++j) {

                buf[j] = j;

                recvbuf[j] = 0xdeadbeef;

            }

            if (rank % 2 == 0)

                MPI_Isend(buf, COUNT, MPI_INT, rank + 1, 5, comm, req);

            else

                MPI_Irecv(recvbuf, COUNT, MPI_INT, rank - 1, 5, comm, req);

        }

        break;

    default:

        fprintf(stderr, "unexpected value for l->case_num=%d)\n", (l->case_num));

        MPI_Abort(comm, 1);

        break;

    }

}

static void check_after_completion(struct laundry *l)

{

    int i, j;

    int rank, size;

    MPI_Comm comm = l->comm;

    int *buf = l->buf;

    int *recvbuf = l->recvbuf;

    int *sendcounts = l->sendcounts;

    int *recvcounts = l->recvcounts;

    int *sdispls = l->sdispls;

    int *rdispls = l->rdispls;

    int *sendtypes = l->sendtypes;

    int *recvtypes = l->recvtypes;

    char *buf_alias = (char *) buf;

    MPI_Comm_rank(comm, &rank;;

    MPI_Comm_size(comm, &size);

    /* these cases all correspond to cases in start_random_nonblocking */

    switch (l->case_num) {

    case 0:    /* MPI_Ibcast */

        for (i = 0; i < COUNT; ++i) {

            if (buf[i] != i)

                printf("buf[%d]=%d i=%d\n", i, buf[i], i);

            my_assert(buf[i] == i);

        }

        break;

    case 1:    /* MPI_Ibcast (again, but designed to stress scatter/allgather impls) */

        for (i = 0; i < PRIME; ++i) {

            if (buf_alias[i] != i)

                printf("buf_alias[%d]=%d i=%d\n", i, buf_alias[i], i);

            my_assert(buf_alias[i] == i);

        }

        break;

    case 2:    /* MPI_Ibarrier */

        /* nothing to check */

        break;

    case 3:    /* MPI_Ireduce */

        if (rank == 0) {

            for (i = 0; i < COUNT; ++i) {

                if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))

                    printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],

                           ((size * (size - 1) / 2) + (i * size)));

                my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));

            }

        }

        break;

    case 4:    /* same again, use a user op and free it before the wait */

        if (rank == 0) {

            for (i = 0; i < COUNT; ++i) {

                if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))

                    printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],

                           ((size * (size - 1) / 2) + (i * size)));

                my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));

            }

        }

        break;

    case 5:    /* MPI_Iallreduce */

        for (i = 0; i < COUNT; ++i) {

            if (recvbuf[i] != ((size * (size - 1) / 2) + (i * size)))

                printf("got recvbuf[%d]=%d, expected %d\n", i, recvbuf[i],

                       ((size * (size - 1) / 2) + (i * size)));

            my_assert(recvbuf[i] == ((size * (size - 1) / 2) + (i * size)));

        }

        break;

    case 6:    /* MPI_Ialltoallv (a weak test, neither irregular nor sparse) */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */

                my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));

            }

        }

        break;

    case 7:    /* MPI_Igather */

        if (rank == 0) {

            for (i = 0; i < size; ++i) {

                for (j = 0; j < COUNT; ++j) {

                    my_assert(recvbuf[i * COUNT + j] == i + j);

                }

            }

        }

        else {

            for (i = 0; i < size * COUNT; ++i) {

                my_assert(recvbuf[i] == 0xdeadbeef);

            }

        }

        break;

    case 8:    /* same test again, just use a dup'ed datatype and free it before the wait */

        if (rank == 0) {

            for (i = 0; i < size; ++i) {

                for (j = 0; j < COUNT; ++j) {

                    my_assert(recvbuf[i * COUNT + j] == i + j);

                }

            }

        }

        else {

            for (i = 0; i < size * COUNT; ++i) {

                my_assert(recvbuf[i] == 0xdeadbeef);

            }

        }

        break;

    case 9:    /* MPI_Iscatter */

        for (j = 0; j < COUNT; ++j) {

            my_assert(recvbuf[j] == rank + j);

        }

        if (rank != 0) {

            for (i = 0; i < size * COUNT; ++i) {

                /* check we didn't corrupt the sendbuf somehow */

                my_assert(buf[i] == 0xdeadbeef);

            }

        }

        break;

    case 10:   /* MPI_Iscatterv */

        for (j = 0; j < COUNT; ++j) {

            my_assert(recvbuf[j] == rank + j);

        }

        if (rank != 0) {

            for (i = 0; i < size * COUNT; ++i) {

                /* check we didn't corrupt the sendbuf somehow */

                my_assert(buf[i] == 0xdeadbeef);

            }

        }

        for (i = 1; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /* check we didn't corrupt the rest of the recvbuf */

                my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);

            }

        }

        break;

    case 11:   /* MPI_Ireduce_scatter */

        for (j = 0; j < COUNT; ++j) {

            my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));

        }

        for (i = 1; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /* check we didn't corrupt the rest of the recvbuf */

                my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);

            }

        }

        break;

    case 12:   /* MPI_Ireduce_scatter_block */

        for (j = 0; j < COUNT; ++j) {

            my_assert(recvbuf[j] == (size * rank + ((size - 1) * size) / 2));

        }

        for (i = 1; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /* check we didn't corrupt the rest of the recvbuf */

                my_assert(recvbuf[i * COUNT + j] == 0xdeadbeef);

            }

        }

        break;

    case 13:   /* MPI_Igatherv */

        if (rank == 0) {

            for (i = 0; i < size; ++i) {

                for (j = 0; j < COUNT; ++j) {

                    my_assert(recvbuf[i * COUNT + j] == i + j);

                }

            }

        }

        else {

            for (i = 0; i < size * COUNT; ++i) {

                my_assert(recvbuf[i] == 0xdeadbeef);

            }

        }

        break;

    case 14:   /* MPI_Ialltoall */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (i * j))); */

                my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));

            }

        }

        break;

    case 15:   /* MPI_Iallgather */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                my_assert(recvbuf[i * COUNT + j] == i + j);

            }

        }

        break;

    case 16:   /* MPI_Iallgatherv */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                my_assert(recvbuf[i * COUNT + j] == i + j);

            }

        }

        break;

    case 17:   /* MPI_Iscan */

        for (i = 0; i < COUNT; ++i) {

            my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1))));

        }

        break;

    case 18:   /* MPI_Iexscan */

        for (i = 0; i < COUNT; ++i) {

            if (rank == 0)

                my_assert(recvbuf[i] == 0xdeadbeef);

            else

                my_assert(recvbuf[i] == ((rank * (rank + 1) / 2) + (i * (rank + 1)) - (rank + i)));

        }

        break;

    case 19:   /* MPI_Ialltoallw (a weak test, neither irregular nor sparse) */

        for (i = 0; i < size; ++i) {

            for (j = 0; j < COUNT; ++j) {

                /*printf("recvbuf[%d*COUNT+%d]=%d, expecting %d\n", i, j, recvbuf[i*COUNT+j], (i + (rank * j))); */

                my_assert(recvbuf[i * COUNT + j] == (i + (rank * j)));

            }

        }

        break;

    case 20:   /* basic pt2pt MPI_Isend/MPI_Irecv pairing */

        /* even ranks send to odd ranks, but only if we have a full pair */

        if ((rank % 2 != 0) || (rank != size - 1)) {

            for (j = 0; j < COUNT; ++j) {

                /* only odd procs did a recv */

                if (rank % 2 == 0) {

                    my_assert(recvbuf[j] == 0xdeadbeef);

                }

                else {

                    if (recvbuf[j] != j)

                        printf("recvbuf[%d]=%d j=%d\n", j, recvbuf[j], j);

                    my_assert(recvbuf[j] == j);

                }

            }

        }

        break;

    default:

        printf("invalid case_num (%d) detected\n", l->case_num);

        assert(0);

        break;

    }

}

#undef NUM_CASES

static void complete_something_somehow(unsigned int rndnum, int numreqs, MPI_Request reqs[],

                                       int *outcount, int indices[])

{

    int i, idx, flag;

#define COMPLETION_CASES (8)

    switch (rand_range(rndnum, 0, COMPLETION_CASES)) {

    case 0:

        MPI_Waitall(numreqs, reqs, MPI_STATUSES_IGNORE);

        *outcount = numreqs;

        for (i = 0; i < numreqs; ++i) {

            indices[i] = i;

        }

        break;

    case 1:

        MPI_Testsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);

        if (*outcount == MPI_UNDEFINED) {

            *outcount = 0;

        }

        break;

    case 2:

        MPI_Waitsome(numreqs, reqs, outcount, indices, MPI_STATUS_IGNORE);

        if (*outcount == MPI_UNDEFINED) {

            *outcount = 0;

        }

        break;

    case 3:

        MPI_Waitany(numreqs, reqs, &idx, MPI_STATUS_IGNORE);

        if (idx == MPI_UNDEFINED) {

            *outcount = 0;

        }

        else {

            *outcount = 1;

            indices[0] = idx;

        }

        break;

    case 4:

        MPI_Testany(numreqs, reqs, &idx, &flag, MPI_STATUS_IGNORE);

        if (idx == MPI_UNDEFINED) {

            *outcount = 0;

        }

        else {

            *outcount = 1;

            indices[0] = idx;

        }

        break;

    case 5:

        MPI_Testall(numreqs, reqs, &flag, MPI_STATUSES_IGNORE);

        if (flag) {

            *outcount = numreqs;

            for (i = 0; i < numreqs; ++i) {

                indices[i] = i;

            }

        }

        else {

            *outcount = 0;

        }

        break;

    case 6:

        /* select a new random index and wait on it */

        rndnum = gen_prn(rndnum);

        idx = rand_range(rndnum, 0, numreqs);

        MPI_Wait(&reqs[idx], MPI_STATUS_IGNORE);

        *outcount = 1;

        indices[0] = idx;

        break;

    case 7:

        /* select a new random index and wait on it */

        rndnum = gen_prn(rndnum);

        idx = rand_range(rndnum, 0, numreqs);

        MPI_Test(&reqs[idx], &flag, MPI_STATUS_IGNORE);

        *outcount = (flag ? 1 : 0);

        indices[0] = idx;

        break;

    default:

        assert(0);

        break;

    }

#undef COMPLETION_CASES

}

int main(int argc, char **argv)

{

    int i, num_posted, num_completed;

    int wrank, wsize;

    unsigned int seed = 0x10bc;

    unsigned int post_seq, complete_seq;

    struct laundry larr[WINDOW];

    MPI_Request reqs[WINDOW];

    int outcount;

    int indices[WINDOW];

    MPI_Comm comms[NUM_COMMS];

    MPI_Comm comm;

    MPI_Init(&argc, &argv);

    MPI_Comm_rank(MPI_COMM_WORLD, &wrank);

    MPI_Comm_size(MPI_COMM_WORLD, &wsize);

    /* it is critical that all processes in the communicator start with a

     * consistent value for "post_seq" */

    post_seq = complete_seq = gen_prn(seed);

    num_completed = 0;

    num_posted = 0;

    /* construct all of the communicators, just dups of comm world for now */

    for (i = 0; i < NUM_COMMS; ++i) {

        MPI_Comm_dup(MPI_COMM_WORLD, &comms[i]);

    }

    /* fill the entire window of ops */

    for (i = 0; i < WINDOW; ++i) {

        reqs[i] = MPI_REQUEST_NULL;

        memset(&larr[i], 0, sizeof(struct laundry));

        larr[i].case_num = -1;

        /* randomly select a comm, using a new seed to avoid correlating

         * particular kinds of NBC ops with particular communicators */

        comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];

        start_random_nonblocking(comm, post_seq, &reqs[i], &larr[i]);

        ++num_posted;

        post_seq = gen_prn(post_seq);

    }

    /* now loop repeatedly, completing ops with "random" completion functions,

     * until we've posted and completed MAIN_ITERATIONS ops */

    while (num_completed < MAIN_ITERATIONS) {

        complete_something_somehow(complete_seq, WINDOW, reqs, &outcount, indices);

        complete_seq = gen_prn(complete_seq);

        for (i = 0; i < outcount; ++i) {

            int idx = indices[i];

            assert(reqs[idx] == MPI_REQUEST_NULL);

            if (larr[idx].case_num != -1) {

                check_after_completion(&larr[idx]);

                cleanup_laundry(&larr[idx]);

                ++num_completed;

                if (num_posted < MAIN_ITERATIONS) {

                    comm = comms[rand_range(gen_prn(post_seq), 0, NUM_COMMS)];

                    start_random_nonblocking(comm, post_seq, &reqs[idx], &larr[idx]);

                    ++num_posted;

                    post_seq = gen_prn(post_seq);

                }

            }

        }

        /* "randomly" and infrequently introduce some jitter into the system */

        if (0 == rand_range(gen_prn(complete_seq + wrank), 0, CHANCE_OF_SLEEP)) {

            usleep(JITTER_DELAY);       /* take a short nap */

        }

    }

    for (i = 0; i < NUM_COMMS; ++i) {

        MPI_Comm_free(&comms[i]);

    }

    if (wrank == 0) {

        if (errs)

            printf("found %d errors\n", errs);

        else

            printf(" No errors\n");

    }

    MPI_Finalize();

    return 0;

}