/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2004 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include "mpitest.h"
#include <assert.h>

/*
static char MTEST_Descrip[] = "Test MPI_Allreduce with non-commutative user-defined operations using matrix rotations";
*/

/* This example is similar to allred3.c, but uses only 3x3 matrics with
   integer-valued entries.  This is an associative but not commutative
   operation.
   The number of matrices is the count argument. The matrix is stored
   in C order, so that
     c(i,j) is cin[j+i*3]

   Three different matrices are used:
   I = identity matrix
   A = (1 0 0    B = (0 1 0
        0 0 1         1 0 0
        0 1 0)        0 0 1)

   The product

         I^k A I^(p-2-k-j) B I^j

   is

   (0 1 0
     0 0 1
     1 0 0)

   for all values of k, p, and j.
 */

void matmult(void *cinPtr, void *coutPtr, int *count, MPI_Datatype * dtype);

void matmult(void *cinPtr, void *coutPtr, int *count, MPI_Datatype * dtype)
{
    const int *cin = (const int *) cinPtr;
    int *cout = (int *) coutPtr;
    int i, j, k, nmat;
    int tempcol[3];
    int offset1, offset2;

    for (nmat = 0; nmat < *count; nmat++) {
        for (j = 0; j < 3; j++) {
            for (i = 0; i < 3; i++) {
                tempcol[i] = 0;
                for (k = 0; k < 3; k++) {
                    /* col[i] += cin(i,k) * cout(k,j) */
                    offset1 = k + i * 3;
                    offset2 = j + k * 3;
                    tempcol[i] += cin[offset1] * cout[offset2];
                }
            }
            for (i = 0; i < 3; i++) {
                offset1 = j + i * 3;
                cout[offset1] = tempcol[i];
            }
        }
        /* Advance to the next matrix */
        cin += 9;
        cout += 9;
    }
}

/* Initialize the integer matrix as one of the
   above matrix entries, as a function of count.
   We guarantee that both the A and B matrices are included.
*/
static void initMat(int rank, int size, int nmat, int mat[])
{
    int i, kind;

    /* Zero the matrix */
    for (i = 0; i < 9; i++) {
        mat[i] = 0;
    }

    /* Decide which matrix to create (I, A, or B) */
    if (size == 2) {
        /* rank 0 is A, 1 is B */
        kind = 1 + rank;
    }
    else {
        int tmpA, tmpB;
        /* Most ranks are identity matrices */
        kind = 0;
        /* Make sure exactly one rank gets the A matrix
         * and one the B matrix */
        tmpA = size / 4;
        tmpB = (3 * size) / 4;

        if (rank == tmpA)
            kind = 1;
        if (rank == tmpB)
            kind = 2;
    }

    switch (kind) {
    case 0:    /* Identity */
        mat[0] = 1;
        mat[4] = 1;
        mat[8] = 1;
        break;
    case 1:    /* A */
        mat[0] = 1;
        mat[5] = 1;
        mat[7] = 1;
        break;
    case 2:    /* B */
        mat[1] = 1;
        mat[3] = 1;
        mat[8] = 1;
        break;
    }
}

/* Compare a matrix with the known result */
static int checkResult(int nmat, int mat[], const char *msg)
{
    int n, k, errs = 0, wrank;
    static int solution[9] = { 0, 1, 0,
        0, 0, 1,
        1, 0, 0
    };

    MPI_Comm_rank(MPI_COMM_WORLD, &wrank);

    for (n = 0; n < nmat; n++) {
        for (k = 0; k < 9; k++) {
            if (mat[k] != solution[k]) {
                errs++;
                if (errs == 1) {
                    printf("Errors for communicators %s\n", MTestGetIntracommName());
                    fflush(stdout);

                }
                if (errs < 10) {
                    printf("[%d]matrix #%d(%s): Expected mat[%d,%d] = %d, got %d\n",
                           wrank, n, msg, k / 3, k % 3, solution[k], mat[k]);
                    fflush(stdout);
                }
            }
        }
        /* Advance to the next matrix */
        mat += 9;
    }
    return errs;
}

int main(int argc, char *argv[])
{
    int errs = 0;
    int size, rank;
    int minsize = 2, count;
    MPI_Comm comm;
    int *buf, *bufout;
    MPI_Op op;
    MPI_Datatype mattype;
    int i;

    MTest_Init(&argc, &argv);

    MPI_Op_create(matmult, 0, &op);

    /* A single rotation matrix (3x3, stored as 9 consequetive elements) */
    MPI_Type_contiguous(9, MPI_INT, &mattype);
    MPI_Type_commit(&mattype);

    /* Sanity check: test that our routines work properly */
    {
        int one = 1;
        buf = (int *) malloc(4 * 9 * sizeof(int));
        initMat(0, 4, 0, &buf[0]);
        initMat(1, 4, 0, &buf[9]);
        initMat(2, 4, 0, &buf[18]);
        initMat(3, 4, 0, &buf[27]);
        matmult(&buf[0], &buf[9], &one, &mattype);
        matmult(&buf[9], &buf[18], &one, &mattype);
        matmult(&buf[18], &buf[27], &one, &mattype);
        checkResult(1, &buf[27], "Sanity Check");
        free(buf);
    }

    while (MTestGetIntracommGeneral(&comm, minsize, 1)) {
        if (comm == MPI_COMM_NULL)
            continue;

        MPI_Comm_size(comm, &size);
        MPI_Comm_rank(comm, &rank);

        for (count = 1; count < size; count++) {

            /* Allocate the matrices */
            buf = (int *) malloc(count * 9 * sizeof(int));
            if (!buf) {
                MPI_Abort(MPI_COMM_WORLD, 1);
            }

            bufout = (int *) malloc(count * 9 * sizeof(int));
            if (!bufout) {
                MPI_Abort(MPI_COMM_WORLD, 1);
            }

            for (i = 0; i < count; i++) {
                initMat(rank, size, i, &buf[i * 9]);
            }

            MPI_Allreduce(buf, bufout, count, mattype, op, comm);
            errs += checkResult(count, bufout, "");

            /* Try the same test, but using MPI_IN_PLACE */
            for (i = 0; i < count; i++) {
                initMat(rank, size, i, &bufout[i * 9]);
            }
            MPI_Allreduce(MPI_IN_PLACE, bufout, count, mattype, op, comm);
            errs += checkResult(count, bufout, "IN_PLACE");

            free(buf);
            free(bufout);
        }
        MTestFreeComm(&comm);
    }

    MPI_Op_free(&op);
    MPI_Type_free(&mattype);

    MTest_Finalize(errs);
    MPI_Finalize();
    return 0;
}