/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
* (C) 2006 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
/* This program provides a simple test of send-receive performance between
two (or more) processes. This sometimes called head-to-head or
ping-ping test, as both processes send at the same time.
*/
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#define MAXTESTS 32
#define ERROR_MARGIN 1.0 /* FIXME: This number is pretty much randomly chosen */
static int verbose = 0;
int main(int argc, char *argv[])
{
int wsize, wrank, partner, len, maxlen, k, reps, repsleft;
double t1;
MPI_Request rreq;
char *rbuf, *sbuf;
double times[3][MAXTESTS];
MPI_Init(&argc, &argv);
if (getenv("MPITEST_VERBOSE"))
verbose = 1;
MPI_Comm_size(MPI_COMM_WORLD, &wsize);
MPI_Comm_rank(MPI_COMM_WORLD, &wrank);
if (wsize < 2) {
fprintf(stderr, "This program requires at least 2 processes\n");
MPI_Abort(MPI_COMM_WORLD, 1);
}
/* Set partner based on whether rank is odd or even */
if (wrank & 0x1) {
partner = wrank - 1;
}
else if (wrank < wsize - 1) {
partner = wrank + 1;
}
else
/* Handle wsize odd */
partner = MPI_PROC_NULL;
/* Allocate and initialize buffers */
maxlen = 1024 * 1024;
rbuf = (char *) malloc(maxlen);
sbuf = (char *) malloc(maxlen);
if (!rbuf || !sbuf) {
fprintf(stderr, "Could not allocate %d byte buffers\n", maxlen);
MPI_Abort(MPI_COMM_WORLD, 2);
}
for (k = 0; k < maxlen; k++) {
rbuf[k] = 0;
sbuf[k] = 0;
}
MPI_Barrier(MPI_COMM_WORLD);
/* Test Irecv and send, head to head */
if (wrank == 0 && verbose) {
printf("Irecv-send\n");
printf("len\ttime \trate\n");
}
/* Send powers of 2 bytes */
len = 1;
for (k = 0; k < 20; k++) {
/* We use a simple linear form for the number of tests to
* reduce the impact of the granularity of the timer */
reps = 50 - k;
repsleft = reps;
/* Make sure that both processes are ready to start */
MPI_Sendrecv(MPI_BOTTOM, 0, MPI_BYTE, partner, 0,
MPI_BOTTOM, 0, MPI_BYTE, partner, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
t1 = MPI_Wtime();
while (repsleft--) {
MPI_Irecv(rbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD, &rreq);
MPI_Send(sbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD);
MPI_Wait(&rreq, MPI_STATUS_IGNORE);
}
t1 = MPI_Wtime() - t1;
times[0][k] = t1 / reps;
if (wrank == 0) {
t1 = t1 / reps;
if (t1 > 0) {
double rate;
rate = (len / t1) / 1.e6;
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\t%g\n", len, t1, len / t1);
}
else {
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\tINF\n", len, t1);
}
if (verbose)
fflush(stdout);
}
len *= 2;
}
MPI_Barrier(MPI_COMM_WORLD);
/* Test Sendrecv, head to head */
if (wrank == 0 && verbose) {
printf("Sendrecv\n");
printf("len\ttime (usec)\trate (MB/s)\n");
}
/* Send powers of 2 bytes */
len = 1;
for (k = 0; k < 20; k++) {
/* We use a simple linear form for the number of tests to
* reduce the impact of the granularity of the timer */
reps = 50 - k;
repsleft = reps;
/* Make sure that both processes are ready to start */
MPI_Sendrecv(MPI_BOTTOM, 0, MPI_BYTE, partner, 0,
MPI_BOTTOM, 0, MPI_BYTE, partner, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
t1 = MPI_Wtime();
while (repsleft--) {
MPI_Sendrecv(sbuf, len, MPI_BYTE, partner, k,
rbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
t1 = MPI_Wtime() - t1;
times[1][k] = t1 / reps;
if (wrank == 0) {
t1 = t1 / reps;
if (t1 > 0) {
double rate;
rate = (len / t1) / 1.e6;
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\t%g\n", len, t1, len / t1);
}
else {
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\tINF\n", len, t1);
}
if (verbose)
fflush(stdout);
}
len *= 2;
}
MPI_Barrier(MPI_COMM_WORLD);
/* Test Send/recv, ping-pong */
if (wrank == 0 && verbose) {
printf("Pingpong\n");
printf("len\ttime (usec)\trate (MB/s)\n");
}
/* Send powers of 2 bytes */
len = 1;
for (k = 0; k < 20; k++) {
/* We use a simple linear form for the number of tests to
* reduce the impact of the granularity of the timer */
reps = 50 - k;
repsleft = reps;
/* Make sure that both processes are ready to start */
MPI_Sendrecv(MPI_BOTTOM, 0, MPI_BYTE, partner, 0,
MPI_BOTTOM, 0, MPI_BYTE, partner, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
t1 = MPI_Wtime();
while (repsleft--) {
if (wrank & 0x1) {
MPI_Send(sbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD);
MPI_Recv(rbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
else {
MPI_Recv(rbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
MPI_Send(sbuf, len, MPI_BYTE, partner, k, MPI_COMM_WORLD);
}
}
t1 = MPI_Wtime() - t1;
times[2][k] = t1 / reps;
if (wrank == 0) {
t1 = t1 / reps;
if (t1 > 0) {
double rate;
rate = (len / t1) / 1.e6;
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\t%g\n", len, t1, len / t1);
}
else {
t1 = t1 * 1.e6;
if (verbose)
printf("%d\t%g\tINF\n", len, t1);
}
if (verbose)
fflush(stdout);
}
len *= 2;
}
/* At this point, we could optionally analyze the results and report
* success or failure based on some criteria, such as near monotone
* increases in bandwidth. This test was created because of a
* fall-off in performance noted in the ch3:sock device:channel */
if (wrank == 0) {
int nPerfErrors = 0;
len = 1;
for (k = 0; k < 20; k++) {
double T0, T1, T2;
T0 = times[0][k] * 1.e6;
T1 = times[1][k] * 1.e6;
T2 = times[2][k] * 1.e6;
if (verbose)
printf("%d\t%12.2f\t%12.2f\t%12.2f\n", len, T0, T1, T2);
/* Lets look at long messages only */
if (k > 10) {
double T0Old, T1Old, T2Old;
T0Old = times[0][k - 1] * 1.0e6;
T1Old = times[1][k - 1] * 1.0e6;
T2Old = times[2][k - 1] * 1.0e6;
if (T0 > (2 + ERROR_MARGIN) * T0Old) {
nPerfErrors++;
if (verbose)
printf("Irecv-Send:\t%d\t%12.2f\t%12.2f\n", len, T0Old, T0);
}
if (T1 > (2 + ERROR_MARGIN) * T1Old) {
nPerfErrors++;
if (verbose)
printf("Sendrecv:\t%d\t%12.2f\t%12.2f\n", len, T1Old, T1);
}
if (T2 > (2 + ERROR_MARGIN) * T2Old) {
nPerfErrors++;
if (verbose)
printf("Pingpong:\t%d\t%12.2f\t%12.2f\n", len, T2Old, T2);
}
}
len *= 2;
}
if (nPerfErrors > 8) {
/* Allow for 1-2 errors for eager-rendezvous shifting
* point and cache effects. There should be a better way
* of doing this. */
printf(" Found %d performance errors\n", nPerfErrors);
}
else {
printf(" No Errors\n");
}
fflush(stdout);
}
free(sbuf);
free(rbuf);
MPI_Finalize();
return 0;
}