#! @PERL@

# -*- Mode: perl; -*-

#

# This script is the beginnings of a script to run a sequence of test 

# programs.  See the MPICH document for a description of the test

# strategy and requirements.

#

# Description

#   Tests are controlled by a file listing test programs; if the file is

#   a directory, then all of the programs in the directory and subdirectories

#   are run

#

#   To run a test, the following steps are executed

#   Build the executable:

#      make programname

#   Run the executable

#      mpiexec -n <np> ./programname >out 2>err

#   Check the return code (non zero is failure)

#   Check the stderr output (non empty is failure)

#   Check the stdout output (No Errors or Test passed are the only valid

#      output)

#   Remove executable, out, err files

#

# The format of a list file is

# programname number-of-processes

# If number-of-processes is missing, $np_default is used (this is 2 but can

# be overridden with -np=new-value)

#

# Special feature:

# Because these tests can take a long time to run, there is an

# option to cause the tests to stop is a "stopfile" is found.

# The stopfile can be created by a separate, watchdog process, to ensure that

# tests end at a certain time.

# The name of this file is (by default) .stoptest

# in the  top-level run directory.  The environment variable

#    MPITEST_STOPTEST

# can specify a different file name.

#

# Import the mkpath command

use File::Path;

use File::Copy qw(move);

# Use high resolution timers

use Time::HiRes qw(gettimeofday tv_interval);

# Global variables

$MPIMajorVersion = "@MPI_VERSION@";

$MPIMinorVersion = "@MPI_SUBVERSION@";

$mpiexec = "@MPIEXEC@";    # Name of mpiexec program (including path, if necessary)

# ppnMax is the maximum number of processes per node.  -1 means ignore.

# ppnArg is the argument to use to mpiexec - format is "string%d"; e.g.,

# "-ppn %d"

$ppnArg  = "";

$ppnMax  = -1;

# timelimitArg is the argument to use to mpiexec to set the timelimit

# in seconds.  The format is "string%d", e.g., "-t %d" for Cray aprun

$timelimitArg="";

#

$testIsStrict = "@MPI_IS_STRICT@";

$MPIhasMPIX   = "@MPI_HAS_MPIX@";

$runxfail     = "@RUN_XFAIL@";

$np_arg  = "-n";         # Name of argument to specify the number of processes

$err_count = 0;          # Number of programs that failed.

$skip_count = 0;         # Number of programs skipped

$total_run = 0;          # Number of programs tested

$total_seen = 0;         # Number of programs considered for testing

$np_default = 2;         # Default number of processes to use

$np_max     = -1;        # Maximum number of processes to use (overrides any

                         # value in the test list files.  -1 is Infinity

$defaultTimeLimit = 180; # default timeout

$defaultTimeLimitMultiplier = 1.0; # default multiplier for timeout limit

$srcdir = ".";           # Used to set the source dir for testlist files

$curdir = ".";           # used to track the relative current directory

# Output forms

$xmloutput = 0;          # Set to true to get xml output (also specify file)

$closeXMLOutput = 1;     # Set to false to leave XML output file open to

                         # accept additional data

$verbose = 0;            # Set to true to get more output

$showProgress = 0;       # Set to true to get a "." with each run program.

$newline = "\r\n";       # Set to \r\n for Windows-friendly, \n for Unix only

$batchRun = 0;           # Set to true to batch the execution of the tests

                         # (i.e., run them together, then test output, 

                         # rather than build/run/check for each test)

$testCount = 0;          # Used with batchRun to count tests.

$batrundir = ".";        # Set to the directory into which to run the examples

# TAP (Test Anything Protocol) output

my $tapoutput = 0;

my $tapfile = '';

my $tapfullfile = '';

# Junit format output

my $junitoutput = 0;

my $junitfile = '';

my $junitfullfile = '';

$debug = 1;

$depth = 0;              # This is used to manage multiple open list files

# Build flags

$remove_this_pgm = 0;

$clean_pgms      = 1;

my $program_wrapper = '';

#---------------------------------------------------------------------------

# Get some arguments from the environment

#   Currently, only the following are understood:

#   VERBOSE

#   RUNTESTS_VERBOSE  (an alias for VERBOSE in case you want to 

#                      reserve VERBOSE)

#   RUNTESTS_SHOWPROGRESS

#   MPITEST_STOPTEST

#   MPITEST_TIMEOUT

#   MPITEST_TIMEOUT_MULTIPLIER

#   MPITEST_PROGRAM_WRAPPER (Value is added after -np but before test

#                            executable.  Tools like valgrind may be inserted

#                            this way.)

#---------------------------------------------------------------------------

if ( defined($ENV{"VERBOSE"}) || defined($ENV{"V"}) || defined($ENV{"RUNTESTS_VERBOSE"}) ) {

    $verbose = 1;

}

if ( defined($ENV{"RUNTESTS_SHOWPROGRESS"} ) ) {

    $showProgress = 1;       

}

if (defined($ENV{"MPITEST_STOPTEST"})) {

    $stopfile = $ENV{"MPITEST_STOPTEST"};

}

else {

    $stopfile = `pwd` . "/.stoptest";

    $stopfile =~ s/\r*\n*//g;    # Remove any newlines (from pwd)

}

if (defined($ENV{"MPITEST_TIMEOUT"})) {

    $defaultTimeLimit = $ENV{"MPITEST_TIMEOUT"};

}

if (defined($ENV{"MPITEST_TIMEOUT_MULTIPLIER"})) {

    $defaultTimeLimitMultiplier = $ENV{"MPITEST_TIMEOUT_MULTIPLIER"};

}

# Define this to leave the XML output file open to receive additional data

if (defined($ENV{'NOXMLCLOSE'}) && $ENV{'NOXMLCLOSE'} eq 'YES') {

    $closeXMLOutput = 0;

}

if (defined($ENV{'MPITEST_PROGRAM_WRAPPER'})) {

    $program_wrapper = $ENV{'MPITEST_PROGRAM_WRAPPER'};

}

if (defined($ENV{'MPITEST_BATCH'})) {

    if ($ENV{'MPITEST_BATCH'} eq 'YES' || $ENV{'MPITEST_BATCH'} eq 'yes') {

	$batchRun = 1;

    } elsif ($ENV{'MPITEST_BATCH'} eq 'NO' || $ENV{'MPITEST_BATCH'} eq 'no') {

	$batchRun = 0;

    }

    else {

	print STDERR "Unrecognized value for MPITEST_BATCH = $ENV{'MPITEST_BATCH'}\n";

    }

}

if (defined($ENV{'MPITEST_BATCHDIR'})) {

    $batrundir = $ENV{'MPITEST_BATCHDIR'};

}

# PPN support

if (defined($ENV{'MPITEST_PPNARG'})) {

    $ppnArg = $ENV{'MPITEST_PPNARG'};

}

if (defined($ENV{'MPITEST_PPNMAX'})) {

    $ppnMax = $ENV{'MPITEST_PPNMAX'};

}

if (defined($ENV{'MPITEST_TIMELIMITARG'})) {

    $timelimitArg = $ENV{'MPITEST_TIMELIMITARG'};

}

# MPI version for testing

if (defined($ENV{'MPITEST_MPIVERSION'})) {

    $_ = $ENV{'MPITEST_MPIVERSION'};

    if (/(\d+)\.(\d+)/) {

        $MPIMajorVersion = $1;

        $MPIMinorVersion = $2;

    }

}

#---------------------------------------------------------------------------

# Process arguments and override any defaults

#---------------------------------------------------------------------------

foreach $_ (@ARGV) {

    if (/--?mpiexec=(.*)/) { 

	# Use mpiexec as given - it may be in the path, and 

	# we don't want to bother to try and find it.

	$mpiexec = $1; 

    }

    elsif (/--?mpiversion=(\d+)\.(\d+)/) { $MPIMajorVersion = $1;

                                           $MPIMinorVersion = $2; }

    elsif (/--?np=(\d+)/)   { $np_default = $1; }

    elsif (/--?maxnp=(\d+)/) { $np_max = $1; }

    elsif (/--?ppnarg=(.*)/) { $ppnArg = $1; }

    elsif (/--?ppn=(\d+)/)  { $ppnMax = $1; }

    elsif (/--?timelimitarg=(.*)/) { $timelimitArg = $1; }

    elsif (/--?tests=(.*)/) { $listfiles = $1; }

    elsif (/--?srcdir=(.*)/) { $srcdir = $1; }

    elsif (/--?verbose/) { $verbose = 1; }

    elsif (/--?showprogress/) { $showProgress = 1; }

    elsif (/--?debug/) { $debug = 1; }

    elsif (/--?batchdir=(.*)/) { $batrundir = $1; }

    elsif (/--?batch/) { $batchRun = 1; }

    elsif (/--?timeoutarg=(.*)/) { $timeoutArgPattern = $1; }

    elsif (/--?xmlfile=(.*)/) {

	$xmlfile   = $1;

	if (! ($xmlfile =~ /^\//)) {

	    $thisdir = `pwd`;

	    chop $thisdir;

	    $xmlfullfile = $thisdir . "/" . $xmlfile ;

	}

	else {

	    $xmlfullfile = $xmlfile;

	}

	$xmloutput = 1;

	open( XMLOUT, ">$xmlfile" ) || die "Cannot open $xmlfile\n";

	my $date = `date "+%Y-%m-%d-%H-%M"`;

	$date =~ s/\r?\n//;

	# MPISOURCE can be used to describe the source of MPI for this

	# test.

	print XMLOUT "$newline";

	print XMLOUT "$newline";

	print XMLOUT "<MPITESTRESULTS>$newline";

	print XMLOUT "<DATE>$date</DATE>$newline";

	print XMLOUT "<MPISOURCE>@MPI_SOURCE@</MPISOURCE>$newline";

    }

    elsif (/--?noxmlclose/) {

	$closeXMLOutput = 0;

    }

    elsif (/--?tapfile=(.*)/) {

        $tapfile = $1;

        if ($tapfile !~ m|^/|) {

            $thisdir = `pwd`;

            chomp $thisdir;

            $tapfullfile = $thisdir . "/" . $tapfile ;

        }

        else {

            $tapfullfile = $tapfile;

        }

        $tapoutput = 1;

        open( TAPOUT, ">$tapfile" ) || die "Cannot open $tapfile\n";

        my $date = `date "+%Y-%m-%d-%H-%M"`;

        $date =~ s/\r?\n//;

        print TAPOUT "TAP version 13\n";

        print TAPOUT "# MPICH test suite results (TAP format)\n";

        print TAPOUT "# date ${date}\n";

        # we do not know at this point how many tests will be run, so do

        # not print a test plan line like "1..450" until the very end

    }

    elsif (/--?junitfile=(.*)/) {

        $junitfile = $1;

        if ($junitfile !~ m|^/|) {

            $thisdir = `pwd`;

            chomp $thisdir;

            $junitfullfile = $thisdir . "/" . $junitfile ;

        }

        else {

            $junitfullfile = $junitfile;

        }

        $junitoutput = 1;

        open( JUNITOUT, ">$junitfile" ) || die "Cannot open $junitfile\n";

    }

    else {

	print STDERR "Unrecognized argument $_\n";

	print STDERR "runtests [-tests=testfile] [-np=nprocesses] \

        [-maxnp=max-nprocesses] [-srcdir=location-of-tests] \

        [-ppn=max-proc-per-node] [-ppnarg=string] \

        [-timelimitarg=string] [-mpiversion=major.minor] \

        [-xmlfile=filename ] [-tapfile=filename ] \

        [-junitfile=filename ] [-noxmlclose] \

        [-verbose] [-showprogress] [-debug] [-batch]\n";

	exit(1);

    }

}

# Perform any post argument processing

if ($batchRun) {

    if (! -d $batrundir) {

	mkpath $batrundir || die "Could not create $batrundir\n";

    }

    open( BATOUT, ">$batrundir/runtests.batch" ) || die "Could not open $batrundir/runtests.batch\n";

}

else {

    # We must have mpiexec

    if ("$mpiexec" eq "") {

	print STDERR "No mpiexec found!\n";

	exit(1);

    }

}

#

# Process any files

if ($listfiles eq "") {

    if ($batchRun) {

	print STDERR "An implicit list of tests is not permitted in batch mode. See README for more details\n";

	exit(1);

    } 

    else {

	&ProcessImplicitList;

    }

}

elsif (-d $listfiles) { 

    print STDERR "Testing by directories not yet supported\n";

}

else {

    &RunList( $listfiles );

}

if ($xmloutput && $closeXMLOutput) { 

    print XMLOUT "</MPITESTRESULTS>$newline";

    close XMLOUT; 

}

if ($tapoutput) {

    print TAPOUT "1..$total_seen\n";

    close TAPOUT;

}

if ($junitoutput) {

    print JUNITOUT "    <system-out></system-out>\n";

    print JUNITOUT "    <system-err></system-err>\n";

    print JUNITOUT "  </testsuite>\n";

    print JUNITOUT "</testsuites>\n";

    close JUNITOUT;

    # the second pass: insert the header

    # Note: the field "errors" is not used now, but reserved for future uses.

    open my $JUNITIN,  '<',  $junitfile      or die "Can't read old file: $!";

    open my $JUNITOUTNEW, '>', "$junitfile.new" or die "Can't write new file: $!";

    my $date = `date "+%Y-%m-%d-%H-%M"`;

    $date =~ s/\r?\n//;

    print $JUNITOUTNEW "<testsuites>\n";

    print $JUNITOUTNEW "  

    print $JUNITOUTNEW "             errors=\"0\"\n";

    print $JUNITOUTNEW "             skipped=\"$skip_count\"\n";

    print $JUNITOUTNEW "             tests=\"$total_run\"\n";

    print $JUNITOUTNEW "             date=\"${date}\"\n";

    print $JUNITOUTNEW "             name=\"summary_junit_xml\">\n";

    while( <$JUNITIN> ) {

        print $JUNITOUTNEW $_;

    }

    close $JUNITIN;

    close $JUNITOUTNEW;

    move("$junitfile.new","$junitfile");

}

# Output a summary:

if ($batchRun) {

    print "Programs created along with a runtest.batch file in $batrundir\n";

    print "Run that script and then use checktests to summarize the results\n";

}

else {

    if ($err_count) {

	print "$err_count tests failed out of $total_run\n";

	if ($xmloutput) {

	    print "Details in $xmlfullfile\n";

	}

    }

    else {

	print " All $total_run tests passed!\n";

    }

    if ($tapoutput) {

        print "TAP formatted results in $tapfullfile\n";

    }

    if ($junitoutput) {

        print "JUNIT formatted results in $junitfullfile\n";

    }

}

#

# ---------------------------------------------------------------------------

# Routines

# 

# Enter a new directory and process a list file.  

#  ProcessDir( directory-name, list-file-name )

sub ProcessDir {

    my $dir = $_[0]; $dir =~ s/\/$//;

    my $listfile = $_[1];

    my $savedir = `pwd`;

    my $savecurdir = $curdir;

    my $savesrcdir = $srcdir;

    chop $savedir;

    if (substr($srcdir,0,3) eq "../") {

      $srcdir = "../$srcdir";

    }

    print "Processing directory $dir\n" if ($verbose || $debug);

    chdir $dir;

    if ($dir =~ /\//) {

	print STDERR "only direct subdirectories allowed in list files";

    }

    $curdir .= "/$dir";

    &RunList( $listfile );

    print "\n" if $showProgress; # Terminate line from progress output

    chdir $savedir;

    $curdir = $savecurdir;

    $srcdir = $savesrcdir;

}

# ---------------------------------------------------------------------------

# Run the programs listed in the file given as the argument. 

# This file describes the tests in the format

#  programname number-of-processes [ key=value ... ]

# If the second value is not given, the default value is used.

# 

sub RunList { 

    my $LIST = "LIST$depth"; $depth++;

    my $listfile = $_[0];

    my $ResultTest = "";

    my $InitForRun = "";

    my $listfileSource = $listfile;

    my @TYPES = "";

    my $DTP_SWITCH = "@DTP_SWITCH@";

    if (! defined $ENV{"DTP_NUM_OBJS"}) {

        $ENV{"DTP_NUM_OBJS"} = "5";

    }

    if (! defined $ENV{"DTP_RUNTIME_TYPES"}) {

        @TYPES = split ' ', "MPI_INT MPI_DOUBLE";

    }

    else {

        @TYPES = split ' ', $ENV{"DTP_RUNTIME_TYPES"};

    }

    print "Looking in $curdir/$listfile\n" if $debug;

    if (! -s "$listfile" && -s "$srcdir/$curdir/$listfile" ) {

	$listfileSource = "$srcdir/$curdir/$listfile";

    }

    open( $LIST, "<$listfileSource" ) || 

	die "Could not open $listfileSource\n";

    while (<$LIST>) {

	# Check for stop file

	if (-s $stopfile) {

	    # Exit because we found a stopfile

	    print STDERR "Terminating test because stopfile $stopfile found\n";

	    last;

	}

	# Skip comments

	s/#.*//g;

	# Remove any trailing newlines/returns

	s/\r?\n//;

        # Remove any leading whitespace

        s/^\s*//;

	# Some tests require that support routines are built first

	# This is specified with !<dir>:<target>

	if (/^\s*\!([^:]*):(.*)/) {

	    # Hack: just execute in a subshell.  This discards any 

	    # output.

	    `cd $1 && make $2`;

	    next;

	}

	# List file entries have the form:

	# program [ np [ name=value ... ] ]

	# See files errhan/testlist, init/testlist, and spawn/testlist

	# for examples of using the key=value form

	my @args = split(/\s+/,$_);

	my $programname = $args[0];

	my $np = "";

	my $ResultTest = "";

	my $InitForRun = "";

	my $timeLimit  = "";

	my $progArgs   = "";

	my $mpiexecArgs = "";

	my $requiresStrict = "";

	my $requiresMPIX   = "";

	my $progEnv    = "";

	my $mpiVersion = "";

        my $xfail = "";

    my $dtp_match_type = 0;

    my $dtp_test_type = "";    # this test type

    # get the type of this test

    if ($programname =~ /BASIC/) {

        $dtp_test_type = "BASIC";

    } elsif ($programname =~ /STRUCT/) {

        $dtp_test_type = "STRUCT";

    }

	if ($#args >= 1) { $np = $args[1]; }

	# Process the key=value arguments

	for (my $i=2; $i <= $#args; $i++) {

	    if ($args[$i] =~ /([^=]+)=(.*)/) {

		my $key = $1;

		my $value = $2;

		if ($key eq "resultTest") {

		    $ResultTest = $value;

		}

		elsif ($key eq "init") {

		    $InitForRun = $value;

		}

		elsif ($key eq "timeLimit") {

		    $timeLimit = $value;

		}

		elsif ($key eq "arg") {

            # match allowed datatypes here

            for my $j (@TYPES) {

                if ($value eq "-type=$j") {

                    $dtp_match_type=1;

                    break;

                }

            }

            $progArgs = "$progArgs $value";

        }

		elsif ($key eq "mpiexecarg") {

		    $mpiexecArgs = "$mpiexecArgs $value";

		}

		elsif ($key eq "env") {

		    if ($progEnv eq "") {

			$progEnv = "$value";

		    }

		    else {

			$progEnv = "$progEnv $value";

		    }

		}

		elsif ($key eq "mpiversion") {

		    $mpiVersion = $value;

		}

		elsif ($key eq "strict") {

		    $requiresStrict = $value

		}

		elsif ($key eq "mpix") {

		    $requiresMPIX = $value

		}

                elsif ($key eq "xfail") {

                    if ($value eq "") {

                        print STDERR "\"xfail=\" requires an argument\n";

                    }

                    $xfail = $value;

                }

		else {

		    print STDERR "Unrecognized key $key in $listfileSource\n";

		}

	    }

	}

    if ($DTP_SWITCH eq "ON") {

        # if dtpools tests are ON only run matching types for basic tests

        if ($dtp_test_type eq "BASIC" && $dtp_match_type == 0) {

            $programname = "";

        }

    } else {

        # if dtpools tests are OFF only disable dtpools tests, run the rest

        if ($dtp_test_type eq "BASIC" || $dtp_test_type eq "STRUCT") {

            $programname = "";

        }

    }

	# skip empty lines

	if ($programname eq "") { next; }

	if ($np eq "") { $np = $np_default; }

	if ($np_max > 0 && $np > $np_max) { $np = $np_max; }

        # allows us to accurately output TAP test numbers without disturbing the

        # original totals that have traditionally been reported

        #

        # These "unless" blocks are ugly, but permit us to honor skipping

        # criteria for directories as well without counting directories as tests

        # in our XML/TAP output.

        unless (-d $programname) {

            $total_seen++;

        }

	# If a minimum MPI version is specified, check against the

	# available MPI.  If the version is unknown, we ignore this

	# test (thus, all tests will be run).  

	if ($mpiVersion ne "" && $MPIMajorVersion ne "unknown" &&

	    $MPIMinorVersion ne "unknown") {

	    my ($majorReq,$minorReq) = split(/\./,$mpiVersion);

            if ($majorReq > $MPIMajorVersion or

                ($majorReq == $MPIMajorVersion && $minorReq > $MPIMinorVersion))

            {

                unless (-d $programname) {

                    SkippedTest($programname, $np, $progArgs, $progEnv, $curdir, "requires MPI version $mpiVersion");

                }

                next;

            }

	}

	# Check whether strict is required by MPI but not by the

	# test (use strict=false for tests that use non-standard extensions)

        if (lc($requiresStrict) eq "false" && lc($testIsStrict) eq "true") {

            unless (-d $programname) {

                SkippedTest($programname, $np, $progArgs, $progEnv, $curdir, "non-strict test, strict MPI mode requested");

            }

            next;

        }

        if (lc($testIsStrict) eq "true") {

            # Strict MPI testing was requested, so assume that a non-MPICH MPI

            # implementation is being tested and the "xfail" implementation

            # assumptions do not hold.

            $xfail = '';

        }

	if ($xfail ne '' && $runxfail eq "false") {

	    # Skip xfail tests if they are not configured. Strict MPI tests that are

	    # marked xfail will still run with --enable-strictmpi.

            unless (-d $programname) {

                SkippedTest($programname, $np, $progArgs, $progEnv, $curdir, "xfail tests disabled");

	    }

	    next;

	}

        if (lc($requiresMPIX) eq "true" && lc($MPIHasMPIX) eq "no") {

            unless (-d $programname) {

                SkippedTest($programname, $np, $progArgs, $progEnv, $curdir, "tests MPIX extensions, MPIX testing disabled");

            }

            next;

        }

	if (-d $programname) {

	    # If a directory, go into the that directory and 

	    # look for a new list file

	    &ProcessDir( $programname, $listfile );

	}

	else {

	    $total_run++;

	    if (&BuildMPIProgram( $programname, $xfail ) == 0) {

		if ($batchRun == 1) {

		    &AddMPIProgram( $programname, $np, $ResultTest, 

				    $InitForRun, $timeLimit, $progArgs,

				    $progEnv, $mpiexecArgs, $xfail );

		}

		else {

		    &RunMPIProgram( $programname, $np, $ResultTest, 

				    $InitForRun, $timeLimit, $progArgs, 

				    $progEnv, $mpiexecArgs, $xfail );

		}

	    }

	    elsif ($xfail eq '') {

		# We expected to run this program, so failure to build

		# is an error

		$found_error = 1;

		$err_count++;

	    }

	    if ($batchRun == 0) {

		&CleanUpAfterRun( $programname );

	    }

	}

    }

    close( $LIST );

}

#

# This routine tries to run all of the files in the current

# directory

sub ProcessImplicitList {

    # The default is to run every file in the current directory.

    # If there are no built programs, build and run every file

    # WARNING: This assumes that anything executable should be run as

    # an MPI test.

    $found_exec = 0;

    $found_src  = 0;

    open (PGMS, "ls -1 |" ) || die "Cannot list directory\n";

    while (<PGMS>) {

	s/\r?\n//;

	$programname = $_;

	if (-d $programname) { next; }  # Ignore directories

	if ($programname eq "runtests") { next; } # Ignore self

	if ($programname eq "checktests") { next; } # Ignore helper

	if ($programname eq "configure") { next; } # Ignore configure script

	if ($programname eq "config.status") { next; } # Ignore configure helper

	if (-x $programname) { $found_exec++; }

	if ($programname =~ /\.[cf]$/) { $found_src++; } 

    }

    close PGMS;

    if ($found_exec) {

	print "Found executables\n" if $debug;

	open (PGMS, "ls -1 |" ) || die "Cannot list programs\n";

	while (<PGMS>) {

	    # Check for stop file

	    if (-s $stopfile) {

		# Exit because we found a stopfile

		print STDERR "Terminating test because stopfile $stopfile found\n";

		last;

	    }

	    s/\r?\n//;

	    $programname = $_;

	    if (-d $programname) { next; }  # Ignore directories

	    if ($programname eq "runtests") { next; } # Ignore self

	    if (-x $programname) {

		$total_run++;

		&RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" );

	    }

	}

	close PGMS;

    }

    elsif ($found_src) { 

	print "Found source files\n" if $debug;

	open (PGMS, "ls -1 *.c |" ) || die "Cannot list programs\n";

	while (<PGMS>) {

	    if (-s $stopfile) {

		# Exit because we found a stopfile

		print STDERR "Terminating test because stopfile $stopfile found\n";

		last;

	    }

	    s/\r?\n//;

	    $programname = $_;

	    # Skip messages from ls about no files

	    if (! -s $programname) { next; }

	    $programname =~ s/\.c//;

	    $total_run++;

	    if (&BuildMPIProgram( $programname, "") == 0) {

		&RunMPIProgram( $programname, $np_default, "", "", "", "", "", "", "" );

	    }

	    else {

		# We expected to run this program, so failure to build

		# is an error

		$found_error = 1;

		$err_count++;

	    }

	    &CleanUpAfterRun( $programname );

	}

	close PGMS;

    }

}

# Run the program.  

# ToDo: Add a way to limit the time that any particular program may run.

# The arguments are

#    name of program, number of processes, name of routine to check results

#    init for testing, timelimit, and any additional program arguments

# If the 3rd arg is not present, the a default that simply checks that the

# return status is 0 and that the output is " No Errors" is used.

sub RunMPIProgram {

    my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs,$xfail) = @_;

    my $found_error   = 0;

    my $found_noerror = 0;

    my $inline = "";

    my $extraArgs = "";

    my $runtime = 0;

    &RunPreMsg( $programname, $np, $curdir );

    unlink "err";

    # Set a default timeout on tests (3 minutes for now)

    my $timeout = $defaultTimeLimit;

    if (defined($timeLimit) && $timeLimit =~ /^\d+$/) {

	$timeout = $timeLimit;

    }

    $timeout *= $defaultTimeLimitMultiplier;

    $ENV{"MPIEXEC_TIMEOUT"} = $timeout;

    # Handle the ppn (processes per node) option.

    $ppnargs = "";

    if ($ppnArg ne "" && $ppnMax > 0) {

	$ppnargs = $ppnArg;

	$nn = $ppnMax;

	# Some systems require setting the number of processes per node

	# no greater than the total number of processes (e.g., aprun on Cray)

	if ($nn > $np) { $nn = $np; }

	$ppnargs =~ s/\%d/$nn/;

	$extraArgs .= " " . $ppnargs;

    }

    # Handle the timelimit option.

    if ($timelimitArg ne "" && $timeout> 0) {

        $tlargs = "";

	$tlargs = $timelimitArg;

	$tlargs =~ s/\%d/$timeout/;

	$extraArgs .= " " . $tlargs;

    }

    # Run the optional setup routine. For example, the timeout tests could

    # be set to a shorter timeout.

    if ($InitForTest ne "") {

	&$InitForTest();

    }

    print STDOUT "Env includes $progEnv\n" if $verbose;

    print STDOUT "$mpiexec $np_arg $np $extraArgs $mpiexecArgs $program_wrapper ./$programname $progArgs\n" if $verbose;

    print STDOUT "." if $showProgress;

    # Save and restore the environment if necessary before running mpiexec.

    if ($progEnv ne "") {

	%saveEnv = %ENV;

	foreach $val (split(/\s+/, $progEnv)) {

	    if ($val =~ /([^=]+)=(.*)/) {

		$ENV{$1} = $2;

	    }

	    else {

		print STDERR "Environment variable/value $val not in a=b form\n";

	    }

	}

    }

    my $start_time = gettimeofday();

    open ( MPIOUT, "$mpiexec $np_arg $np $extraArgs $mpiexecArgs $program_wrapper ./$programname $progArgs 2>&1 |" ) ||

	die "Could not run ./$programname\n";

    if ($progEnv ne "") {

	%ENV = %saveEnv;

    }

    if ($ResultTest ne "") {

	# Read and process the output

	($found_error, $inline) = &$ResultTest( MPIOUT, $programname );

    }

    else {

	if ($verbose) {

	    $inline = "$mpiexec $np_arg $np $extraArgs $mpiexecArgs $program_wrapper ./$programname\n";

	}

	else {

	    $inline = "";

	}

	while (<MPIOUT>) {

	    print STDOUT $_ if $verbose;

	    # Skip FORTRAN STOP

	    if (/FORTRAN STOP/) { next; }

	    $inline .= $_;

	    if (/^\s*No [Ee]rrors\s*$/ && $found_noerror == 0) {

		$found_noerror = 1;

	    }

	    if (! /^\s*No [Ee]rrors\s*$/ && !/^\s*Test Passed\s*$/ && !/requesting checkpoint\s*$/ && !/checkpoint completed\s*$/) {

		print STDERR "Unexpected output in $programname: $_";

		if (!$found_error) {

		    $found_error = 1;

		    $err_count ++;

		}

	    }

	}

	if ($found_noerror == 0) {

	    print STDERR "Program $programname exited without No Errors\n";

	    if (!$found_error) {

		$found_error = 1;

		$err_count ++;

	    }

	}

	$rc = close ( MPIOUT );

	my $end_time = gettimeofday();

	$runtime = $end_time - $start_time;

	print STDOUT "Runtime: $runtime\n" if $verbose;

	if ($rc == 0) {

	    # Only generate a message if we think that the program

	    # passed the test.

	    if (!$found_error) {

		$run_status = $?;

		$signal_num = $run_status & 127;

		if ($run_status > 255) { $run_status >>= 8; }

		print STDERR "Program $programname exited with non-zero status $run_status\n";

		if ($signal_num != 0) {

		    print STDERR "Program $programname exited with signal $signal_num\n";

		}

		$found_error = 1;

		$err_count ++;

	    }

	}

    }

    if ($found_error) {

        &RunTestFailed( $programname, $np, $progArgs, $progEnv, $timeout, $curdir, $inline, $xfail, $runtime );

    }

    else { 

        &RunTestPassed( $programname, $np, $progArgs, $progEnv, $curdir, $xfail, $runtime );

    }

    &RunPostMsg( $programname, $np, $curdir );

}

# This version simply writes the mpiexec command out, with the output going

# into a file, and recording the output status of the run.

sub AddMPIProgram {

    my ($programname,$np,$ResultTest,$InitForTest,$timeLimit,$progArgs,$progEnv,$mpiexecArgs, $xfail) = @_;

    if (! -x $programname) {

	print STDERR "Could not find $programname!";

	return;

    }

    if ($ResultTest ne "") {

	# This test really needs to be run manually, with this test

	# Eventually, we can update this to include handleing in checktests.

	print STDERR "Run $curdir/$programname with $np processes and use $ResultTest to check the results\n";

	return;

    }

    # Set a default timeout on tests (3 minutes for now)

    my $timeout = $defaultTimeLimit;

    if (defined($timeLimit) && $timeLimit =~ /^\d+$/) {

	# On some systems, there is no effective time limit on