|
Packit |
fcad23 |
#!/usr/bin/env perl
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
use Getopt::Long;
|
|
Packit |
fcad23 |
#use Data::Dumper;
|
|
Packit |
fcad23 |
use File::Basename;
|
|
Packit |
fcad23 |
use File::Spec;
|
|
Packit |
fcad23 |
use Cwd qw(abs_path);
|
|
Packit |
fcad23 |
use strict;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# Globals and Command Line options
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my %opts = ('groups' => 'default',
|
|
Packit |
fcad23 |
'master-directory' => 'fulltests',
|
|
Packit |
fcad23 |
'srcdir' => dirname("$0") . "/..",
|
|
Packit |
fcad23 |
'builddir' => '..',
|
|
Packit |
fcad23 |
'failed-file' => 'failed_tests',
|
|
Packit |
fcad23 |
);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Getopt::Long::Configure(qw(no_ignore_case));
|
|
Packit |
fcad23 |
GetOptions(\%opts,
|
|
Packit |
fcad23 |
"verbose",
|
|
Packit |
fcad23 |
"help|?",
|
|
Packit |
fcad23 |
"quiet|q",
|
|
Packit |
fcad23 |
"groups|g=s",
|
|
Packit |
fcad23 |
"r=s",
|
|
Packit |
fcad23 |
"debug",
|
|
Packit |
fcad23 |
"srcdir|D=s",
|
|
Packit |
fcad23 |
"builddir|d=s",
|
|
Packit |
fcad23 |
"f",
|
|
Packit |
fcad23 |
"F",
|
|
Packit |
fcad23 |
"failed-file=s",
|
|
Packit |
fcad23 |
"master-directory=s",
|
|
Packit |
fcad23 |
) || ++$opts{'help'};
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# Change srcdir and builddir to absolute paths
|
|
Packit |
fcad23 |
$opts{'srcdir'} = abs_path($opts{'srcdir'});
|
|
Packit |
fcad23 |
$opts{'builddir'} = abs_path($opts{'builddir'});
|
|
Packit |
fcad23 |
# Set exeext.
|
|
Packit |
fcad23 |
$opts{'exeext'} = join(readpipe($opts{'builddir'} . '/net-snmp-config --exeext'));
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
usage() if ($opts{'help'});
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# Build the harness object
|
|
Packit |
fcad23 |
my %args = (
|
|
Packit |
fcad23 |
verbosity => ($opts{'verbose'} ? 1 : ($opts{'quiet'} ? -1 : 0)),
|
|
Packit |
fcad23 |
exec => \&decide_exec,
|
|
Packit |
fcad23 |
# this option is *really* weird in how it works
|
|
Packit |
fcad23 |
failures => ($opts{'quiet'} ? 0 : ($opts{'verbose'} ? 0 : 1)),
|
|
Packit |
fcad23 |
errors => ($opts{'quiet'} ? 0 : 1),
|
|
Packit |
fcad23 |
);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# list of support infrastructure components
|
|
Packit |
fcad23 |
my %support;
|
|
Packit |
fcad23 |
my %sources;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# if the -d option was specified, pass on the source root directory to all apps
|
|
Packit |
fcad23 |
if (exists($opts{'master-directory'})) {
|
|
Packit |
fcad23 |
$ENV{'NETSNMPSRCDIR'} = $opts{'master-directory'};
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
$ENV{'NETSNMPSRCDIR'} = '.';
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# pass srcdir and builddir to all apps
|
|
Packit |
fcad23 |
$ENV{'srcdir'} = $opts{'srcdir'};
|
|
Packit |
fcad23 |
$ENV{'builddir'} = $opts{'builddir'};
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# set up MIBDIRS to refer to the src directory
|
|
Packit |
fcad23 |
if (!$ENV{'MIBDIRS'}) {
|
|
Packit |
fcad23 |
$ENV{'MIBDIRS'} = "$opts{srcdir}/mibs";
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $cfp_path = File::Spec->catfile(dirname(abs_path($0)), "check_for_pskill");
|
|
Packit |
fcad23 |
my $rc = `$cfp_path`;
|
|
Packit |
fcad23 |
if ($rc != 0) {
|
|
Packit |
fcad23 |
die "$cfp_path failed with error code $rc\n";
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# Protection measures
|
|
Packit |
fcad23 |
$ENV{'SNMPCONFPATH'} = "/dev/null";
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# create the testing harness infrastructure
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $harness;
|
|
Packit |
fcad23 |
if (eval { require TAP::Harness; } ) {
|
|
Packit |
fcad23 |
import TAP::Harness;
|
|
Packit |
fcad23 |
$harness = TAP::Harness->new(\%args);
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
require Test::Harness;
|
|
Packit |
fcad23 |
import Test::Harness;
|
|
Packit |
fcad23 |
if ($opts{'groups'} ne 'default') {
|
|
Packit |
fcad23 |
print STDERR "
|
|
Packit |
fcad23 |
ERROR: I can not find the perl TAP::Harness module. We support the
|
|
Packit |
fcad23 |
more common Test::Harness module but only for the default test group.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Either:
|
|
Packit |
fcad23 |
1) run only the default tests (i.e., just \"make test\")
|
|
Packit |
fcad23 |
2) install the TAP::Harness perl module
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
";
|
|
Packit |
fcad23 |
exit 1;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
DIE("Test::Harness must be of version 1.21 or newer\n" .
|
|
Packit |
fcad23 |
"Install the TAP::Harness module or use the RUNTESTS script\n")
|
|
Packit |
fcad23 |
if $Test::Harness::VERSION < 1.21;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# gather the tests
|
|
Packit |
fcad23 |
my @tests;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
DEBUG("Gathering and building tests:\n");
|
|
Packit |
fcad23 |
find_support();
|
|
Packit |
fcad23 |
if ($opts{'f'}) {
|
|
Packit |
fcad23 |
DIE("The -f and -g options can not be both specified\n")
|
|
Packit |
fcad23 |
if ($opts{'groups'} ne 'default');
|
|
Packit |
fcad23 |
DIE("The -f and -r options can not be both specified\n") if ($opts{'r'});
|
|
Packit |
fcad23 |
DIE("No $opts{'failed-file'} file was found to read failed state from\n")
|
|
Packit |
fcad23 |
if (! -f $opts{'failed-file'});
|
|
Packit |
fcad23 |
open(F, $opts{'failed-file'});
|
|
Packit |
fcad23 |
while (<F>) {
|
|
Packit |
fcad23 |
chomp;
|
|
Packit |
fcad23 |
push @tests, build_test($_);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
@tests = gather_tests($opts{'groups'}, $opts{'r'});
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# rename them to remove parent dirs
|
|
Packit |
fcad23 |
@tests = rename_tests(@tests);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# run the tests
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
DEBUG("Running tests:\n");
|
|
Packit |
fcad23 |
DEBUG("-" x 78, "\n");
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $results;
|
|
Packit |
fcad23 |
if ($harness) {
|
|
Packit |
fcad23 |
$results = $harness->runtests(@tests);
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
# minimal backwards compat with Test::Harness
|
|
Packit |
fcad23 |
run_older_perl_tests(@tests);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my @failed = $results->failed();
|
|
Packit |
fcad23 |
if (!$opts{'q'} && $#failed > -1) {
|
|
Packit |
fcad23 |
print "\nWe failed these ", (1 + $#failed), " tests:\n";
|
|
Packit |
fcad23 |
my @lines = @failed;
|
|
Packit |
fcad23 |
map { if (exists($sources{$_})) { $_ = "$_ ( $sources{$_} )"} } @lines;
|
|
Packit |
fcad23 |
print " ", join("\n ",@lines), "\n";
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
if (!$opts{'F'}) {
|
|
Packit |
fcad23 |
open(F,">$opts{'failed-file'}");
|
|
Packit |
fcad23 |
if ($#failed > -1) {
|
|
Packit |
fcad23 |
print F join("\n", get_sources(@failed)) . "\n";
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
close(F);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
exit($results->all_passed() ? 0 : 1);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
######################################################################
|
|
Packit |
fcad23 |
# Infrastructure
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
########################################
|
|
Packit |
fcad23 |
# decides how we should execute a test
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
sub decide_exec {
|
|
Packit |
fcad23 |
my ( $harness, $testfile ) = @_;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 1) Parse the $testfile argument.
|
|
Packit |
fcad23 |
my ($dirname, $groupname, $basename, $app_extension, $file_extension) =
|
|
Packit |
fcad23 |
($testfile =~ /([^\/]+)\/([^\/]+)\/([^\/]+)_([^\/_]+)\.*([^\/\.]*)$/);
|
|
Packit |
fcad23 |
$app_extension =~ s/$opts{'exeext'}\$//;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 2) we have a RUN_TYPE file in the same directory
|
|
Packit |
fcad23 |
if (exists($support{'run'}{$app_extension}{$groupname})) {
|
|
Packit |
fcad23 |
return [$support{'run'}{$app_extension}{$groupname}, $testfile];
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 3) return a generic run script
|
|
Packit |
fcad23 |
if (exists($support{'run'}{$app_extension}{'generic'})) {
|
|
Packit |
fcad23 |
return [$support{'run'}{$app_extension}{'generic'}, $testfile];
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 4) give up and let the test harness decide itself
|
|
Packit |
fcad23 |
return undef;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub gather_tests {
|
|
Packit |
fcad23 |
my ( $groups, $regexp ) = @_;
|
|
Packit |
fcad23 |
my @groups;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# figure out the list of groups we need to search through
|
|
Packit |
fcad23 |
if ($groups eq 'all') {
|
|
Packit |
fcad23 |
# find every group we can
|
|
Packit |
fcad23 |
# we exclude:
|
|
Packit |
fcad23 |
# - things not a directory
|
|
Packit |
fcad23 |
# - anything with "template" in the name
|
|
Packit |
fcad23 |
@groups =
|
|
Packit |
fcad23 |
grep { !/(template|support)/ &&
|
|
Packit |
fcad23 |
-d $_ && s/$opts{'srcdir'}\/testing\/$opts{'master-directory'}.// } glob("$opts{'srcdir'}/testing/$opts{'master-directory'}/*");
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
# they specified a comma separated list
|
|
Packit |
fcad23 |
@groups = split(/,\s*/, $groups);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
DEBUG("Checking groups: ", join(", ", @groups), "\n");
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my @tests;
|
|
Packit |
fcad23 |
foreach my $group (@groups) {
|
|
Packit |
fcad23 |
my @files;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
DEBUG("checking group $group\n");
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
if (! -d "$opts{'srcdir'}/testing/$opts{'master-directory'}/$group") {
|
|
Packit |
fcad23 |
ERROR("group '$group' is not a directory under '$opts{'srcdir'}/testing/$opts{'master-directory'}'; ignoring\n");
|
|
Packit |
fcad23 |
next;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# push on all files that start with T[NUM]*
|
|
Packit |
fcad23 |
push_or_skip(\@tests, $regexp, glob("$opts{'srcdir'}/testing/$opts{'master-directory'}/$group/T[0-9]*"));
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
return @tests;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub push_or_skip {
|
|
Packit |
fcad23 |
my ($array, $regexp, @files) = @_;
|
|
Packit |
fcad23 |
foreach my $file (@files) {
|
|
Packit |
fcad23 |
next if ($file =~ /.(bak|old|orig|rej)$/);
|
|
Packit |
fcad23 |
next if ($file =~ /~$/);
|
|
Packit |
fcad23 |
next if (defined($regexp) && $file !~ /$regexp/i);
|
|
Packit |
fcad23 |
DEBUG(" Adding file $file\n");
|
|
Packit |
fcad23 |
push @$array, build_test($file);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# rename all the tests to remove the top subdir to help readability
|
|
Packit |
fcad23 |
sub rename_tests {
|
|
Packit |
fcad23 |
my (@tests) = @_;
|
|
Packit |
fcad23 |
my @newtests;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# yes, I could have used map. But I didn't.
|
|
Packit |
fcad23 |
foreach my $file (@tests) {
|
|
Packit |
fcad23 |
my $title = "$file";
|
|
Packit |
fcad23 |
my $foundheader = 0;
|
|
Packit |
fcad23 |
$title = $sources{$file} if (exists($sources{$file}));
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
open(SRC, $title);
|
|
Packit |
fcad23 |
while (<SRC>) {
|
|
Packit |
fcad23 |
if (/(HEADER|TITLE)\s+['"]*(.*)/) {
|
|
Packit |
fcad23 |
$title = $2;
|
|
Packit |
fcad23 |
$title =~ s/\s*\*\/.*//;
|
|
Packit |
fcad23 |
$title =~ s/['"]$//;
|
|
Packit |
fcad23 |
$foundheader = 1;
|
|
Packit |
fcad23 |
last;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
close(SRC);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
if (! $foundheader) {
|
|
Packit |
fcad23 |
$title =~ s/^$opts{'srcdir'}\/testing\///;
|
|
Packit |
fcad23 |
$title =~ s/$opts{'master-directory'}.//;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
$sources{$title} = $sources{$file} || $file;
|
|
Packit |
fcad23 |
push @newtests, [$file, $title];
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
return @newtests;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# called to possibly manipulate the list of tests to run by building some
|
|
Packit |
fcad23 |
sub build_tests {
|
|
Packit |
fcad23 |
my (@tests) = @_;
|
|
Packit |
fcad23 |
my @newtests;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
foreach my $test (@tests) {
|
|
Packit |
fcad23 |
my $title;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $built = build_test($test);
|
|
Packit |
fcad23 |
if (ref($built) eq 'ARRAY') {
|
|
Packit |
fcad23 |
push @newtests, @$built;
|
|
Packit |
fcad23 |
} elsif ($built ne "") {
|
|
Packit |
fcad23 |
push @newtests, $built;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
return @newtests;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
# Finds scripts that are used to build and run actual commands
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
sub find_builders {
|
|
Packit |
fcad23 |
$support{'build'} = {};
|
|
Packit |
fcad23 |
find_scripts('build', $support{'build'});
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub find_runners {
|
|
Packit |
fcad23 |
$support{'run'} = {};
|
|
Packit |
fcad23 |
find_scripts('run', $support{'run'});
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub find_support {
|
|
Packit |
fcad23 |
find_builders();
|
|
Packit |
fcad23 |
find_runners();
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub find_scripts {
|
|
Packit |
fcad23 |
my ($buildname, $hashref) = @_;
|
|
Packit |
fcad23 |
my $count;
|
|
Packit |
fcad23 |
DEBUG("looking for $buildname scripts\n");
|
|
Packit |
fcad23 |
foreach my $builder (glob("$opts{'srcdir'}/testing/$opts{'master-directory'}/*/*_${buildname}")) {
|
|
Packit |
fcad23 |
next if ($builder =~ /~$/);
|
|
Packit |
fcad23 |
next if ($builder =~ /.(bak|orig|rej|old)$/);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my ($group, $type) = ($builder =~ /([^\/]+)\/([^\/]*)_${buildname}/);
|
|
Packit |
fcad23 |
# save this as a certain group builder
|
|
Packit |
fcad23 |
$hashref->{$type}{$group} = $builder;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# save this as a generic builder if there isn't a better
|
|
Packit |
fcad23 |
# generic one, such as one that exists in the support
|
|
Packit |
fcad23 |
# directory.
|
|
Packit |
fcad23 |
if (!exists($hashref->{$type}{'generic'}) || $group eq 'support') {
|
|
Packit |
fcad23 |
$hashref->{$type}{'generic'} = $builder;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
$count++;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
DEBUG(" found $count\n");
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# called to build a test from a registerd builder
|
|
Packit |
fcad23 |
sub build_test {
|
|
Packit |
fcad23 |
my ($testfile) = @_;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my ($dirname, $groupname, $basename, $app_extension, $file_extension) =
|
|
Packit |
fcad23 |
($testfile =~ /([^\/]+)\/([^\/]+)\/([^\/]+)_([^\/_]+)\.([^\/\.]+)$/);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# is this even a buildable type recipe?
|
|
Packit |
fcad23 |
if (!$dirname || !$basename || !$app_extension || !$file_extension) {
|
|
Packit |
fcad23 |
return $testfile;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
DEBUG("found: $testfile => $dirname, $basename, $app_extension, $file_extension\n");
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# choices:
|
|
Packit |
fcad23 |
# 1) we have a registered subroutine to build an extension from
|
|
Packit |
fcad23 |
# XXX
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 2) we have a BUILD_TYPE file in the same directory
|
|
Packit |
fcad23 |
if (exists($support{'build'}{$app_extension}{$dirname})) {
|
|
Packit |
fcad23 |
return
|
|
Packit |
fcad23 |
call_build_script($support{'build'}{$app_extension}{$dirname}, $testfile);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 3) return a generic build script
|
|
Packit |
fcad23 |
if (exists($support{'build'}{$app_extension}{'generic'})) {
|
|
Packit |
fcad23 |
return
|
|
Packit |
fcad23 |
call_build_script($support{'build'}{$app_extension}{'generic'}, $testfile);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# 4) we assume it's fine as is
|
|
Packit |
fcad23 |
return $testfile;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub call_build_script {
|
|
Packit |
fcad23 |
my ($scriptname, $filename) = @_;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $maybenewfile = $filename;
|
|
Packit |
fcad23 |
$maybenewfile =~ s/.[^\.]+$/$opts{'exeext'}/;
|
|
Packit |
fcad23 |
$maybenewfile =~ s/T([^\/]+)$/B$1/; # change prefix to B for 'Built'
|
|
Packit |
fcad23 |
$maybenewfile =~ s/^$opts{'srcdir'}\///;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $newpath = $maybenewfile;
|
|
Packit |
fcad23 |
$newpath =~ s/\/[^\/]*$//;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
if (! -d $newpath) {
|
|
Packit |
fcad23 |
DEBUG("making directory $newpath\n");
|
|
Packit |
fcad23 |
system("$opts{'srcdir'}/mkinstalldirs $newpath");
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
my $lastline;
|
|
Packit |
fcad23 |
DEBUG("BUILDING: $scriptname $filename $maybenewfile\n");
|
|
Packit |
fcad23 |
open(B,"$scriptname $filename $maybenewfile|");
|
|
Packit |
fcad23 |
while () {
|
|
Packit |
fcad23 |
$lastline = $_;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
chomp($lastline);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
DEBUG(" result: $lastline\n");
|
|
Packit |
fcad23 |
return undef if ($lastline eq 'fail');
|
|
Packit |
fcad23 |
return undef if ($lastline eq 'skip');
|
|
Packit |
fcad23 |
return $filename if ($lastline eq '');
|
|
Packit |
fcad23 |
$sources{$lastline} = $filename; # remember where we came from
|
|
Packit |
fcad23 |
return $lastline;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub get_sources {
|
|
Packit |
fcad23 |
my (@names) = @_;
|
|
Packit |
fcad23 |
map { if (exists($sources{$_})) { $_ = $sources{$_} } } @names;
|
|
Packit |
fcad23 |
return @names;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub run_older_perl_tests {
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
# Older versions of perl used a different test suite called Test::Harness
|
|
Packit |
fcad23 |
# It is much more limited than TAP::Harness;
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
# Here we massage our older tests into something that will work under
|
|
Packit |
fcad23 |
# Test::Harness too.
|
|
Packit |
fcad23 |
#
|
|
Packit |
fcad23 |
my @tests = @_;
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# create the temporary files
|
|
Packit |
fcad23 |
my @tempfiles;
|
|
Packit |
fcad23 |
if (! -d "$opts{'master-directory'}") {
|
|
Packit |
fcad23 |
mkdir("$opts{'master-directory'}", 0777);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
if (! -d "$opts{'master-directory'}/temptests") {
|
|
Packit |
fcad23 |
mkdir("$opts{'master-directory'}/temptests", 0777);
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
foreach my $test (@tests) {
|
|
Packit |
fcad23 |
my $tempfile = "$test->[0].t";
|
|
Packit |
fcad23 |
$tempfile =~ s#^$opts{'srcdir'}#$opts{'builddir'}#;
|
|
Packit |
fcad23 |
$tempfile =~ s#$opts{'master-directory'}/default/#$opts{'master-directory'}/temptests/#;
|
|
Packit |
fcad23 |
open(T, ">$tempfile") || die("$tempfile: $!");
|
|
Packit |
fcad23 |
print T "# functionally perl\n\nsystem(\"$opts{'srcdir'}/testing/fulltests/support/simple_run $test->[0]\");\n";
|
|
Packit |
fcad23 |
close(T);
|
|
Packit |
fcad23 |
chmod(0755, $tempfile);
|
|
Packit |
fcad23 |
push @tempfiles, $tempfile;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
$results = runtests(@tempfiles);
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
unlink(@tempfiles) || die("$@ $!");
|
|
Packit |
fcad23 |
exit;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# usage output
|
|
Packit |
fcad23 |
sub usage {
|
|
Packit |
fcad23 |
print "$0 [OPTIONS]\n";
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
print "\nOPTIONS:\n";
|
|
Packit |
fcad23 |
print " -v\t\t\tRuns in verbose mode; dumping all test output\n";
|
|
Packit |
fcad23 |
print " --verbose\n";
|
|
Packit |
fcad23 |
print " -q\t\t\tRuns in quieter mode; dumps less test output\n";
|
|
Packit |
fcad23 |
print " --quiet\n";
|
|
Packit |
fcad23 |
print " -g GROUP\t\tRuns the group of specified tests (default = 'default')\n";
|
|
Packit |
fcad23 |
print " --group GROUP\n";
|
|
Packit |
fcad23 |
print "\t\t\t(use 'all' to run all tests)\n";
|
|
Packit |
fcad23 |
print " -r REGEXP\t\tOnly run test files matching this regexp\n";
|
|
Packit |
fcad23 |
print " -f\t\t\tRun only the failed tests from the last run\n";
|
|
Packit |
fcad23 |
print " -F\t\t\tDon't create the failed_tests file\n";
|
|
Packit |
fcad23 |
print " --failed-file FILE\tThe location of the failed state file\n";
|
|
Packit |
fcad23 |
print " -D PATH\t\tSource directory\n";
|
|
Packit |
fcad23 |
print " --srcdir PATH\n";
|
|
Packit |
fcad23 |
print " (currently '$opts{'srcdir'}')\n";
|
|
Packit |
fcad23 |
print " -d PATH\t\tBuild directory to be tested\n";
|
|
Packit |
fcad23 |
print " --builddir PATH\n";
|
|
Packit |
fcad23 |
print " (currently '$opts{'builddir'}')\n";
|
|
Packit |
fcad23 |
print " --master-directory DIRNAME\n";
|
|
Packit |
fcad23 |
print " (default = 'fulltests')\n";
|
|
Packit |
fcad23 |
print " -h\t\t\tThis help output\n";
|
|
Packit |
fcad23 |
print " --help\n";
|
|
Packit |
fcad23 |
print " --debug\t\tDebugging output\n\n";
|
|
Packit |
fcad23 |
exit;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub DEBUG {
|
|
Packit |
fcad23 |
return if (!$opts{'debug'});
|
|
Packit |
fcad23 |
print @_;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub ERROR {
|
|
Packit |
fcad23 |
print STDERR "Error:", @_;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
sub DIE {
|
|
Packit |
fcad23 |
ERROR(@_);
|
|
Packit |
fcad23 |
exit 1;
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=pod
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 NAME
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
runfulltests - the Net-SNMP test suite coordinator
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 SYNOPSIS
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
runfulltests [OPTIONS]
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# ./RUNFULLTESTS
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# ./RUNFULLTESTS -g all
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 DESCRIPTION
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
B<RUNFULLTESTS> is a TAP (see below) output aggregator and test suite
|
|
Packit |
fcad23 |
management program. It runs groups of tests that it can find in the
|
|
Packit |
fcad23 |
I<fulltests> sub-directory. It defaults to running a basic set of
|
|
Packit |
fcad23 |
high-level Net-SNMP tests found in the fulltests/default directory.
|
|
Packit |
fcad23 |
To run a different set of tests see the -g flag.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
It is able to keep state and remember which tests failed so that
|
|
Packit |
fcad23 |
during development you can simply re-run only the "last failed tests"
|
|
Packit |
fcad23 |
using the -f switch.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Perl Requirements
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Ideally it should be run under a fairly modern version of perl (eg,
|
|
Packit |
fcad23 |
5.10) but minmial support is provided for older versions of perl as
|
|
Packit |
fcad23 |
well.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
If no perl is available on the system, there is also a fall-back
|
|
Packit |
fcad23 |
"RUNTESTS" suite which merely executes the default scripts for testing
|
|
Packit |
fcad23 |
compliance of the high-level applications.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Important Notes About Writing New Tests
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
When designing new tests it is strongly encouraged that some
|
|
Packit |
fcad23 |
conventions are followed:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
- Design the test files so they can be build/run without them
|
|
Packit |
fcad23 |
needing to be build/run within a testing harness like this one
|
|
Packit |
fcad23 |
(B<RUNFULLTESTS>). IE, you should be able to run them directly by
|
|
Packit |
fcad23 |
hand for debugging purposes without requiring them to be invoked
|
|
Packit |
fcad23 |
within this or any other testing harness.
|
|
Packit |
fcad23 |
- Name them beginning with TNNN where NNN is a 3 digit number
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
The rational behind these rules follows in the sections below
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 OPTIONS
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=over
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -g GROUP
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --group GROUP
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
By default the "default" group of tests is run. Which is really just
|
|
Packit |
fcad23 |
everything that B<RUNFULLTESTS> can find from the I<fulltests/default>
|
|
Packit |
fcad23 |
sub-directory. The -g switch can be used to specify other
|
|
Packit |
fcad23 |
sub-directories of tests to run. The argument is a comma-separated
|
|
Packit |
fcad23 |
list of subdirectories to use.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
The special keyword I<all> can be used to run every test in every
|
|
Packit |
fcad23 |
subdirectory of the I<fulltests> directory.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -r REGEXP
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Only run test files that match the I<REGEXP> regular expression.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
To run only tests of a certain file name you might combine this with
|
|
Packit |
fcad23 |
'-g all'. E.G., -g all -r snmpv3 will run all snmpv3 (named) tests
|
|
Packit |
fcad23 |
that B<RUNFULLTESTS> can find.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -f
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Only run the tests that failed from the last run.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --failed-file FILE
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Where to store the state of which tests have failed.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -F
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Don't save state to the failed-file.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -D PATH
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --srcdir PATH
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
If RUNFULLTESTS is being executed from a build directory other than
|
|
Packit |
fcad23 |
where the source files are located, this flag can be used to specify
|
|
Packit |
fcad23 |
where the Net-SNMP root source directory is found.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -d PATH
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --builddir PATH
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Specifies the root of the build directory.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --master-directory DIRNAME
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Specifies an alternate master directory. The default is "fulltests"
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -v
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --verbose
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Turns on verbose output mode.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -q
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --quiet
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Turns on quiet output mode.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --debug
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Turns on debugging output (which primarily shows how B<RUNFULLTESTS>
|
|
Packit |
fcad23 |
is collecting tests to run, etc).
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item -h
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item --help
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Command line usage help output.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=back
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 TEST ARCHITECTURE
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 TAP output
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
TAP stands for "Test Anything Protocol". TAP was originally
|
|
Packit |
fcad23 |
perl-specific but has been turning out to be a generic protocol for
|
|
Packit |
fcad23 |
testing just about anything. It's heavily documented at:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
http://testanything.org/wiki/index.php/Main_Page
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
We're using TAP because it's highly flexible and separates the
|
|
Packit |
fcad23 |
invidual tests from the "collect and report" aspect. RUNFULLTESTS is
|
|
Packit |
fcad23 |
simply a perl-based implementation for collecting and summarizing the
|
|
Packit |
fcad23 |
test results. Other collection agents could be used instead of this
|
|
Packit |
fcad23 |
one, and any sort of test or language could be used for the individual
|
|
Packit |
fcad23 |
tests as well (in fact the default suite has some that are SH-based,
|
|
Packit |
fcad23 |
C-based, ...).
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
It may be that eventually the TAP protocol actually makes it into the
|
|
Packit |
fcad23 |
IETF (http://testanything.org/wiki/index.php/TAP_at_IETF:_Draft_Standard).
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
The syntax of TAP is very simple. See the above web page for a
|
|
Packit |
fcad23 |
complete description, but this will provide you a minimal "getting
|
|
Packit |
fcad23 |
started" example and shows the output of 5 sub-tests run from a single
|
|
Packit |
fcad23 |
test application.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
1..5
|
|
Packit |
fcad23 |
ok 1 - Yay
|
|
Packit |
fcad23 |
ok 2 - Second part succeeded
|
|
Packit |
fcad23 |
not ok 3 - Oh no... A problem occurred.
|
|
Packit |
fcad23 |
not ok 4 - The computer thought 2+2 was 5
|
|
Packit |
fcad23 |
ok 5 - All is well that ends well
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
That's it. Output from a test tool like that is auto-summarized by
|
|
Packit |
fcad23 |
this application in success/fail reports.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Testing Phases
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
The testing process goes through the following phases:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
- Test and Infrastructure Collection
|
|
Packit |
fcad23 |
- Test Execution
|
|
Packit |
fcad23 |
- Build if needed
|
|
Packit |
fcad23 |
- Run
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Test Collection
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
B<RUNFULLTESTS> will search all the requested groups for files that
|
|
Packit |
fcad23 |
begin with the letter 'T'. They are executed in alphabetical order
|
|
Packit |
fcad23 |
within their group. Convention is to number test files with a three
|
|
Packit |
fcad23 |
digit (zero-padded) number after the 'T' letter to ensure ordering is
|
|
Packit |
fcad23 |
as expected.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Files to be collected by B<RUNFULLTESTS> are made up of a number of
|
|
Packit |
fcad23 |
components to support a flexible build and execution system (discussed
|
|
Packit |
fcad23 |
in detail in the following sections). They are structured as follows:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
T<NNN><NAME>_<TYPE>.<SUFFIX>
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Where:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
NNN: Is the 3 digit number mentioned above.
|
|
Packit |
fcad23 |
NAME: The filename of the test describing what it's about
|
|
Packit |
fcad23 |
TYPE: The internal "type" of the test, which is used later in building
|
|
Packit |
fcad23 |
and execution (see below).
|
|
Packit |
fcad23 |
.SUFFIX: An optional file-type suffix
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Examples:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
fulltests/default/T001snmpv1get_simple
|
|
Packit |
fcad23 |
fulltests/snmpv3/T040keymanagetest_capp.c
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Any other files are ignored in terms of tests and may be supplimental
|
|
Packit |
fcad23 |
to the above build systems. (Supporting files, by convention, begin
|
|
Packit |
fcad23 |
with a capital 'S').
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head3 Full Title
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Within the file there may be a line containing "HEADER ..." that will
|
|
Packit |
fcad23 |
be examined for a better title of the test. Anything before "HEADER"
|
|
Packit |
fcad23 |
will be ignored, and the special "*/" will be replaced as well. For
|
|
Packit |
fcad23 |
example, these are valid header source lines:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
# HEADER A cool test
|
|
Packit |
fcad23 |
/* HEADER A cool test from my C-based source file */
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Infrastructure Collection
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
In addition to test files, I<infrastructure> files are searched for
|
|
Packit |
fcad23 |
and remembered for later use (again, see below). These files will be
|
|
Packit |
fcad23 |
of the form:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
<TYPE>_<USEAGE>
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Where:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
TYPE: The type name matching the file to support.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
USAGE: How this file should be used. Currently should be either
|
|
Packit |
fcad23 |
I<build> or I<run> as described below.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Example files
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
fulltests/support/clib_build
|
|
Packit |
fcad23 |
fulltests/support/simple_run
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Infrastructure files may exist in the source directory of where they're
|
|
Packit |
fcad23 |
expected to be run (ie, parallel to the test files) or they may exist
|
|
Packit |
fcad23 |
in the special "support" directory if they're expected to be
|
|
Packit |
fcad23 |
generically used across multilpe test group types.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head2 Test Execution
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Tests are run in two phases using the following pseudo-algorithm:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
+ for each test file
|
|
Packit |
fcad23 |
+ if an appropriate TYPE_build file exists for a test {
|
|
Packit |
fcad23 |
+ run "TYPE_build TESTFILE"
|
|
Packit |
fcad23 |
+ record the last line as the new TESTFILE to run
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
+ if an apporpriate TYPE_run file exists for a test {
|
|
Packit |
fcad23 |
+ run "TYPE_run TESTFILE"
|
|
Packit |
fcad23 |
+ collect it's output as the TAP output
|
|
Packit |
fcad23 |
} else {
|
|
Packit |
fcad23 |
+ run "TESTFILE"
|
|
Packit |
fcad23 |
+ collect it's output as the TAP output
|
|
Packit |
fcad23 |
}
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
For example, if the following files existed:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
fulltests/examplres/T001testing_examp
|
|
Packit |
fcad23 |
fulltests/examplres/examp_build
|
|
Packit |
fcad23 |
fulltests/examplres/examp_run
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Then the following would be the rough execution:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
newfile = `fulltests/examplres/examp_build \
|
|
Packit |
fcad23 |
fulltests/examplres/T001testing_examp | tail -1`
|
|
Packit |
fcad23 |
fulltests/examplres/examp_run $newfile
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 TEST TYPES
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Net-SNMP testing comes with a number of test suite "builders" and
|
|
Packit |
fcad23 |
"runners" that are useful for developing new tests. These are
|
|
Packit |
fcad23 |
documented here:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=over
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item simple
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
I<simple> test files are simple sh-shell-script files used to test
|
|
Packit |
fcad23 |
high-level functionality of Net-SNMP tools. They're easy to write and
|
|
Packit |
fcad23 |
should generally contain the following sort of structure:
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
. ../support/simple_eval_tools.sh
|
|
Packit |
fcad23 |
HEADER my name
|
|
Packit |
fcad23 |
STARTAGENT
|
|
Packit |
fcad23 |
CAPTURE "snmpget..."
|
|
Packit |
fcad23 |
CHECK "for this string"
|
|
Packit |
fcad23 |
STOPAGENT
|
|
Packit |
fcad23 |
FINISHED
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Example file: fulltests/default/T001snmpv1get_simple
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item capp
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
I<capp> files are fundamentally full C-source-code applications that
|
|
Packit |
fcad23 |
are built and linked against the libnetsnmp library. Thus, a file
|
|
Packit |
fcad23 |
named I<T001mytest_capp.c> is compiled using the same compiler used to
|
|
Packit |
fcad23 |
compile Net-SNMP and linked against the required libraries for a basic
|
|
Packit |
fcad23 |
Net-SNMP application. It should, of course, produce TAP output after
|
|
Packit |
fcad23 |
it's compiled and run.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Example file: fulltests/snmpv3/T010scapitest_capp.c
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item clib
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
I<clib> files are simple C-source-code files that are wrapped into a
|
|
Packit |
fcad23 |
main() application with appropriate #include files, etc. I<clib>
|
|
Packit |
fcad23 |
files are designed primarly to write quick unit-tests for the Net-SNMP
|
|
Packit |
fcad23 |
core library.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Example file: fulltests/unit-tests/T001defaultstore_clib.c
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=item Write your own!
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
This test system is designed to be flexible and expandable if the
|
|
Packit |
fcad23 |
basic architecture is followed. The goal is to make it easy to create
|
|
Packit |
fcad23 |
very simple test suites or complex unit-tests or anything in between.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=back
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 DEBUGGING BROKEN TESTS
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
If the individual tests are designed well, you should be able to
|
|
Packit |
fcad23 |
re-run individual tests outside of the B<RUNFULLTESTS> aggregation
|
|
Packit |
fcad23 |
environment using the appropriate _build and _run scripts as needed.
|
|
Packit |
fcad23 |
Test writers are encouraged to output comments in their TAP output to
|
|
Packit |
fcad23 |
help users debug the results.
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=head1 Author
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
Original architecture: Wes Hardaker <hardaker@users.sourceforge.net>
|
|
Packit |
fcad23 |
|
|
Packit |
fcad23 |
=cut
|
|
Packit |
fcad23 |
|