Blame test/testsuite.cpp

Packit 2035a7
/*
Packit 2035a7
 * Cppcheck - A tool for static C/C++ code analysis
Packit 2035a7
 * Copyright (C) 2007-2017 Cppcheck team.
Packit 2035a7
 *
Packit 2035a7
 * This program is free software: you can redistribute it and/or modify
Packit 2035a7
 * it under the terms of the GNU General Public License as published by
Packit 2035a7
 * the Free Software Foundation, either version 3 of the License, or
Packit 2035a7
 * (at your option) any later version.
Packit 2035a7
 *
Packit 2035a7
 * This program is distributed in the hope that it will be useful,
Packit 2035a7
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
Packit 2035a7
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Packit 2035a7
 * GNU General Public License for more details.
Packit 2035a7
 *
Packit 2035a7
 * You should have received a copy of the GNU General Public License
Packit 2035a7
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
Packit 2035a7
 */
Packit 2035a7
Packit 2035a7
#include "testsuite.h"
Packit 2035a7
Packit 2035a7
#include "options.h"
Packit 2035a7
#include "redirect.h"
Packit 2035a7
Packit 2035a7
#include <cstdio>
Packit 2035a7
#include <iostream>
Packit 2035a7
#include <list>
Packit 2035a7
#include <string>
Packit 2035a7
Packit 2035a7
std::ostringstream errout;
Packit 2035a7
std::ostringstream output;
Packit 2035a7
Packit 2035a7
/**
Packit 2035a7
 * TestRegistry
Packit 2035a7
 **/
Packit 2035a7
Packit 2035a7
class TestRegistry {
Packit 2035a7
private:
Packit 2035a7
    std::list<TestFixture *> _tests;
Packit 2035a7
Packit 2035a7
public:
Packit 2035a7
    static TestRegistry &theInstance() {
Packit 2035a7
        static TestRegistry testreg;
Packit 2035a7
        return testreg;
Packit 2035a7
    }
Packit 2035a7
Packit 2035a7
    void addTest(TestFixture *t) {
Packit 2035a7
        _tests.push_back(t);
Packit 2035a7
    }
Packit 2035a7
Packit 2035a7
    const std::list<TestFixture *> &tests() const {
Packit 2035a7
        return _tests;
Packit 2035a7
    }
Packit 2035a7
};
Packit 2035a7
Packit 2035a7
Packit 2035a7
Packit 2035a7
Packit 2035a7
/**
Packit 2035a7
 * TestFixture
Packit 2035a7
 **/
Packit 2035a7
Packit 2035a7
std::ostringstream TestFixture::errmsg;
Packit 2035a7
unsigned int       TestFixture::countTests;
Packit 2035a7
Packit 2035a7
std::size_t TestFixture::fails_counter = 0;
Packit 2035a7
std::size_t TestFixture::todos_counter = 0;
Packit 2035a7
std::size_t TestFixture::succeeded_todos_counter = 0;
Packit 2035a7
std::set<std::string> TestFixture::missingLibs;
Packit 2035a7
Packit 2035a7
TestFixture::TestFixture(const char * const _name)
Packit 2035a7
    :classname(_name)
Packit 2035a7
    ,quiet_tests(false)
Packit 2035a7
{
Packit 2035a7
    TestRegistry::theInstance().addTest(this);
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
Packit 2035a7
bool TestFixture::prepareTest(const char testname[])
Packit 2035a7
{
Packit 2035a7
    // Check if tests should be executed
Packit 2035a7
    if (testToRun.empty() || testToRun == testname) {
Packit 2035a7
        // Tests will be executed - prepare them
Packit 2035a7
        ++countTests;
Packit 2035a7
        if (quiet_tests) {
Packit 2035a7
            std::putchar('.'); // Use putchar to write through redirection of std::cout/cerr
Packit 2035a7
            std::fflush(stdout);
Packit 2035a7
        } else {
Packit 2035a7
            std::cout << classname << "::" << testname << std::endl;
Packit 2035a7
        }
Packit 2035a7
        return true;
Packit 2035a7
    }
Packit 2035a7
    return false;
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
static std::string writestr(const std::string &str, bool gccStyle = false)
Packit 2035a7
{
Packit 2035a7
    std::ostringstream ostr;
Packit 2035a7
    if (gccStyle)
Packit 2035a7
        ostr << '\"';
Packit 2035a7
    for (std::string::const_iterator i = str.begin(); i != str.end(); ++i) {
Packit 2035a7
        if (*i == '\n') {
Packit 2035a7
            ostr << "\\n";
Packit 2035a7
            if ((i+1) != str.end() && !gccStyle)
Packit 2035a7
                ostr << std::endl;
Packit 2035a7
        } else if (*i == '\t')
Packit 2035a7
            ostr << "\\t";
Packit 2035a7
        else if (*i == '\"')
Packit 2035a7
            ostr << "\\\"";
Packit 2035a7
        else
Packit 2035a7
            ostr << *i;
Packit 2035a7
    }
Packit 2035a7
    if (!str.empty() && !gccStyle)
Packit 2035a7
        ostr << std::endl;
Packit 2035a7
    else if (gccStyle)
Packit 2035a7
        ostr << '\"';
Packit 2035a7
    return ostr.str();
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assert_(const char * const filename, const unsigned int linenr, const bool condition) const
Packit 2035a7
{
Packit 2035a7
    if (!condition) {
Packit 2035a7
        ++fails_counter;
Packit 2035a7
        errmsg << filename << ':' << linenr << ": Assertion failed." << std::endl << "_____" << std::endl;
Packit 2035a7
    }
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    if (expected != actual) {
Packit 2035a7
        ++fails_counter;
Packit 2035a7
        errmsg << filename << ':' << linenr << ": Assertion failed. " << std::endl
Packit 2035a7
               << "Expected: " <<  std::endl
Packit 2035a7
               << writestr(expected)  << std::endl
Packit 2035a7
               << "Actual: " << std::endl
Packit 2035a7
               << writestr(actual) << std::endl;
Packit 2035a7
        if (!msg.empty())
Packit 2035a7
            errmsg << "Hint:" << std::endl <<  msg << std::endl;
Packit 2035a7
        errmsg << "_____" << std::endl;
Packit 2035a7
    }
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
std::string TestFixture::deleteLineNumber(const std::string &message) const
Packit 2035a7
{
Packit 2035a7
    std::string result(message);
Packit 2035a7
    // delete line number in "...:NUMBER:..."
Packit 2035a7
    std::string::size_type pos = 0;
Packit 2035a7
    std::string::size_type after = 0;
Packit 2035a7
    while ((pos = result.find(':', pos)) != std::string::npos) {
Packit 2035a7
        // get number
Packit 2035a7
        if (pos + 1 == result.find_first_of("0123456789", pos + 1)) {
Packit 2035a7
            if ((after = result.find_first_not_of("0123456789", pos + 1)) != std::string::npos
Packit 2035a7
                && result.at(after) == ':') {
Packit 2035a7
                // erase NUMBER
Packit 2035a7
                result.erase(pos + 1, after - pos - 1);
Packit 2035a7
                pos = after;
Packit 2035a7
            } else {
Packit 2035a7
                ++pos;
Packit 2035a7
            }
Packit 2035a7
        } else {
Packit 2035a7
            ++pos;
Packit 2035a7
        }
Packit 2035a7
    }
Packit 2035a7
    return result;
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertEqualsWithoutLineNumbers(const char * const filename, const unsigned int linenr, const std::string &expected, const std::string &actual, const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    assertEquals(filename, linenr, deleteLineNumber(expected), deleteLineNumber(actual), msg);
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const std::string& actual, const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    assertEquals(filename, linenr, std::string(expected), actual, msg);
Packit 2035a7
}
Packit 2035a7
void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const char expected[], const char actual[], const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    assertEquals(filename, linenr, std::string(expected), std::string(actual), msg);
Packit 2035a7
}
Packit 2035a7
void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const std::string& expected, const char actual[], const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    assertEquals(filename, linenr, expected, std::string(actual), msg);
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertEquals(const char * const filename, const unsigned int linenr, const long long expected, const long long actual, const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    if (expected != actual) {
Packit 2035a7
        std::ostringstream ostr1;
Packit 2035a7
        ostr1 << expected;
Packit 2035a7
        std::ostringstream ostr2;
Packit 2035a7
        ostr2 << actual;
Packit 2035a7
        assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
Packit 2035a7
    }
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertEqualsDouble(const char * const filename, const unsigned int linenr, const double expected, const double actual, const double tolerance, const std::string &msg) const
Packit 2035a7
{
Packit 2035a7
    if (expected < (actual - tolerance) || expected > (actual + tolerance)) {
Packit 2035a7
        std::ostringstream ostr1;
Packit 2035a7
        ostr1 << expected;
Packit 2035a7
        std::ostringstream ostr2;
Packit 2035a7
        ostr2 << actual;
Packit 2035a7
        assertEquals(filename, linenr, ostr1.str(), ostr2.str(), msg);
Packit 2035a7
    }
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr,
Packit 2035a7
                                   const std::string &wanted,
Packit 2035a7
                                   const std::string &current,
Packit 2035a7
                                   const std::string &actual) const
Packit 2035a7
{
Packit 2035a7
    if (wanted == actual) {
Packit 2035a7
        errmsg << filename << ':' << linenr << ": Assertion succeeded unexpectedly. "
Packit 2035a7
               << "Result: " << writestr(wanted, true)  << std::endl << "_____" << std::endl;
Packit 2035a7
Packit 2035a7
        ++succeeded_todos_counter;
Packit 2035a7
    } else {
Packit 2035a7
        assertEquals(filename, linenr, current, actual);
Packit 2035a7
        ++todos_counter;
Packit 2035a7
    }
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::todoAssertEquals(const char * const filename, const unsigned int linenr, const long long wanted, const long long current, const long long actual) const
Packit 2035a7
{
Packit 2035a7
    std::ostringstream wantedStr, currentStr, actualStr;
Packit 2035a7
    wantedStr << wanted;
Packit 2035a7
    currentStr << current;
Packit 2035a7
    actualStr << actual;
Packit 2035a7
    todoAssertEquals(filename, linenr, wantedStr.str(), currentStr.str(), actualStr.str());
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertThrow(const char * const filename, const unsigned int linenr) const
Packit 2035a7
{
Packit 2035a7
    ++fails_counter;
Packit 2035a7
    errmsg << filename << ':' << linenr << ": Assertion succeeded. "
Packit 2035a7
           << "The expected exception was thrown" << std::endl << "_____" << std::endl;
Packit 2035a7
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertThrowFail(const char * const filename, const unsigned int linenr) const
Packit 2035a7
{
Packit 2035a7
    ++fails_counter;
Packit 2035a7
    errmsg << filename << ':' << linenr << ": Assertion failed. "
Packit 2035a7
           << "The expected exception was not thrown"  << std::endl << "_____" << std::endl;
Packit 2035a7
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::assertNoThrowFail(const char * const filename, const unsigned int linenr) const
Packit 2035a7
{
Packit 2035a7
    ++fails_counter;
Packit 2035a7
    errmsg << filename << ':' << linenr << ": Assertion failed. "
Packit 2035a7
           << "Unexpected exception was thrown"  << std::endl << "_____" << std::endl;
Packit 2035a7
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::complainMissingLib(const char * const libname) const
Packit 2035a7
{
Packit 2035a7
    missingLibs.insert(libname);
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::run(const std::string &str)
Packit 2035a7
{
Packit 2035a7
    testToRun = str;
Packit 2035a7
    if (quiet_tests) {
Packit 2035a7
        std::cout << '\n' << classname << ':';
Packit 2035a7
    }
Packit 2035a7
    if (quiet_tests) {
Packit 2035a7
        REDIRECT;
Packit 2035a7
        run();
Packit 2035a7
    } else
Packit 2035a7
        run();
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::processOptions(const options& args)
Packit 2035a7
{
Packit 2035a7
    quiet_tests = args.quiet();
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
std::size_t TestFixture::runTests(const options& args)
Packit 2035a7
{
Packit 2035a7
    std::string classname(args.which_test());
Packit 2035a7
    std::string testname;
Packit 2035a7
    if (classname.find("::") != std::string::npos) {
Packit 2035a7
        testname = classname.substr(classname.find("::") + 2);
Packit 2035a7
        classname.erase(classname.find("::"));
Packit 2035a7
    }
Packit 2035a7
Packit 2035a7
    countTests = 0;
Packit 2035a7
    errmsg.str("");
Packit 2035a7
Packit 2035a7
    const std::list<TestFixture *> &tests = TestRegistry::theInstance().tests();
Packit 2035a7
Packit 2035a7
    for (std::list<TestFixture *>::const_iterator it = tests.begin(); it != tests.end(); ++it) {
Packit 2035a7
        if (classname.empty() || (*it)->classname == classname) {
Packit 2035a7
            (*it)->processOptions(args);
Packit 2035a7
            (*it)->run(testname);
Packit 2035a7
        }
Packit 2035a7
    }
Packit 2035a7
Packit 2035a7
    std::cout << "\n\nTesting Complete\nNumber of tests: " << countTests << std::endl;
Packit 2035a7
    std::cout << "Number of todos: " << todos_counter;
Packit 2035a7
    if (succeeded_todos_counter > 0)
Packit 2035a7
        std::cout << " (" << succeeded_todos_counter << " succeeded)";
Packit 2035a7
    std::cout << std::endl;
Packit 2035a7
    // calling flush here, to do all output before the error messages (in case the output is buffered)
Packit 2035a7
    std::cout.flush();
Packit 2035a7
Packit 2035a7
    std::cerr << "Tests failed: " << fails_counter << std::endl << std::endl;
Packit 2035a7
    std::cerr << errmsg.str();
Packit 2035a7
Packit 2035a7
    if (!missingLibs.empty()) {
Packit 2035a7
        std::cerr << "Missing libraries: ";
Packit 2035a7
        for (std::set<std::string>::const_iterator i = missingLibs.begin(); i != missingLibs.end(); ++i)
Packit 2035a7
            std::cerr << *i << "  ";
Packit 2035a7
        std::cerr << std::endl << std::endl;
Packit 2035a7
    }
Packit 2035a7
    std::cerr.flush();
Packit 2035a7
    return fails_counter;
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::reportOut(const std::string & outmsg)
Packit 2035a7
{
Packit 2035a7
    output << outmsg << std::endl;
Packit 2035a7
}
Packit 2035a7
Packit 2035a7
void TestFixture::reportErr(const ErrorLogger::ErrorMessage &msg)
Packit 2035a7
{
Packit 2035a7
    const std::string errormessage(msg.toString(false));
Packit 2035a7
    if (errout.str().find(errormessage) == std::string::npos)
Packit 2035a7
        errout << errormessage << std::endl;
Packit 2035a7
}