|
Packit |
345191 |
/*
|
|
Packit |
345191 |
* Copyright (C) 2014 - 2019 Intel Corporation.
|
|
Packit |
345191 |
* All rights reserved.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* Redistribution and use in source and binary forms, with or without
|
|
Packit |
345191 |
* modification, are permitted provided that the following conditions are met:
|
|
Packit |
345191 |
* 1. Redistributions of source code must retain the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer.
|
|
Packit |
345191 |
* 2. Redistributions in binary form must reproduce the above copyright notice(s),
|
|
Packit |
345191 |
* this list of conditions and the following disclaimer in the documentation
|
|
Packit |
345191 |
* and/or other materials provided with the distribution.
|
|
Packit |
345191 |
*
|
|
Packit |
345191 |
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS
|
|
Packit |
345191 |
* OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
Packit |
345191 |
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
Packit |
345191 |
* EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
Packit |
345191 |
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
Packit |
345191 |
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
|
|
Packit |
345191 |
* PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
Packit |
345191 |
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
Packit |
345191 |
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
Packit |
345191 |
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
Packit |
345191 |
*/
|
|
Packit |
345191 |
|
|
Packit |
345191 |
#include "perf_tests.hpp"
|
|
Packit |
345191 |
#include <iostream>
|
|
Packit |
345191 |
#include <cmath>
|
|
Packit |
345191 |
#include <gtest/gtest.h>
|
|
Packit |
345191 |
|
|
Packit |
345191 |
// Memkind performance tests
|
|
Packit |
345191 |
using std::cout;
|
|
Packit |
345191 |
using std::endl;
|
|
Packit |
345191 |
using std::abs;
|
|
Packit |
345191 |
using std::ostringstream;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
// Memkind tests
|
|
Packit |
345191 |
class PerformanceTest : public testing::Test
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
protected:
|
|
Packit |
345191 |
const double Tolerance = 0.15;
|
|
Packit |
345191 |
const double Confidence = 0.10;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
Metrics referenceMetrics;
|
|
Packit |
345191 |
Metrics performanceMetrics;
|
|
Packit |
345191 |
PerfTestCase<performance_tests::MallocOperation> referenceTest;
|
|
Packit |
345191 |
PerfTestCase<performance_tests::MemkindOperation> performanceTest;
|
|
Packit |
345191 |
|
|
Packit |
345191 |
template<class T>
|
|
Packit |
345191 |
static void RecordProperty(const string &key, const T value)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
ostringstream values;
|
|
Packit |
345191 |
values << value;
|
|
Packit |
345191 |
testing::Test::RecordProperty(key, values.str());
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void SetUp()
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void TearDown()
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
bool checkDelta(double value, double reference, const string &info,
|
|
Packit |
345191 |
double delta, bool higherIsBetter = false)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
// If higherIsBetter is true, current value should be >= (reference - delta); otherwise, it should be <= (reference + delta)
|
|
Packit |
345191 |
double threshold = higherIsBetter ? reference * (1 - delta) : reference *
|
|
Packit |
345191 |
(1 + delta);
|
|
Packit |
345191 |
cout <<
|
|
Packit |
345191 |
"Metric: " << info << ". Reference value: " << reference << ". "
|
|
Packit |
345191 |
"Expected: " << (higherIsBetter ? ">= " : "<= ") << threshold << " (delta = " <<
|
|
Packit |
345191 |
delta << ")."
|
|
Packit |
345191 |
"Actual: " << value << " (delta = " <<
|
|
Packit |
345191 |
((value > reference ? (value / reference) : (reference / value)) - 1.0) /
|
|
Packit |
345191 |
((higherIsBetter != (value > reference)) ? 1.0 : -1.0)
|
|
Packit |
345191 |
<< ")."
|
|
Packit |
345191 |
<< endl ;
|
|
Packit |
345191 |
if (higherIsBetter ? (value <= threshold) : (value >= threshold)) {
|
|
Packit |
345191 |
cout << "WARNING : Value of '" << info << "' outside expected bounds!" << endl;
|
|
Packit |
345191 |
return false;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
return true;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
bool compareMetrics(Metrics metrics, Metrics reference, double delta)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
bool result = true;
|
|
Packit |
345191 |
result &= checkDelta(metrics.operationsPerSecond, reference.operationsPerSecond,
|
|
Packit |
345191 |
"operationsPerSecond", delta, true);
|
|
Packit |
345191 |
result &= checkDelta(metrics.avgOperationDuration,
|
|
Packit |
345191 |
reference.avgOperationDuration, "avgOperationDuration", delta);
|
|
Packit |
345191 |
return result;
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void writeMetrics(Metrics metrics, Metrics reference)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
RecordProperty("ops_per_sec", metrics.operationsPerSecond);
|
|
Packit |
345191 |
RecordProperty("ops_per_sec_vs_ref",
|
|
Packit |
345191 |
(reference.operationsPerSecond - metrics.operationsPerSecond) * 100.0f
|
|
Packit |
345191 |
/ reference.operationsPerSecond);
|
|
Packit |
345191 |
RecordProperty("avg_op_time_nsec", metrics.avgOperationDuration);
|
|
Packit |
345191 |
RecordProperty("avg_op_time_nsec_vs_ref",
|
|
Packit |
345191 |
(metrics.avgOperationDuration - reference.avgOperationDuration) * 100.0f
|
|
Packit |
345191 |
/ reference.avgOperationDuration);
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
void run()
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
cout << "Running reference std::malloc test" << endl;
|
|
Packit |
345191 |
referenceMetrics = referenceTest.runTest();
|
|
Packit |
345191 |
|
|
Packit |
345191 |
cout << "Running memkind test" << endl;
|
|
Packit |
345191 |
performanceMetrics = performanceTest.runTest();
|
|
Packit |
345191 |
|
|
Packit |
345191 |
writeMetrics(performanceMetrics, referenceMetrics);
|
|
Packit |
345191 |
EXPECT_TRUE(compareMetrics(performanceMetrics, referenceMetrics,
|
|
Packit |
345191 |
Tolerance + Confidence));
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
};
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_single_op_single_iter)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_singleOpSingleIter();
|
|
Packit |
345191 |
performanceTest.setupTest_singleOpSingleIter();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_many_ops_single_iter)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_manyOpsSingleIter();
|
|
Packit |
345191 |
performanceTest.setupTest_manyOpsSingleIter();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_many_ops_single_iter_huge_alloc)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_manyOpsSingleIterHugeAlloc();
|
|
Packit |
345191 |
performanceTest.setupTest_manyOpsSingleIterHugeAlloc();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_single_op_many_iters)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_singleOpManyIters();
|
|
Packit |
345191 |
performanceTest.setupTest_singleOpManyIters();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_many_ops_many_iters)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_manyOpsManyIters();
|
|
Packit |
345191 |
performanceTest.setupTest_manyOpsManyIters();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|
|
Packit |
345191 |
|
|
Packit |
345191 |
TEST_F(PerformanceTest, test_TC_MEMKIND_perf_many_ops_many_iters_many_kinds)
|
|
Packit |
345191 |
{
|
|
Packit |
345191 |
referenceTest.setupTest_manyOpsManyIters();
|
|
Packit |
345191 |
performanceTest.setupTest_manyOpsManyIters();
|
|
Packit |
345191 |
run();
|
|
Packit |
345191 |
}
|