Skip to content

Commit

Permalink
Aggressive clean of LayerTestsUtils::LayerTestsCommon (openvinotool…
Browse files Browse the repository at this point in the history
  • Loading branch information
vurusovs authored Feb 3, 2024
1 parent a2586ce commit b60526c
Show file tree
Hide file tree
Showing 19 changed files with 14 additions and 979 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ typedef std::tuple<
ElementType, // Input precision
ElementType, // Output precision
InputShape, // Input shape
LayerTestsUtils::TargetDevice // Device name
ov::test::TargetDevice // Device name
> convLayerTestParamsSet;

typedef std::tuple<
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ typedef std::tuple<InputShape, // input_shapes
typedef std::tuple<embeddingBagOffsetsSumParams,
ElementType, // embedding table
ElementType, // indices
LayerTestsUtils::TargetDevice>
ov::test::TargetDevice>
embeddingBagOffsetsSumLayerTestParamsSet;

class EmbeddingBagOffsetsSumLayerCPUTest : public testing::WithParamInterface<embeddingBagOffsetsSumLayerTestParamsSet>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ typedef std::tuple<InputShape, // input_shapes
typedef std::tuple<embeddingBagPackedSumParams,
ElementType, // embedding table
ElementType, // indices
LayerTestsUtils::TargetDevice>
ov::test::TargetDevice>
embeddingBagPackedSumLayerTestParamsSet;

class EmbeddingBagPackedSumLayerCPUTest : public testing::WithParamInterface<embeddingBagPackedSumLayerTestParamsSet>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ typedef std::tuple<InputShape, // input_shapes
typedef std::tuple<embeddingSegmentsSumParams,
ElementType, // embedding table
ElementType, // indices
LayerTestsUtils::TargetDevice>
ov::test::TargetDevice>
embeddingSegmentsSumLayerTestParamsSet;

class EmbeddingSegmentsSumLayerCPUTest : public testing::WithParamInterface<embeddingSegmentsSumLayerTestParamsSet>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
// typedef std::tuple<
// RangeSpecificParams,
// InferenceEngine::Precision, // Net precision
// LayerTestsUtils::TargetDevice // Device name
// ov::test::TargetDevice // Device name
//> RangeLayerTestParams;
//
// typedef std::tuple<
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ using roiPoolingParams = std::tuple<roiPoolingShapes, // Input sh
float, // Spatial scale
utils::ROIPoolingTypes, // ROIPooling method
ov::element::Type, // Net precision
LayerTestsUtils::TargetDevice>; // Device name
ov::test::TargetDevice>; // Device name

using ROIPoolingCPUTestParamsSet = std::tuple<roiPoolingParams,
CPUSpecificParams,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ using ROIAlignSpecificParams = std::tuple<
using ROIAlignLayerTestParams = std::tuple<
ROIAlignSpecificParams,
ElementType, // Net precision
LayerTestsUtils::TargetDevice // Device name
ov::test::TargetDevice // Device name
>;

using ROIAlignLayerCPUTestParamsSet = std::tuple<
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ class ConcatConstantInPlaceTest : public testing::WithParamInterface<ov::element
// convolution
std::vector<float> weightValuesFP32(12);
ov::Shape convFilterShape = {1, 12, 1, 1};
FuncTestUtils::fillInputsBySinValues(weightValuesFP32.data(), weightValuesFP32.size());
for (size_t i = 0; i < weightValuesFP32.size(); i++) {
weightValuesFP32.data()[i] = sin(static_cast<float>(i));
}
auto weightsNode = std::make_shared<ov::op::v0::Constant>(ov::element::f32, convFilterShape, weightValuesFP32);
std::shared_ptr<ov::Node> conv = std::make_shared<ov::op::v1::Convolution>(concat,
weightsNode,
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -63,139 +63,6 @@ class BenchmarkLayerTestReporter {
pugi::xml_document report_xml_{};
};

} // namespace test
} // namespace ov

namespace LayerTestsDefinitions {

template <typename BaseLayerTest>
class BenchmarkLayerTest : public BaseLayerTest {
static_assert(std::is_base_of<LayerTestsUtils::LayerTestsCommon, BaseLayerTest>::value,
"BaseLayerTest should inherit from LayerTestsUtils::LayerTestsCommon");

public:
static constexpr int kDefaultNumberOfAttempts = 100;
static constexpr double kMaxAllowedBenchmarkDifference = 0.05;

void RunBenchmark(const std::initializer_list<std::string>& nodeTypeNames,
const std::chrono::milliseconds warmupTime = std::chrono::milliseconds(2000),
const int numAttempts = kDefaultNumberOfAttempts) {
bench_node_type_names_ = nodeTypeNames;
warmup_time_ = warmupTime;
num_attempts_ = numAttempts;
this->configuration.insert({"PERF_COUNT", "YES"});
this->Run();
}

void RunBenchmark(const std::string& nodeTypeName,
const std::chrono::milliseconds warmupTime = std::chrono::milliseconds(2000),
const int numAttempts = kDefaultNumberOfAttempts) {
if (!nodeTypeName.empty()) {
RunBenchmark({nodeTypeName}, warmupTime, numAttempts);
} else {
RunBenchmark({}, warmupTime, numAttempts);
}
}

void Validate() override {
for (const auto& res : curr_bench_results_) {
const auto& node_type_name = res.first;
const auto curr_time = static_cast<int64_t>(res.second);
if (prev_bench_results_.count(node_type_name) > 0) {
const auto prev_time = static_cast<int64_t>(prev_bench_results_[node_type_name]);
const auto delta_time = static_cast<double>(curr_time - prev_time);
if (delta_time/prev_time > kMaxAllowedBenchmarkDifference) {
std::cerr << "node_type_name: " << node_type_name <<
", for test case: " << BaseLayerTest::GetTestName() <<
", has exceeded the benchmark threshold: " << kMaxAllowedBenchmarkDifference <<
". Current: " << curr_time << " us, previous: " << prev_time << " us" << std::endl;
}
}
}
}

protected:
void Infer() override {
this->inferRequest = this->executableNetwork.CreateInferRequest();
this->ConfigureInferRequest();

#ifdef ENABLE_BENCHMARK_FILE_REPORT
reporter_ = std::unique_ptr<ov::test::BenchmarkLayerTestReporter>(
new ::ov::test::BenchmarkLayerTestReporter{false});
#else
reporter_ = std::unique_ptr<ov::test::BenchmarkLayerTestReporter>(
new ::ov::test::BenchmarkLayerTestReporter{true});
#endif
for (const auto& node_type_name : bench_node_type_names_) {
try {
const auto time = reporter_->get_time(node_type_name, BaseLayerTest::GetTestName());
prev_bench_results_[node_type_name] = time;
} catch (...) {
}
}

std::map<std::string, uint64_t> results_us{};
for (const auto& node_type_name : bench_node_type_names_) {
results_us[node_type_name] = {};
}

// Warmup
auto warm_current = std::chrono::steady_clock::now();
const auto warm_end = warm_current + warmup_time_;
while (warm_current < warm_end) {
this->inferRequest.Infer();
warm_current = std::chrono::steady_clock::now();
}

// Benchmark
for (size_t i = 0; i < num_attempts_; ++i) {
this->inferRequest.Infer();
const auto& perf_results = this->inferRequest.GetPerformanceCounts();
for (auto& res : results_us) {
const std::string node_type_name = res.first;
uint64_t& time = res.second;
auto found_profile = std::find_if(perf_results.begin(), perf_results.end(),
[&node_type_name](const InferenceEngine::InferenceEngineProfileInfo& profile) {
return profile.layer_type == node_type_name;
});
if (found_profile == perf_results.end()) {
IE_THROW() << "Cannot find operator by node type: " << node_type_name;
}
time += found_profile->second.realTime_uSec;
}
}

std::stringstream report{};
uint64_t total_us = 0;
for (const auto& res : results_us) {
const std::string node_type_name = res.first;
uint64_t time = res.second;
time /= num_attempts_;
total_us += time;
report << std::fixed << std::setfill('0') << node_type_name << ": " << time << " us\n";
#ifdef ENABLE_BENCHMARK_FILE_REPORT
curr_bench_results_[node_type_name] = time;
reporter_->report(node_type_name, BaseLayerTest::GetTestName(), time);
#endif
}
report << std::fixed << std::setfill('0') << "Total time: " << total_us << " us\n";
std::cout << report.str();
}

private:
std::unique_ptr<ov::test::BenchmarkLayerTestReporter> reporter_;
std::unordered_map<std::string, uint64_t> prev_bench_results_;
std::unordered_map<std::string, uint64_t> curr_bench_results_;
std::vector<std::string> bench_node_type_names_;
std::chrono::milliseconds warmup_time_;
int num_attempts_;
};

} // namespace LayerTestsDefinitions

namespace ov {
namespace test {

template <typename BaseLayerTest>
class BenchmarkLayerTest : public BaseLayerTest {
static_assert(std::is_base_of<SubgraphBaseTest, BaseLayerTest>::value,
Expand Down
Loading

0 comments on commit b60526c

Please sign in to comment.