Add google benchmark for NormalEstimation #4506
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| set(SUBSYS_NAME benchmarks) | ||
| set(SUBSYS_DESC "Point cloud library benchmarks") | ||
| set(SUBSYS_DEPS common features search kdtree io) | ||
| set(DEFAULT OFF) | ||
| set(build TRUE) | ||
| set(REASON "Disabled by default") | ||
| PCL_SUBSYS_OPTION(build "${SUBSYS_NAME}" "${SUBSYS_DESC}" ${DEFAULT} "${REASON}") | ||
| PCL_SUBSYS_DEPEND(build "${SUBSYS_NAME}" DEPS ${SUBSYS_DEPS}) | ||
| if(NOT build) | ||
| return() | ||
| endif() | ||
|
|
||
| find_package(benchmark REQUIRED) | ||
| add_custom_target(run_benchmarks) | ||
|
|
||
| PCL_ADD_BENCHMARK(features LINK_WITH pcl_io pcl_search pcl_features ARGUMENTS "${PCL_SOURCE_DIR}/test/table_scene_mug_stereo_textured.pcd") |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| #include <benchmark/benchmark.h> | ||
| #include <pcl/features/normal_3d.h> // for NormalEstimation | ||
| #ifdef _OPENMP | ||
| #include <pcl/features/normal_3d_omp.h> // for NormalEstimationOMP | ||
| #endif | ||
| #include <pcl/io/pcd_io.h> // for PCDReader | ||
|
|
||
| static void BM_NormalEstimation(benchmark::State& state, const std::string& file) { | ||
| // Perform setup here | ||
| pcl::PointCloud<pcl::PointXYZ>::Ptr cloud (new pcl::PointCloud<pcl::PointXYZ>); | ||
| pcl::PCDReader reader; | ||
| reader.read (file, *cloud); | ||
| pcl::NormalEstimation<pcl::PointXYZ, pcl::Normal> ne; | ||
| ne.setInputCloud (cloud); | ||
| ne.setKSearch (100); | ||
| pcl::PointCloud<pcl::Normal>::Ptr cloud_normals (new pcl::PointCloud<pcl::Normal>); | ||
| for (auto _ : state) { | ||
| // This code gets timed | ||
| ne.compute (*cloud_normals); | ||
| } | ||
| } | ||
| #ifdef _OPENMP | ||
| static void BM_NormalEstimationOMP(benchmark::State& state, const std::string& file) { | ||
| // Perform setup here | ||
| pcl::PointCloud<pcl::PointXYZ>::Ptr cloud(new pcl::PointCloud<pcl::PointXYZ>); | ||
| pcl::PCDReader reader; | ||
| reader.read(file, *cloud); | ||
| pcl::NormalEstimationOMP<pcl::PointXYZ, pcl::Normal> ne; | ||
| ne.setInputCloud(cloud); | ||
| ne.setKSearch(100); | ||
| pcl::PointCloud<pcl::Normal>::Ptr cloud_normals(new pcl::PointCloud<pcl::Normal>); | ||
| for (auto _ : state) { | ||
| // This code gets timed | ||
| ne.compute(*cloud_normals); | ||
| } | ||
| } | ||
| #endif | ||
|
|
||
| int main(int argc, char** argv) { | ||
| if (argc < 2) | ||
| { | ||
| std::cerr << "No test file given. Please download `table_scene_mug_stereo_textured.pcd` and pass its path to the test." << std::endl; | ||
| return (-1); | ||
| } | ||
| benchmark::RegisterBenchmark("BM_NormalEstimation", BM_NormalEstimation, argv[1])->Unit(benchmark::kMillisecond); | ||
| #ifdef _OPENMP | ||
| benchmark::RegisterBenchmark("BM_NormalEstimationOMP", BM_NormalEstimationOMP, argv[1])->Unit(benchmark::kMillisecond); | ||
| #endif | ||
| benchmark::Initialize(&argc, argv); | ||
| benchmark::RunSpecifiedBenchmarks(); | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -411,6 +411,45 @@ macro(PCL_ADD_TEST _name _exename) | |
| add_dependencies(tests ${_exename}) | ||
| endmacro() | ||
|
|
||
| ############################################################################### | ||
| # Add a benchmark target. | ||
| # _name The benchmark name. | ||
| # _exename The exe name. | ||
| # ARGN : | ||
| # ARGUMENTS Arguments for benchmark executable | ||
| # LINK_WITH link benchmark executable with libraries | ||
| macro(PCL_ADD_BENCHMARK _name) | ||
| set(options) | ||
| set(oneValueArgs) | ||
| set(multiValueArgs ARGUMENTS LINK_WITH) | ||
| cmake_parse_arguments(PCL_ADD_BENCHMARK "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) | ||
|
|
||
| add_executable(benchmark_${_name} ${_name}.cpp) | ||
| set_target_properties(benchmark_${_name} PROPERTIES FOLDER "Benchmarks") | ||
| target_link_libraries(benchmark_${_name} benchmark::benchmark ${PCL_ADD_BENCHMARK_LINK_WITH}) | ||
| set_target_properties(benchmark_${_name} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) | ||
|
|
||
| #Only applies to MSVC | ||
| if(MSVC) | ||
| #Requires CMAKE version 3.13.0 | ||
| if(CMAKE_VERSION VERSION_LESS "3.13.0" AND (NOT ArgumentWarningShown)) | ||
|
There was a problem hiding this comment.
There was a problem hiding this comment. Well, it works if its a macro, so its inserted in However, if its changed to a function, we probably should change it to a global variable. There was a problem hiding this comment.
No, this our main CMakeLists will not see this variable. Time for a short explanation. Example 1Let's asume we have following CMake files: benchmark/CMakeList.txt PCL_ADD_BENCHMARK(features_normal_3d FILES ...)
add_subdirectory(another_dir_with_benachmark)
add_subdirectory(another_dir_with_benachmark2)benchmark/another_dir_with_benachmark/CMakeList.txt PCL_ADD_BENCHMARK(another_benchmark FILES ...)benchmark/another_dir_with_benachmark2/CMakeList.txt PCL_ADD_BENCHMARK(another_benchmark2 FILES ...)This works as you want Example 1 - Explanationbenchmark/CMakeList.txt # ArgumentWarningShown is not set until now => PCL_ADD_BENCHMARK will show a warning
PCL_ADD_BENCHMARK(features_normal_3d FILES ...)
# ArgumentWarningShown is now TRUE after calling PCL_ADD_BENCHMARK
add_subdirectory(another_dir_with_benachmark)
add_subdirectory(another_dir_with_benachmark2)benchmark/another_dir_with_benachmark/CMakeList.txt # ArgumentWarningShown is TRUE like in parent CMakeLists => no warning
PCL_ADD_BENCHMARK(another_benchmark FILES ...)
#benchmark/another_dir_with_benachmark2/CMakeList.txt # ArgumentWarningShown is TRUE like in parent CMakeLists => no warning
PCL_ADD_BENCHMARK(another_benchmark2 FILES ...)Example 2Now let us a modify the snippet a little bit, by moving first benchmark/CMakeList.txt add_subdirectory(another_dir_with_benachmark)
add_subdirectory(another_dir_with_benachmark2)
PCL_ADD_BENCHMARK(features_normal_3d FILES ...)All other files are equal And now you are getting the warning 3 times. Why? See below Example 2 - Explanationbenchmark/CMakeList.txt # ArgumentWarningShown is not set until now
add_subdirectory(another_dir_with_benachmark)
add_subdirectory(another_dir_with_benachmark2)
# ArgumentWarningShown is not set until now, as changes from child scope are not injected into parent scope => PCL_ADD_BENCHMARK will show a warning
PCL_ADD_BENCHMARK(features_normal_3d FILES ...)benchmark/another_dir_with_benachmark/CMakeList.txt # ArgumentWarningShown is not set until now => PCL_ADD_BENCHMARK will show a warning
PCL_ADD_BENCHMARK(another_benchmark FILES ...)
# ArgumentWarningShown is now TRUE after calling PCL_ADD_BENCHMARK
#benchmark/another_dir_with_benachmark2/CMakeList.txt # ArgumentWarningShown is not set until now => PCL_ADD_BENCHMARK will show a warning
PCL_ADD_BENCHMARK(another_benchmark2 FILES ...)
# ArgumentWarningShown is now TRUE after calling PCL_ADD_BENCHMARKConclusionCalling Nevertheless it is possible in CMake to modify parent scope:
As the variable set_target_properties(ArgumentWarningShown run_benchmark PCL_ARGUMENTS_WARNING_SHOWN)
if(NOT ArgumentWarningShown)
..
set_target_properties(run_benchmark PROPERTIES PCL_ARGUMENTS_WARNING_SHOWN TRUE)
endif()There was a problem hiding this comment. Yes, thats true if its organized differently as first intended, it will stop working. Thanks for the thorough explanation. There was a problem hiding this comment. So, macros are basically evil in almost all languages. (:crab:) There was a problem hiding this comment. @SunBlack I assume you mean |
||
| message(WARNING "Arguments for unit test projects are not added - this requires at least CMake 3.13. Can be added manually in \"Project settings -> Debugging -> Command arguments\"") | ||
mvieth marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| SET (ArgumentWarningShown TRUE PARENT_SCOPE) | ||
larshg marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| else() | ||
| #Only add if there are arguments to test | ||
| if(PCL_ADD_BENCHMARK_ARGUMENTS) | ||
kunaltyagi marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| string (REPLACE ";" " " PCL_ADD_BENCHMARK_ARGUMENTS_STR "${PCL_ADD_BENCHMARK_ARGUMENTS}") | ||
| set_target_properties(benchmark_${_name} PROPERTIES VS_DEBUGGER_COMMAND_ARGUMENTS ${PCL_ADD_BENCHMARK_ARGUMENTS_STR}) | ||
| endif() | ||
| endif() | ||
| endif() | ||
|
|
||
| add_custom_target(run_benchmark_${_name} benchmark_${_name} ${PCL_ADD_BENCHMARK_ARGUMENTS}) | ||
| set_target_properties(run_benchmark_${_name} PROPERTIES FOLDER "Benchmarks") | ||
|
|
||
| add_dependencies(run_benchmarks run_benchmark_${_name}) | ||
| endmacro() | ||
|
|
||
| ############################################################################### | ||
| # Add an example target. | ||
| # _name The example name. | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this give satisfactory graphs? Or does the compiler optimize the loop to no-op?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I'm not sure if this is the same, but the method is only run for one iteration on the CI:
https://dev.azure.com/PointCloudLibrary/pcl/_build/results?buildId=17871&view=logs&jobId=cd6332f9-d180-53f5-8d9a-3b5d9c3252a2&j=cd6332f9-d180-53f5-8d9a-3b5d9c3252a2&t=7e929b77-dd35-56c3-fbaf-db0044c91b15
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The input file is too large for the benchmark to run it multiple times
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I just tried with the bun0.pcd and here it runs for about 224 times - so it just seems to be saturated when using 2.5 sec for a benchmark, ie using table_scene_mug_stereo_textured.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Saw this after my own test 😄
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It is not optimized away, I assume that the
for (auto _ : state)has some hidden do-not-optimize-me-magic.I think it is possible to request it to run more than once, or for a minimum time, if you think once is not enough.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
More runs are better since they remove the noise due to ambiguousness of the new cpu architectures (and client sharing on the cloud )
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That is true. The number of repetitions can be set for each benchmark individually in the code by calling
Repetitions, or for all benchmarks in one executable with--benchmark_repetitions=x. I would suggest the latter, since more flexible (adding it toARGUMENTSofPCL_ADD_BENCHMARK)