From b9d0f2124b6ab99ee3329c7fb51b1b8db770e0e2 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 16 Nov 2024 02:16:17 +0000 Subject: [PATCH 1/4] update --- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 0aa93bce354e8..52ab2ffa6c341 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -2413,7 +2413,7 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor delete[] ptr->trt_profile_opt_shapes; delete[] ptr->trt_ep_context_file_path; delete[] ptr->trt_onnx_model_folder_path; - if (!ptr->trt_op_types_to_exclude) delete[] ptr->trt_op_types_to_exclude; + if (ptr->trt_op_types_to_exclude) delete[] ptr->trt_op_types_to_exclude; } std::unique_ptr p(ptr); From 765cb7ff80e6595f7677ed8d4f7c0b69d2183240 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Sat, 16 Nov 2024 06:57:05 +0000 Subject: [PATCH 2/4] fix delete issue --- .../core/providers/tensorrt/tensorrt_provider_options.h | 1 + .../providers/tensorrt/tensorrt_execution_provider_info.cc | 5 ++++- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h index 5e5319a34ee9f..95a59d83fbfcd 100644 --- a/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h +++ b/include/onnxruntime/core/providers/tensorrt/tensorrt_provider_options.h @@ -90,4 +90,5 @@ struct OrtTensorRTProviderOptionsV2 { const char* trt_op_types_to_exclude{"NonMaxSuppression,NonZero,RoiAlign"}; // Exclude specific ops from running on TRT. // There is a known performance issue with the DDS ops (NonMaxSuppression, NonZero and RoiAlign) from TRT versions 10.0 to 10.7. // TRT EP excludes DDS ops from running on TRT by default, user can override default value with empty string to include all ops. + int trt_op_types_to_exclude_str_is_dynamic_allocation = 0; // Indicate trt_op_types_to_exclude points to a static allocation or dynamic allocation. It's for internal use to free the dynamic allocation buffer. }; diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index bc0d00ec6791f..f9b5367ccbafa 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -360,6 +360,9 @@ void TensorrtExecutionProviderInfo::UpdateProviderOptions(void* provider_options trt_provider_options_v2.trt_engine_hw_compatible = internal_options.engine_hw_compatible; trt_provider_options_v2.trt_onnx_bytestream = internal_options.onnx_bytestream; trt_provider_options_v2.trt_onnx_bytestream_size = internal_options.onnx_bytestream_size; - trt_provider_options_v2.trt_op_types_to_exclude = copy_string_if_needed(internal_options.op_types_to_exclude); + if (options.find("trt_op_types_to_exclude") != options.end()) { + trt_provider_options_v2.trt_op_types_to_exclude = copy_string_if_needed(internal_options.op_types_to_exclude); + trt_provider_options_v2.trt_op_types_to_exclude_str_is_dynamic_allocation = 1; + } } } // namespace onnxruntime diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 52ab2ffa6c341..b39acac4dd171 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -2413,7 +2413,7 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor delete[] ptr->trt_profile_opt_shapes; delete[] ptr->trt_ep_context_file_path; delete[] ptr->trt_onnx_model_folder_path; - if (ptr->trt_op_types_to_exclude) delete[] ptr->trt_op_types_to_exclude; + if (!ptr->trt_op_types_to_exclude && ptr->trt_op_types_to_exclude_str_is_dynamic_allocation) delete[] ptr->trt_op_types_to_exclude; } std::unique_ptr p(ptr); From e66af212fea25c59ff8dee7cbc19c95814aace1f Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 18 Nov 2024 07:16:09 +0000 Subject: [PATCH 3/4] update --- .../core/providers/tensorrt/tensorrt_execution_provider_info.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc index f9b5367ccbafa..429cc293f5dd8 100644 --- a/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc +++ b/onnxruntime/core/providers/tensorrt/tensorrt_execution_provider_info.cc @@ -362,7 +362,7 @@ void TensorrtExecutionProviderInfo::UpdateProviderOptions(void* provider_options trt_provider_options_v2.trt_onnx_bytestream_size = internal_options.onnx_bytestream_size; if (options.find("trt_op_types_to_exclude") != options.end()) { trt_provider_options_v2.trt_op_types_to_exclude = copy_string_if_needed(internal_options.op_types_to_exclude); - trt_provider_options_v2.trt_op_types_to_exclude_str_is_dynamic_allocation = 1; + if (string_copy) trt_provider_options_v2.trt_op_types_to_exclude_str_is_dynamic_allocation = 1; } } } // namespace onnxruntime From e6c43751bf4dce49d3eeb2828415784c904fffb5 Mon Sep 17 00:00:00 2001 From: Chi Lo Date: Mon, 18 Nov 2024 16:52:33 +0000 Subject: [PATCH 4/4] fix wrong logic --- onnxruntime/core/session/provider_bridge_ort.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index b39acac4dd171..770c93a750e8f 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -2413,7 +2413,7 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor delete[] ptr->trt_profile_opt_shapes; delete[] ptr->trt_ep_context_file_path; delete[] ptr->trt_onnx_model_folder_path; - if (!ptr->trt_op_types_to_exclude && ptr->trt_op_types_to_exclude_str_is_dynamic_allocation) delete[] ptr->trt_op_types_to_exclude; + if (ptr->trt_op_types_to_exclude && ptr->trt_op_types_to_exclude_str_is_dynamic_allocation) delete[] ptr->trt_op_types_to_exclude; } std::unique_ptr p(ptr);