From 89e3082078e5d31524d63f99d61ade5b51636ae8 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 9 Nov 2022 16:13:26 -0800 Subject: [PATCH 01/14] Add sklearn to tox --- newrelic/config.py | 131 +++++++++++++++++++++++++++++++++++++++++++++ tox.ini | 3 ++ 2 files changed, 134 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index f0b638cd4e..335339d86e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2790,6 +2790,137 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") + _process_module_definition( + "sklearn.base", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_base_models", + ) + _process_module_definition( + "sklearn.calibration", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_calibration_models", + ) + _process_module_definition( + "sklearn.cluster", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_cluster_models", + ) + _process_module_definition( + "sklearn.compose", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_compose_models", + ) + _process_module_definition( + "sklearn.covariance", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_covariance_models", + ) + _process_module_definition( + "sklearn.cross_decomposition", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_cross_decomposition_models", + ) + _process_module_definition( + "sklearn.discriminant_analysis", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_discriminant_analysis_models", + ) + _process_module_definition( + "sklearn.dummy", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_dummy_models", + ) + _process_module_definition( + "sklearn.ensemble", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_ensemble_models", + ) + _process_module_definition( + "sklearn.feature_selection", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_feature_selection_models", + ) + _process_module_definition( + "sklearn.gaussian_process", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_gaussian_process_models", + ) + _process_module_definition( + "sklearn.isotonic", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_isotonic_models", + ) + _process_module_definition( + "sklearn.kernel_ridge", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_kernel_ridge_models", + ) + _process_module_definition( + "sklearn.linear_model", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_linear_model_models", + ) + _process_module_definition( + "sklearn.metrics", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_metrics_models", + ) + _process_module_definition( + "sklearn.mixture", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_mixture_models", + ) + _process_module_definition( + "sklearn.model_selection", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_model_selection_models", + ) + _process_module_definition( + "sklearn.multiclass", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_multiclass_models", + ) + _process_module_definition( + "sklearn.multioutput", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_multioutput_models", + ) + _process_module_definition( + "sklearn.naive_bayes", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_naive_bayes_models", + ) + _process_module_definition( + "sklearn.neighbors", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_neighbors_models", + ) + _process_module_definition( + "sklearn.neural_network", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_neural_network_models", + ) + _process_module_definition( + "sklearn.pipeline", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_pipeline_models", + ) + _process_module_definition( + "sklearn.semi_supervised", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_semi_supervised_models", + ) + _process_module_definition( + "sklearn.svm", + "newrelic.hooks.framework_sklearn", + "instrument_sklearn_svm_models", + ) + _process_module_definition( + "sklearn.tree._classes", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_tree_models", + ) + _process_module_definition( "rest_framework.views", "newrelic.hooks.component_djangorestframework", diff --git a/tox.ini b/tox.ini index c815defc27..732f1dda2f 100644 --- a/tox.ini +++ b/tox.ini @@ -60,6 +60,7 @@ envlist = python-agent_unittests-{pypy,pypy37}-without_extensions, python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, + python-component_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -203,6 +204,7 @@ deps = application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 + component_sklearn-scikitlearnlatest: scikit-learn component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -431,6 +433,7 @@ changedir = agent_unittests: tests/agent_unittests application_celery: tests/application_celery application_gearman: tests/application_gearman + component_sklearn: tests/component_sklearn component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 2a9ce60bc6a50b1cd9d613650bf6f36bc4ddcdc9 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 9 Nov 2022 16:14:03 -0800 Subject: [PATCH 02/14] Add function traces around model methods --- newrelic/config.py | 125 -------------------- newrelic/hooks/component_sklearn.py | 67 +++++++++++ tests/component_sklearn/conftest.py | 38 ++++++ tests/component_sklearn/test_tree_models.py | 92 ++++++++++++++ 4 files changed, 197 insertions(+), 125 deletions(-) create mode 100644 newrelic/hooks/component_sklearn.py create mode 100644 tests/component_sklearn/conftest.py create mode 100644 tests/component_sklearn/test_tree_models.py diff --git a/newrelic/config.py b/newrelic/config.py index 335339d86e..e7eb69e98a 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2790,131 +2790,6 @@ def _process_module_builtin_defaults(): ) _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") - _process_module_definition( - "sklearn.base", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_base_models", - ) - _process_module_definition( - "sklearn.calibration", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_calibration_models", - ) - _process_module_definition( - "sklearn.cluster", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_cluster_models", - ) - _process_module_definition( - "sklearn.compose", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_compose_models", - ) - _process_module_definition( - "sklearn.covariance", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_covariance_models", - ) - _process_module_definition( - "sklearn.cross_decomposition", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_cross_decomposition_models", - ) - _process_module_definition( - "sklearn.discriminant_analysis", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_discriminant_analysis_models", - ) - _process_module_definition( - "sklearn.dummy", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_dummy_models", - ) - _process_module_definition( - "sklearn.ensemble", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_ensemble_models", - ) - _process_module_definition( - "sklearn.feature_selection", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_feature_selection_models", - ) - _process_module_definition( - "sklearn.gaussian_process", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_gaussian_process_models", - ) - _process_module_definition( - "sklearn.isotonic", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_isotonic_models", - ) - _process_module_definition( - "sklearn.kernel_ridge", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_kernel_ridge_models", - ) - _process_module_definition( - "sklearn.linear_model", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_linear_model_models", - ) - _process_module_definition( - "sklearn.metrics", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_metrics_models", - ) - _process_module_definition( - "sklearn.mixture", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_mixture_models", - ) - _process_module_definition( - "sklearn.model_selection", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_model_selection_models", - ) - _process_module_definition( - "sklearn.multiclass", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_multiclass_models", - ) - _process_module_definition( - "sklearn.multioutput", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_multioutput_models", - ) - _process_module_definition( - "sklearn.naive_bayes", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_naive_bayes_models", - ) - _process_module_definition( - "sklearn.neighbors", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_neighbors_models", - ) - _process_module_definition( - "sklearn.neural_network", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_neural_network_models", - ) - _process_module_definition( - "sklearn.pipeline", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_pipeline_models", - ) - _process_module_definition( - "sklearn.semi_supervised", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_semi_supervised_models", - ) - _process_module_definition( - "sklearn.svm", - "newrelic.hooks.framework_sklearn", - "instrument_sklearn_svm_models", - ) _process_module_definition( "sklearn.tree._classes", "newrelic.hooks.component_sklearn", diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py new file mode 100644 index 0000000000..03ab6eb534 --- /dev/null +++ b/newrelic/hooks/component_sklearn.py @@ -0,0 +1,67 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.function_trace import FunctionTraceWrapper +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper + + +def wrap_model_method(method_name): + @function_wrapper + def _wrap_method(wrapped, instance, args, kwargs): + # If there is no transaction, do not wrap anything. + if not current_transaction(): + return wrapped(*args, **kwargs) + + # If the method has already been wrapped do not wrap it again. This happens + # when one model inherits from another and they both implement the method. + if getattr(instance, "_nr_wrapped_%s" % method_name, False): + return wrapped(*args, **kwargs) + + # Set the _nr_wrapped attribute to denote that this method has now been wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, True) + + # MLModel/Sklearn/Named/. + func_name = wrapped.__name__ + name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) + return FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + + return _wrap_method + + +def wrap_model_init(wrapped, instance, args, kwargs): + return_val = wrapped(*args, **kwargs) + + methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") + for method_name in methods_to_wrap: + if hasattr(instance, method_name): + setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) + + return return_val + + +def _nr_instrument_model(module, model_class): + wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) + + +def instrument_sklearn_tree_models(module): + model_classes = ( + "DecisionTreeClassifier", + "DecisionTreeRegressor", + "ExtraTreeClassifier", + "ExtraTreeRegressor", + ) + for model_cls in model_classes: + if hasattr(module, model_cls): + _nr_instrument_model(module, model_cls) diff --git a/tests/component_sklearn/conftest.py b/tests/component_sklearn/conftest.py new file mode 100644 index 0000000000..e251d91bb7 --- /dev/null +++ b/tests/component_sklearn/conftest.py @@ -0,0 +1,38 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + code_coverage_fixture, + collector_agent_registration_fixture, + collector_available_fixture, +) + +_coverage_source = [ + "newrelic.hooks.component_sklearn", +] + +code_coverage = code_coverage_fixture(source=_coverage_source) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (component_sklearn)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (component_sklearn)"], +) diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py new file mode 100644 index 0000000000..235e161e03 --- /dev/null +++ b/tests/component_sklearn/test_tree_models.py @@ -0,0 +1,92 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.packages import six + + +def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), + ], + "DecisionTreeClassifier": [ + ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 1), + ], + "ExtraTreeClassifier": [ + ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 1), + ], + "DecisionTreeRegressor": [ + ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), + ], + } + expected_transaction_name = "test_tree_models:_test" + if six.PY3: + expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + run_tree_model() + + _test() + + +@pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) +def tree_model_name(request): + return request.param + + +@pytest.fixture +def run_tree_model(tree_model_name): + def _run(): + import sklearn.tree + + x_train = [[0, 0], [1, 1]] + y_train = [0, 1] + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + clf = getattr(sklearn.tree, tree_model_name)(random_state=0) + model = clf.fit(x_train, y_train) + + labels = model.predict(x_test) + model.score(x_test, y_test) + # Only classifier models have proba methods. + if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): + model.predict_log_proba(x_test) + model.predict_proba(x_test) + + return _run From e954532532bf62d0080baf6367de4ebf10511b85 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 15 Nov 2022 16:36:55 -0800 Subject: [PATCH 03/14] Support Python 2.7 & 3.7 sklearn --- newrelic/config.py | 5 +++++ tox.ini | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e7eb69e98a..439305a646 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2795,6 +2795,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.component_sklearn", "instrument_sklearn_tree_models", ) + _process_module_definition( + "sklearn.tree.tree", + "newrelic.hooks.component_sklearn", + "instrument_sklearn_tree_models", + ) _process_module_definition( "rest_framework.views", diff --git a/tox.ini b/tox.ini index 732f1dda2f..d0726affcb 100644 --- a/tox.ini +++ b/tox.ini @@ -61,6 +61,8 @@ envlist = python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, python-component_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, + python-component_sklearn-{py37,pypy37}-scikitlearn101, + python-component_sklearn-{py27,pypy27}-scikitlearn020, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -205,6 +207,8 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_sklearn-scikitlearnlatest: scikit-learn + component_sklearn-scikitlearn020: scikit-learn < 0.21 + component_sklearn-scikitlearn101: scikit-learn < 1.1 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django From 6e5a9da45c220e22045f592ac2c95acd35b8b7e3 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 16 Nov 2022 15:24:58 -0800 Subject: [PATCH 04/14] Add test for multiple calls to model method --- newrelic/hooks/component_sklearn.py | 9 ++- tests/component_sklearn/test_tree_models.py | 68 +++++++++++++++++++-- 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index 03ab6eb534..fd740ef87a 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -29,13 +29,18 @@ def _wrap_method(wrapped, instance, args, kwargs): if getattr(instance, "_nr_wrapped_%s" % method_name, False): return wrapped(*args, **kwargs) - # Set the _nr_wrapped attribute to denote that this method has now been wrapped. + # Set the _nr_wrapped attribute to denote that this method is being wrapped. setattr(instance, "_nr_wrapped_%s" % method_name, True) # MLModel/Sklearn/Named/. func_name = wrapped.__name__ name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) - return FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + return_val = FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + + # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, False) + + return return_val return _wrap_method diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 235e161e03..2ba2e88629 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -25,26 +25,26 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 2), ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), ], "DecisionTreeClassifier": [ ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 2), ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 2), ], "ExtraTreeClassifier": [ ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 2), ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 2), ], "DecisionTreeRegressor": [ ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 2), ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), ], } @@ -55,6 +55,7 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model @validate_transaction_metrics( expected_transaction_name, scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], background_task=True, ) @background_task() @@ -64,6 +65,60 @@ def _test(): _test() +def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): + expected_scoped_metrics = { + "ExtraTreeRegressor": [ + ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 4), + ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 2), + ], + "DecisionTreeClassifier": [ + ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 4), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 2), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 2), + ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 4), + ], + "ExtraTreeClassifier": [ + ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 4), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 2), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 2), + ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 4), + ], + "DecisionTreeRegressor": [ + ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 4), + ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 2), + ], + } + expected_transaction_name = "test_tree_models:_test" + if six.PY3: + expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + + @validate_transaction_metrics( + expected_transaction_name, + scoped_metrics=expected_scoped_metrics[tree_model_name], + rollup_metrics=expected_scoped_metrics[tree_model_name], + background_task=True, + ) + @background_task() + def _test(): + x_test = [[2.0, 2.0], [2.0, 1.0]] + y_test = [1, 1] + + model = run_tree_model() + + model.predict(x_test) + model.score(x_test, y_test) + # Only classifier models have proba methods. + if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): + model.predict_log_proba(x_test) + model.predict_proba(x_test) + + _test() + + @pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) def tree_model_name(request): return request.param @@ -88,5 +143,6 @@ def _run(): if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): model.predict_log_proba(x_test) model.predict_proba(x_test) + return model return _run From f5568cad3dfb050223a9f3da18dafcdb757d0740 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 16 Nov 2022 17:57:52 -0800 Subject: [PATCH 05/14] Fixup: add comments & organize --- newrelic/config.py | 1 + tests/component_sklearn/test_tree_models.py | 8 +++++++- tox.ini | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 439305a646..c8661055b3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -2795,6 +2795,7 @@ def _process_module_builtin_defaults(): "newrelic.hooks.component_sklearn", "instrument_sklearn_tree_models", ) + # In scikit-learn < 0.21 the model classes are in tree.py instead of _classes.py. _process_module_definition( "sklearn.tree.tree", "newrelic.hooks.component_sklearn", diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 2ba2e88629..8499f5177e 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -22,6 +22,9 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice instead of + # once like the rest of the methods. expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), @@ -66,6 +69,9 @@ def _test(): def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): + # Note: in the following expected metrics, predict and predict_proba are called by + # score and predict_log_proba so they are expected to be called twice as often as + # the other methods. expected_scoped_metrics = { "ExtraTreeRegressor": [ ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), @@ -94,7 +100,7 @@ def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): } expected_transaction_name = "test_tree_models:_test" if six.PY3: - expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + expected_transaction_name = "test_tree_models:test_multiple_calls_to_model_methods.._test" @validate_transaction_metrics( expected_transaction_name, diff --git a/tox.ini b/tox.ini index d0726affcb..8fec115725 100644 --- a/tox.ini +++ b/tox.ini @@ -207,8 +207,8 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_sklearn-scikitlearnlatest: scikit-learn - component_sklearn-scikitlearn020: scikit-learn < 0.21 component_sklearn-scikitlearn101: scikit-learn < 1.1 + component_sklearn-scikitlearn020: scikit-learn < 0.21 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django From c147ce3afb2412f2d7c8c64037a1b8920d52e7e9 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 23 Nov 2022 16:48:47 -0800 Subject: [PATCH 06/14] Refactor --- newrelic/hooks/component_sklearn.py | 58 +++++++++++++---------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index fd740ef87a..7a27a5ae95 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -12,52 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.function_trace import FunctionTraceWrapper +from newrelic.api.function_trace import FunctionTrace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.object_wrapper import wrap_function_wrapper -def wrap_model_method(method_name): - @function_wrapper - def _wrap_method(wrapped, instance, args, kwargs): - # If there is no transaction, do not wrap anything. - if not current_transaction(): - return wrapped(*args, **kwargs) +def wrap_method(wrapped, instance, args, kwargs): + # If there is no transaction, do not wrap anything. + if not current_transaction(): + return wrapped(*args, **kwargs) - # If the method has already been wrapped do not wrap it again. This happens - # when one model inherits from another and they both implement the method. - if getattr(instance, "_nr_wrapped_%s" % method_name, False): - return wrapped(*args, **kwargs) + method_name = wrapped.__name__ - # Set the _nr_wrapped attribute to denote that this method is being wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, True) + # If the method has already been wrapped do not wrap it again. This happens + # when one model inherits from another and they both implement the method. + if getattr(instance, "_nr_wrapped_%s" % method_name, False): + return wrapped(*args, **kwargs) - # MLModel/Sklearn/Named/. - func_name = wrapped.__name__ - name = "%s.%s" % (wrapped.__self__.__class__.__name__, func_name) - return_val = FunctionTraceWrapper(wrapped, name=name, group="MLModel/Sklearn/Named")(*args, **kwargs) + # Set the _nr_wrapped attribute to denote that this method is being wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, True) - # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, False) + # MLModel/Sklearn/Named/. + func_name = wrapped.__name__ + model_name = wrapped.__self__.__class__.__name__ + name = "%s.%s" % (model_name, func_name) + with FunctionTrace(name=name, group="MLModel/Sklearn/Named", source=wrapped): + return_val = wrapped(*args, **kwargs) - return return_val - - return _wrap_method - - -def wrap_model_init(wrapped, instance, args, kwargs): - return_val = wrapped(*args, **kwargs) - - methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") - for method_name in methods_to_wrap: - if hasattr(instance, method_name): - setattr(instance, method_name, wrap_model_method(method_name)(getattr(instance, method_name))) + # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. + setattr(instance, "_nr_wrapped_%s" % method_name, False) return return_val def _nr_instrument_model(module, model_class): - wrap_function_wrapper(module, "%s.%s" % (model_class, "__init__"), wrap_model_init) + methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") + for method_name in methods_to_wrap: + if hasattr(getattr(module, model_class), method_name): + wrap_function_wrapper(module, "%s.%s" % (model_class, method_name), wrap_method) def instrument_sklearn_tree_models(module): From b038f21c51f1a247326efa920d9dd09866575e40 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 30 Nov 2022 12:52:26 -0800 Subject: [PATCH 07/14] Follow two digit convention --- tox.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tox.ini b/tox.ini index 8fec115725..112ccee06e 100644 --- a/tox.ini +++ b/tox.ini @@ -61,8 +61,8 @@ envlist = python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, gearman-application_gearman-{py27,pypy}, python-component_sklearn-{py38,py39,py310,py311}-scikitlearnlatest, - python-component_sklearn-{py37,pypy37}-scikitlearn101, - python-component_sklearn-{py27,pypy27}-scikitlearn020, + python-component_sklearn-{py37,pypy37}-scikitlearn0101, + python-component_sklearn-{py27,pypy27}-scikitlearn0020, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, @@ -207,8 +207,8 @@ deps = application_celery-py{py37,37}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_sklearn-scikitlearnlatest: scikit-learn - component_sklearn-scikitlearn101: scikit-learn < 1.1 - component_sklearn-scikitlearn020: scikit-learn < 0.21 + component_sklearn-scikitlearn0101: scikit-learn < 1.1 + component_sklearn-scikitlearn0020: scikit-learn < 0.21 component_djangorestframework-djangorestframework0300: Django < 1.9 component_djangorestframework-djangorestframework0300: djangorestframework < 3.1 component_djangorestframework-djangorestframeworklatest: Django From 032ba7da9aec68f81c0b3dcc56d3fd678b19c888 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 30 Nov 2022 12:55:10 -0800 Subject: [PATCH 08/14] Make if-else a one-liner --- tests/component_sklearn/test_tree_models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py index 8499f5177e..de5408ab97 100644 --- a/tests/component_sklearn/test_tree_models.py +++ b/tests/component_sklearn/test_tree_models.py @@ -51,9 +51,11 @@ def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), ], } - expected_transaction_name = "test_tree_models:_test" - if six.PY3: - expected_transaction_name = "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + expected_transaction_name = ( + "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" + if six.PY3 + else "test_tree_models:_test" + ) @validate_transaction_metrics( expected_transaction_name, From e72afd9d185aaa4f95bf8dfff49c488fd99349ad Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Wed, 30 Nov 2022 15:40:18 -0800 Subject: [PATCH 09/14] Abstract to re-usable instrumentation function --- newrelic/hooks/component_sklearn.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py index 7a27a5ae95..e4c3655b15 100644 --- a/newrelic/hooks/component_sklearn.py +++ b/newrelic/hooks/component_sklearn.py @@ -52,6 +52,12 @@ def _nr_instrument_model(module, model_class): wrap_function_wrapper(module, "%s.%s" % (model_class, method_name), wrap_method) +def _instrument_sklearn_models(module, model_classes): + for model_cls in model_classes: + if hasattr(module, model_cls): + _nr_instrument_model(module, model_cls) + + def instrument_sklearn_tree_models(module): model_classes = ( "DecisionTreeClassifier", @@ -59,6 +65,4 @@ def instrument_sklearn_tree_models(module): "ExtraTreeClassifier", "ExtraTreeRegressor", ) - for model_cls in model_classes: - if hasattr(module, model_cls): - _nr_instrument_model(module, model_cls) + _instrument_sklearn_models(module, model_classes) From ec6012f745d5bd12bac4d407ff876c6673670ee4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Dec 2022 14:48:26 -0800 Subject: [PATCH 10/14] Add ML inference event capture config setting. --- newrelic/config.py | 5 +++++ newrelic/core/agent_protocol.py | 1 + newrelic/core/config.py | 13 +++++++++++++ 3 files changed, 19 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index c8661055b3..cbb2f8baa4 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -539,6 +539,7 @@ def _process_configuration(section): _process_setting(section, "application_logging.metrics.enabled", "getboolean", None) _process_setting(section, "application_logging.local_decorating.enabled", "getboolean", None) + _process_setting(section, "machine_learning.inference_event_value.enabled", "getboolean", None) # Loading of configuration from specified file and for specified # deployment environment. Can also indicate whether configuration @@ -875,6 +876,10 @@ def apply_local_high_security_mode_setting(settings): settings.application_logging.forwarding.enabled = False _logger.info(log_template, "application_logging.forwarding.enabled", True, False) + if settings.machine_learning.inference_event_value.enabled: + settings.machine_learning.inference_event_value.enabled = False + _logger.info(log_template, "machine_learning.inference_event_value.enabled", True, False) + return settings diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index ba277d4de1..c5ac95e237 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -144,6 +144,7 @@ class AgentProtocol(object): "strip_exception_messages.enabled", "custom_insights_events.enabled", "application_logging.forwarding.enabled", + "machine_learning.inference_event_value.enabled", ) LOGGER_FUNC_MAPPING = { diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 4111c71495..edc7e820d6 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -121,6 +121,14 @@ class GCRuntimeMetricsSettings(Settings): enabled = False +class MachineLearningSettings(Settings): + pass + + +class MachineLearningInferenceEventValueSettings(Settings): + pass + + class CodeLevelMetricsSettings(Settings): pass @@ -359,6 +367,8 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.application_logging.forwarding = ApplicationLoggingForwardingSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() +_settings.machine_learning = MachineLearningSettings() +_settings.machine_learning.inference_event_value = MachineLearningInferenceEventValueSettings() _settings.attributes = AttributesSettings() _settings.gc_runtime_metrics = GCRuntimeMetricsSettings() _settings.code_level_metrics = CodeLevelMetricsSettings() @@ -821,6 +831,9 @@ def default_host(license_key): _settings.application_logging.local_decorating.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.machine_learning.inference_event_value.enabled = _environ_as_bool( + "NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=True +) def global_settings(): From 764bf8096ddb3f8bb42adcc2f813ee47d6e6835d Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Tue, 6 Dec 2022 00:57:55 +0000 Subject: [PATCH 11/14] [Mega-Linter] Apply linters fixes --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index 0eaa97bfaf..b4b752a6e9 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -541,6 +541,7 @@ def _process_configuration(section): _process_setting(section, "machine_learning.inference_event_value.enabled", "getboolean", None) + # Loading of configuration from specified file and for specified # deployment environment. Can also indicate whether configuration # and instrumentation errors should raise an exception or not. From 6cc4111b81588a678756eeec74816ac97c64df14 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Tue, 6 Dec 2022 12:08:05 -0800 Subject: [PATCH 12/14] Fixup: remove component_sklearn files --- newrelic/hooks/component_sklearn.py | 68 --------- tests/component_sklearn/conftest.py | 38 ----- tests/component_sklearn/test_tree_models.py | 156 -------------------- 3 files changed, 262 deletions(-) delete mode 100644 newrelic/hooks/component_sklearn.py delete mode 100644 tests/component_sklearn/conftest.py delete mode 100644 tests/component_sklearn/test_tree_models.py diff --git a/newrelic/hooks/component_sklearn.py b/newrelic/hooks/component_sklearn.py deleted file mode 100644 index e4c3655b15..0000000000 --- a/newrelic/hooks/component_sklearn.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.api.function_trace import FunctionTrace -from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper - - -def wrap_method(wrapped, instance, args, kwargs): - # If there is no transaction, do not wrap anything. - if not current_transaction(): - return wrapped(*args, **kwargs) - - method_name = wrapped.__name__ - - # If the method has already been wrapped do not wrap it again. This happens - # when one model inherits from another and they both implement the method. - if getattr(instance, "_nr_wrapped_%s" % method_name, False): - return wrapped(*args, **kwargs) - - # Set the _nr_wrapped attribute to denote that this method is being wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, True) - - # MLModel/Sklearn/Named/. - func_name = wrapped.__name__ - model_name = wrapped.__self__.__class__.__name__ - name = "%s.%s" % (model_name, func_name) - with FunctionTrace(name=name, group="MLModel/Sklearn/Named", source=wrapped): - return_val = wrapped(*args, **kwargs) - - # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. - setattr(instance, "_nr_wrapped_%s" % method_name, False) - - return return_val - - -def _nr_instrument_model(module, model_class): - methods_to_wrap = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") - for method_name in methods_to_wrap: - if hasattr(getattr(module, model_class), method_name): - wrap_function_wrapper(module, "%s.%s" % (model_class, method_name), wrap_method) - - -def _instrument_sklearn_models(module, model_classes): - for model_cls in model_classes: - if hasattr(module, model_cls): - _nr_instrument_model(module, model_cls) - - -def instrument_sklearn_tree_models(module): - model_classes = ( - "DecisionTreeClassifier", - "DecisionTreeRegressor", - "ExtraTreeClassifier", - "ExtraTreeRegressor", - ) - _instrument_sklearn_models(module, model_classes) diff --git a/tests/component_sklearn/conftest.py b/tests/component_sklearn/conftest.py deleted file mode 100644 index e251d91bb7..0000000000 --- a/tests/component_sklearn/conftest.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 - code_coverage_fixture, - collector_agent_registration_fixture, - collector_available_fixture, -) - -_coverage_source = [ - "newrelic.hooks.component_sklearn", -] - -code_coverage = code_coverage_fixture(source=_coverage_source) - -_default_settings = { - "transaction_tracer.explain_threshold": 0.0, - "transaction_tracer.transaction_threshold": 0.0, - "transaction_tracer.stack_trace_threshold": 0.0, - "debug.log_data_collector_payloads": True, - "debug.record_transaction_failure": True, -} -collector_agent_registration = collector_agent_registration_fixture( - app_name="Python Agent Test (component_sklearn)", - default_settings=_default_settings, - linked_applications=["Python Agent Test (component_sklearn)"], -) diff --git a/tests/component_sklearn/test_tree_models.py b/tests/component_sklearn/test_tree_models.py deleted file mode 100644 index de5408ab97..0000000000 --- a/tests/component_sklearn/test_tree_models.py +++ /dev/null @@ -1,156 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -from testing_support.validators.validate_transaction_metrics import ( - validate_transaction_metrics, -) - -from newrelic.api.background_task import background_task -from newrelic.packages import six - - -def test_model_methods_wrapped_in_function_trace(tree_model_name, run_tree_model): - # Note: in the following expected metrics, predict and predict_proba are called by - # score and predict_log_proba so they are expected to be called twice instead of - # once like the rest of the methods. - expected_scoped_metrics = { - "ExtraTreeRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 1), - ], - "DecisionTreeClassifier": [ - ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 2), - ], - "ExtraTreeClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 2), - ], - "DecisionTreeRegressor": [ - ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 2), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 1), - ], - } - expected_transaction_name = ( - "test_tree_models:test_model_methods_wrapped_in_function_trace.._test" - if six.PY3 - else "test_tree_models:_test" - ) - - @validate_transaction_metrics( - expected_transaction_name, - scoped_metrics=expected_scoped_metrics[tree_model_name], - rollup_metrics=expected_scoped_metrics[tree_model_name], - background_task=True, - ) - @background_task() - def _test(): - run_tree_model() - - _test() - - -def test_multiple_calls_to_model_methods(tree_model_name, run_tree_model): - # Note: in the following expected metrics, predict and predict_proba are called by - # score and predict_log_proba so they are expected to be called twice as often as - # the other methods. - expected_scoped_metrics = { - "ExtraTreeRegressor": [ - ("MLModel/Sklearn/Named/ExtraTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.predict", 4), - ("MLModel/Sklearn/Named/ExtraTreeRegressor.score", 2), - ], - "DecisionTreeClassifier": [ - ("MLModel/Sklearn/Named/DecisionTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict", 4), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.score", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_log_proba", 2), - ("MLModel/Sklearn/Named/DecisionTreeClassifier.predict_proba", 4), - ], - "ExtraTreeClassifier": [ - ("MLModel/Sklearn/Named/ExtraTreeClassifier.fit", 1), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict", 4), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.score", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_log_proba", 2), - ("MLModel/Sklearn/Named/ExtraTreeClassifier.predict_proba", 4), - ], - "DecisionTreeRegressor": [ - ("MLModel/Sklearn/Named/DecisionTreeRegressor.fit", 1), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.predict", 4), - ("MLModel/Sklearn/Named/DecisionTreeRegressor.score", 2), - ], - } - expected_transaction_name = "test_tree_models:_test" - if six.PY3: - expected_transaction_name = "test_tree_models:test_multiple_calls_to_model_methods.._test" - - @validate_transaction_metrics( - expected_transaction_name, - scoped_metrics=expected_scoped_metrics[tree_model_name], - rollup_metrics=expected_scoped_metrics[tree_model_name], - background_task=True, - ) - @background_task() - def _test(): - x_test = [[2.0, 2.0], [2.0, 1.0]] - y_test = [1, 1] - - model = run_tree_model() - - model.predict(x_test) - model.score(x_test, y_test) - # Only classifier models have proba methods. - if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): - model.predict_log_proba(x_test) - model.predict_proba(x_test) - - _test() - - -@pytest.fixture(params=["ExtraTreeRegressor", "DecisionTreeClassifier", "ExtraTreeClassifier", "DecisionTreeRegressor"]) -def tree_model_name(request): - return request.param - - -@pytest.fixture -def run_tree_model(tree_model_name): - def _run(): - import sklearn.tree - - x_train = [[0, 0], [1, 1]] - y_train = [0, 1] - x_test = [[2.0, 2.0], [2.0, 1.0]] - y_test = [1, 1] - - clf = getattr(sklearn.tree, tree_model_name)(random_state=0) - model = clf.fit(x_train, y_train) - - labels = model.predict(x_test) - model.score(x_test, y_test) - # Only classifier models have proba methods. - if tree_model_name in ("DecisionTreeClassifier", "ExtraTreeClassifier"): - model.predict_log_proba(x_test) - model.predict_proba(x_test) - return model - - return _run From b421ad2dd04c91878bef12e1699004ecbd8639ef Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Tue, 6 Dec 2022 16:29:25 -0800 Subject: [PATCH 13/14] Add high security mode testing for ML events setting. --- .../agent_features/test_high_security_mode.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index dad7edc295..de17bb189a 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -79,6 +79,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": False, @@ -88,6 +89,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": False, @@ -97,6 +99,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": False, @@ -106,6 +109,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, ] @@ -118,6 +122,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -127,6 +132,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -136,6 +142,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -145,6 +152,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -154,6 +162,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -163,6 +172,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "high_security": True, @@ -172,6 +182,7 @@ def test_hsm_configuration_default(): "custom_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, ] @@ -196,6 +207,7 @@ def test_local_config_file_override_hsm_disabled(settings): original_custom_events = settings.custom_insights_events.enabled original_message_segment_params_enabled = settings.message_tracer.segment_parameters_enabled original_application_logging_forwarding_enabled = settings.application_logging.forwarding.enabled + original_machine_learning_inference_event_value_enabled = settings.machine_learning.inference_event_value.enabled apply_local_high_security_mode_setting(settings) @@ -205,6 +217,7 @@ def test_local_config_file_override_hsm_disabled(settings): assert settings.custom_insights_events.enabled == original_custom_events assert settings.message_tracer.segment_parameters_enabled == original_message_segment_params_enabled assert settings.application_logging.forwarding.enabled == original_application_logging_forwarding_enabled + assert settings.machine_learning.inference_event_value.enabled == original_machine_learning_inference_event_value_enabled @parameterize_hsm_local_config(_hsm_local_config_file_settings_enabled) @@ -217,6 +230,7 @@ def test_local_config_file_override_hsm_enabled(settings): assert settings.custom_insights_events.enabled is False assert settings.message_tracer.segment_parameters_enabled is False assert settings.application_logging.forwarding.enabled is False + assert settings.machine_learning.inference_event_value.enabled is False _server_side_config_settings_hsm_disabled = [ @@ -228,6 +242,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "agent_config": { @@ -236,6 +251,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, }, ), @@ -247,6 +263,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, { "agent_config": { @@ -255,6 +272,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, }, ), @@ -269,6 +287,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -277,12 +296,14 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, "agent_config": { "capture_params": False, "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, }, ), @@ -294,6 +315,7 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, }, { "high_security": True, @@ -302,12 +324,14 @@ def test_local_config_file_override_hsm_enabled(settings): "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_event_value.enabled": False, "agent_config": { "capture_params": True, "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_event_value.enabled": True, }, }, ), @@ -328,6 +352,7 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): original_strip_messages = agent_config["strip_exception_messages.enabled"] original_custom_events = agent_config["custom_insights_events.enabled"] original_log_forwarding = agent_config["application_logging.forwarding.enabled"] + original_machine_learning_events = agent_config["machine_learning.inference_event_value.enabled"] _settings = global_settings() settings = override_generic_settings(_settings, local_settings)(AgentProtocol._apply_high_security_mode_fixups)( @@ -343,6 +368,7 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): assert agent_config["strip_exception_messages.enabled"] == original_strip_messages assert agent_config["custom_insights_events.enabled"] == original_custom_events assert agent_config["application_logging.forwarding.enabled"] == original_log_forwarding + assert agent_config["machine_learning.inference_event_value.enabled"] == original_machine_learning_events @pytest.mark.parametrize("local_settings,server_settings", _server_side_config_settings_hsm_enabled) @@ -365,12 +391,14 @@ def test_remote_config_fixups_hsm_enabled(local_settings, server_settings): assert "strip_exception_messages.enabled" not in settings assert "custom_insights_events.enabled" not in settings assert "application_logging.forwarding.enabled" not in settings + assert "machine_learning.inference_event_value.enabled" not in settings assert "capture_params" not in agent_config assert "transaction_tracer.record_sql" not in agent_config assert "strip_exception_messages.enabled" not in agent_config assert "custom_insights_events.enabled" not in agent_config assert "application_logging.forwarding.enabled" not in agent_config + assert "machine_learning.inference_event_value.enabled" not in agent_config def test_remote_config_hsm_fixups_server_side_disabled(): From 309f7e078124765f99a3652cc01c8dfab17d43c9 Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Wed, 7 Dec 2022 06:29:14 +0000 Subject: [PATCH 14/14] [Mega-Linter] Apply linters fixes --- tests/agent_features/test_high_security_mode.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index de17bb189a..e37caac437 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -217,7 +217,10 @@ def test_local_config_file_override_hsm_disabled(settings): assert settings.custom_insights_events.enabled == original_custom_events assert settings.message_tracer.segment_parameters_enabled == original_message_segment_params_enabled assert settings.application_logging.forwarding.enabled == original_application_logging_forwarding_enabled - assert settings.machine_learning.inference_event_value.enabled == original_machine_learning_inference_event_value_enabled + assert ( + settings.machine_learning.inference_event_value.enabled + == original_machine_learning_inference_event_value_enabled + ) @parameterize_hsm_local_config(_hsm_local_config_file_settings_enabled)