From b5711e381f547dae94ce1018032b506daefe04f9 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Sep 2022 17:38:19 +0530 Subject: [PATCH 001/525] initial changes for k2 integration --- newrelic/admin/record_deploy.py | 2 ++ newrelic/config.py | 6 ++++++ newrelic/core/config.py | 2 ++ newrelic/newrelic.ini | 13 +++++++++++-- setup.py | 1 + 5 files changed, 22 insertions(+), 2 deletions(-) diff --git a/newrelic/admin/record_deploy.py b/newrelic/admin/record_deploy.py index 3a6229de66..c4079b346c 100644 --- a/newrelic/admin/record_deploy.py +++ b/newrelic/admin/record_deploy.py @@ -139,6 +139,8 @@ def _args( settings.monitor_mode = False + settings.k2_enabled = False + initialize(config_file) host = settings.host diff --git a/newrelic/config.py b/newrelic/config.py index 456cd722dc..287be6c542 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,6 +328,7 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) + _process_setting(section, "k2_enabled", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) @@ -3080,6 +3081,11 @@ def initialize( _setup_agent_console() else: _settings.enabled = False + if _settings.k2_enabled: + log_message("Invoking K2 security module.", critical=True) + # run k2 agent + import k2_python_agent + k2_python_agent.init_k2({}, {}) def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 60520c1134..77a12ebcdc 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -578,6 +578,8 @@ def default_host(license_key): _settings.monitor_mode = _environ_as_bool("NEW_RELIC_MONITOR_MODE", True) +_settings.k2_enabled = _environ_as_bool("ENABLE_K2_MODE", True) + _settings.developer_mode = _environ_as_bool("NEW_RELIC_DEVELOPER_MODE", False) _settings.high_security = _environ_as_bool("NEW_RELIC_HIGH_SECURITY", False) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d06d8a2926..84f2d99a13 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -42,12 +42,18 @@ license_key = *** REPLACE ME *** # https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/ app_name = Python Application +# When "true", the agent enables the k2 security module +# This global switch is normally overridden for +# each environment below. It may also be set using the +# ENABLE_K2_MODE environment variable. +monitor_mode = true + # When "true", the agent collects performance data about your # application and reports this data to the New Relic UI at # newrelic.com. This global switch is normally overridden for # each environment below. It may also be set using the # NEW_RELIC_MONITOR_MODE environment variable. -monitor_mode = true +k2_enabled = true # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the @@ -241,15 +247,18 @@ distributed_tracing.enabled = true [newrelic:development] monitor_mode = false +k2_enabled = false [newrelic:test] monitor_mode = false +k2_enabled = false [newrelic:staging] app_name = Python Application (Staging) monitor_mode = true +k2_enabled = true [newrelic:production] monitor_mode = true - +k2_enabled = true # --------------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 5fdf2005fe..3f90f1a04f 100644 --- a/setup.py +++ b/setup.py @@ -156,6 +156,7 @@ def build_extension(self, ext): }, scripts=["scripts/newrelic-admin"], extras_require={"infinite-tracing": ["grpcio", "protobuf<4"]}, + install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@feature/newrelic_integration#egg=k2_python_agent"] ) if with_setuptools: From 20d5117dae5693cc6e920abf2e88240c9dec3936 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Sep 2022 17:48:48 +0530 Subject: [PATCH 002/525] minor fix --- newrelic/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 287be6c542..2bba619cd0 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3082,7 +3082,6 @@ def initialize( else: _settings.enabled = False if _settings.k2_enabled: - log_message("Invoking K2 security module.", critical=True) # run k2 agent import k2_python_agent k2_python_agent.init_k2({}, {}) From dfbcf0a8ffa8ea71d83ddb091b0803897f108017 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 22 Sep 2022 15:44:02 +0530 Subject: [PATCH 003/525] k2 agent would now init without any args --- newrelic/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 2bba619cd0..a3bf07b2cf 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3084,7 +3084,7 @@ def initialize( if _settings.k2_enabled: # run k2 agent import k2_python_agent - k2_python_agent.init_k2({}, {}) + k2_python_agent.init_k2() def filter_app_factory(app, global_conf, config_file, environment=None): From f399c3a58422926da6069da792fccb31d985201f Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 27 Sep 2022 16:56:36 +0530 Subject: [PATCH 004/525] initial implementation of securty config, removal of switch based k2 startup --- newrelic/admin/record_deploy.py | 2 -- newrelic/config.py | 14 +++++++++----- newrelic/core/config.py | 13 ++++++++++--- newrelic/newrelic.ini | 26 +++++++++++++++----------- 4 files changed, 34 insertions(+), 21 deletions(-) diff --git a/newrelic/admin/record_deploy.py b/newrelic/admin/record_deploy.py index c4079b346c..3a6229de66 100644 --- a/newrelic/admin/record_deploy.py +++ b/newrelic/admin/record_deploy.py @@ -139,8 +139,6 @@ def _args( settings.monitor_mode = False - settings.k2_enabled = False - initialize(config_file) host = settings.host diff --git a/newrelic/config.py b/newrelic/config.py index a3bf07b2cf..1c45e910eb 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,7 +328,11 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "k2_enabled", "getboolean", None) + _process_setting(section, "security.enabled", "getboolean", False) + _process_setting(section, "security.mode", "get", "RASP") + _process_setting(section, "security.validator_service_endpoint_url", "get", "") + _process_setting(section, "security.resource_service_endpoint_url", "get", "") + _process_setting(section, "security.accessor_token", "get", "") _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) @@ -3081,10 +3085,10 @@ def initialize( _setup_agent_console() else: _settings.enabled = False - if _settings.k2_enabled: - # run k2 agent - import k2_python_agent - k2_python_agent.init_k2() + + # run k2 agent + import k2_python_agent + k2_python_agent.init_k2() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 77a12ebcdc..83efe8be5c 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -275,6 +275,9 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass +class SecurityModuleSettings(Settings): + pass + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -398,7 +401,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.infinite_tracing = InfiniteTracingSettings() _settings.event_harvest_config = EventHarvestConfigSettings() _settings.event_harvest_config.harvest_limits = EventHarvestConfigHarvestLimitSettings() - +_settings.security = SecurityModuleSettings() _settings.log_file = os.environ.get("NEW_RELIC_LOG", None) _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) @@ -578,8 +581,6 @@ def default_host(license_key): _settings.monitor_mode = _environ_as_bool("NEW_RELIC_MONITOR_MODE", True) -_settings.k2_enabled = _environ_as_bool("ENABLE_K2_MODE", True) - _settings.developer_mode = _environ_as_bool("NEW_RELIC_DEVELOPER_MODE", False) _settings.high_security = _environ_as_bool("NEW_RELIC_HIGH_SECURITY", False) @@ -823,6 +824,12 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.security.enabled = False +_settings.security.mode = "RASP" +_settings.security.validator_service_endpoint_url = "" +_settings.security.resource_service_endpoint_url = "" +_settings.security.accessor_token = "" + def global_settings(): """This returns the default global settings. Generally only used diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 84f2d99a13..58a62f695d 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -42,18 +42,26 @@ license_key = *** REPLACE ME *** # https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/ app_name = Python Application -# When "true", the agent enables the k2 security module -# This global switch is normally overridden for -# each environment below. It may also be set using the -# ENABLE_K2_MODE environment variable. -monitor_mode = true - # When "true", the agent collects performance data about your # application and reports this data to the New Relic UI at # newrelic.com. This global switch is normally overridden for # each environment below. It may also be set using the # NEW_RELIC_MONITOR_MODE environment variable. -k2_enabled = true +monitor_mode = true + +# When enabled, the agent collects security data about your +# application and reports this data to the New Relic. +security.enable = true +# security module provides two modes IAST or RASP +# RASP stands for Runtime Application Self Protection +# while IAST for Interactive Application Security Testing +# Default mode is RASP +security.mode = RASP +# web-protect agent endpoint connection URLs +security.validator_service_endpoint_url = ws://localhost:54321 +security.resource_service_endpoint_url = http://localhost:54322 +# web-protect agent accessor token +security.accessor_token = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the @@ -247,18 +255,14 @@ distributed_tracing.enabled = true [newrelic:development] monitor_mode = false -k2_enabled = false [newrelic:test] monitor_mode = false -k2_enabled = false [newrelic:staging] app_name = Python Application (Staging) monitor_mode = true -k2_enabled = true [newrelic:production] monitor_mode = true -k2_enabled = true # --------------------------------------------------------------------------- From c3232c915a303dad2fbb60ff3b823a7f99d38244 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 28 Sep 2022 13:02:42 +0530 Subject: [PATCH 005/525] minor fix with incorrect value for configuration mapper --- newrelic/config.py | 10 +++++----- newrelic/newrelic.ini | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1c45e910eb..7d4fdea705 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,11 +328,11 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.enabled", "getboolean", False) - _process_setting(section, "security.mode", "get", "RASP") - _process_setting(section, "security.validator_service_endpoint_url", "get", "") - _process_setting(section, "security.resource_service_endpoint_url", "get", "") - _process_setting(section, "security.accessor_token", "get", "") + _process_setting(section, "security.enabled", "getboolean", None) + _process_setting(section, "security.mode", "get", None) + _process_setting(section, "security.validator_service_endpoint_url", "get", None) + _process_setting(section, "security.resource_service_endpoint_url", "get", None) + _process_setting(section, "security.accessor_token", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 58a62f695d..3b64167fea 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -51,7 +51,7 @@ monitor_mode = true # When enabled, the agent collects security data about your # application and reports this data to the New Relic. -security.enable = true +security.enable = false # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing From 206160e865931fa806b3adde6fc9da60b6b19cfa Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 01:19:29 +0530 Subject: [PATCH 006/525] changes corresponding to latest k2 agent changes --- newrelic/config.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 7d4fdea705..ae16462aac 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3087,8 +3087,14 @@ def initialize( _settings.enabled = False # run k2 agent - import k2_python_agent - k2_python_agent.init_k2() + from k2_python_agent import AgentConfig, ModuleLoadAgent + config = AgentConfig() + config.set_base_config(_settings.security) + config.application_name = _settings.app_name + # TODO: replace with identified app id + config.application_id = _settings.application_id + + ModuleLoadAgent().initialise() def filter_app_factory(app, global_conf, config_file, environment=None): From dd7bd2d4fe0bd22ac58c5747e6a4ef209349455c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 02:31:37 +0530 Subject: [PATCH 007/525] minor fixes --- newrelic/config.py | 4 ++-- newrelic/core/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index ae16462aac..e4b21d281b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3092,9 +3092,9 @@ def initialize( config.set_base_config(_settings.security) config.application_name = _settings.app_name # TODO: replace with identified app id - config.application_id = _settings.application_id + config.application_id = None - ModuleLoadAgent().initialise() + ModuleLoadAgent(config).initialise() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 83efe8be5c..63ca6f7ac3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -824,7 +824,7 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.enabled = False +_settings.security.enable = False _settings.security.mode = "RASP" _settings.security.validator_service_endpoint_url = "" _settings.security.resource_service_endpoint_url = "" From 7578dc00df8e2626329ae595ef25cbe656dfb1f7 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 16:25:29 +0530 Subject: [PATCH 008/525] updated integration source for k2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3f90f1a04f..c52cd3baa1 100644 --- a/setup.py +++ b/setup.py @@ -156,7 +156,7 @@ def build_extension(self, ext): }, scripts=["scripts/newrelic-admin"], extras_require={"infinite-tracing": ["grpcio", "protobuf<4"]}, - install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@feature/newrelic_integration#egg=k2_python_agent"] + install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@newrelic_integration#egg=k2_python_agent"] ) if with_setuptools: From 857f3f9102c72c34baf5366c5a8393044f22c391 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 30 Sep 2022 18:05:51 +0530 Subject: [PATCH 009/525] Added customer id --- newrelic/config.py | 1 + newrelic/core/config.py | 1 + newrelic/newrelic.ini | 1 + 3 files changed, 3 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e4b21d281b..886b321f20 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -333,6 +333,7 @@ def _process_configuration(section): _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.accessor_token", "get", None) + _process_setting(section, "security.customer_id", "get", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 63ca6f7ac3..3eb37299a3 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -829,6 +829,7 @@ def default_host(license_key): _settings.security.validator_service_endpoint_url = "" _settings.security.resource_service_endpoint_url = "" _settings.security.accessor_token = "" +_settings.security.customer_id = "" def global_settings(): diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 3b64167fea..e5e8f3e531 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -62,6 +62,7 @@ security.validator_service_endpoint_url = ws://localhost:54321 security.resource_service_endpoint_url = http://localhost:54322 # web-protect agent accessor token security.accessor_token = +security.customer_id = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From f0c30a14b815a5ac95e14f7aa99a98860791d027 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sat, 1 Oct 2022 18:42:33 +0530 Subject: [PATCH 010/525] initial changes for application id propagation --- newrelic/config.py | 48 +++++++++++++++++++++++++++++++++-------- newrelic/core/config.py | 2 +- newrelic/newrelic.ini | 11 ++++++++-- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 886b321f20..dfa14cee03 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -329,6 +329,7 @@ def _process_configuration(section): _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) _process_setting(section, "security.enabled", "getboolean", None) + _process_setting(section, "security.force_complete_disable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) @@ -3060,6 +3061,43 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _generate_security_module_config(): + from k2_python_agent import AgentConfig + config = AgentConfig() + config.set_base_config(_settings.security) + # propogate app name and id + config.application_name = _settings.app_name + config.application_id = _settings.application_id + + return config + + +def _update_security_module(agent): + config = _generate_security_module_config() + agent.refresh_agent(config) + + +def _setup_security_module(): + """Initiates k2 security module and adds a + callback to agent startup to propagate NR config + """ + if _settings.security.force_complete_disable: + return + + # run security module + from k2_python_agent import AgentConfig, ModuleLoadAgent + from functools import partial as Partial + + config =_generate_security_module_config() + + security_module_agent = ModuleLoadAgent(config) + security_module_agent.initialise() + + # create a callback to reinitialise the security module + callback = Partial(_update_security_module, security_module_agent) + newrelic.core.agent.Agent.run_on_startup(callback) + + def initialize( config_file=None, environment=None, @@ -3087,15 +3125,7 @@ def initialize( else: _settings.enabled = False - # run k2 agent - from k2_python_agent import AgentConfig, ModuleLoadAgent - config = AgentConfig() - config.set_base_config(_settings.security) - config.application_name = _settings.app_name - # TODO: replace with identified app id - config.application_id = None - - ModuleLoadAgent(config).initialise() + _setup_security_module() def filter_app_factory(app, global_conf, config_file, environment=None): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3eb37299a3..e1e1381021 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -824,6 +824,7 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.security.force_complete_disable = False _settings.security.enable = False _settings.security.mode = "RASP" _settings.security.validator_service_endpoint_url = "" @@ -831,7 +832,6 @@ def default_host(license_key): _settings.security.accessor_token = "" _settings.security.customer_id = "" - def global_settings(): """This returns the default global settings. Generally only used directly in test scripts and test harnesses or when applying global diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index e5e8f3e531..1946eb3abd 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -52,6 +52,7 @@ monitor_mode = true # When enabled, the agent collects security data about your # application and reports this data to the New Relic. security.enable = false +security.force_complete_disable = false # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing @@ -60,9 +61,15 @@ security.mode = RASP # web-protect agent endpoint connection URLs security.validator_service_endpoint_url = ws://localhost:54321 security.resource_service_endpoint_url = http://localhost:54322 + # web-protect agent accessor token -security.accessor_token = -security.customer_id = +# security.sec_home_path = +# security.sec_log_file_name = + +# disable vulnerability type flags +# security.detection.disable_rci = +# security.detection.disable_rxss = +# security.detection.disable_desearlization = # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 42d621a200e2948429d4cb6978d1bb8676eff1cc Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 01:20:44 +0530 Subject: [PATCH 011/525] minor fix in application_id extraction --- newrelic/config.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index dfa14cee03..8b421b89d1 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3066,8 +3066,13 @@ def _generate_security_module_config(): config = AgentConfig() config.set_base_config(_settings.security) # propogate app name and id + agent_instance = newrelic.core.agent.agent_instance() + application = agent_instance.application(_settings.app_name) + configuration = application.configuration + + if configuration and hasattr(configuration, 'primary_application_id'): + config.application_id = configuration.primary_application_id config.application_name = _settings.app_name - config.application_id = _settings.application_id return config From e700fb2df80db2c75c897365d84b70e21a76c712 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 01:23:42 +0530 Subject: [PATCH 012/525] minor fix --- newrelic/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 8b421b89d1..0c04fbca8b 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3068,9 +3068,8 @@ def _generate_security_module_config(): # propogate app name and id agent_instance = newrelic.core.agent.agent_instance() application = agent_instance.application(_settings.app_name) - configuration = application.configuration - - if configuration and hasattr(configuration, 'primary_application_id'): + if application: + configuration = application.configuration config.application_id = configuration.primary_application_id config.application_name = _settings.app_name From 1d5964b801a6086ab4d140c544ee0dfaeb696fe2 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 2 Oct 2022 11:58:17 +0530 Subject: [PATCH 013/525] guid propagation instead of application_id --- newrelic/config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 0c04fbca8b..c9c472f79e 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3070,7 +3070,7 @@ def _generate_security_module_config(): application = agent_instance.application(_settings.app_name) if application: configuration = application.configuration - config.application_id = configuration.primary_application_id + config.application_id = configuration.entity_guid config.application_name = _settings.app_name return config @@ -3079,6 +3079,7 @@ def _generate_security_module_config(): def _update_security_module(agent): config = _generate_security_module_config() agent.refresh_agent(config) + agent.connect() def _setup_security_module(): From 163645b09455c440a3d594be98afcec7102c29f5 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 3 Oct 2022 00:19:52 +0530 Subject: [PATCH 014/525] This implements propagation of all possible k2 config in NR's config --- newrelic/config.py | 23 ++++++- newrelic/core/config.py | 148 +++++++++++++++++++++++++++++++++++++--- newrelic/newrelic.ini | 47 ++++++++++--- 3 files changed, 197 insertions(+), 21 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index c9c472f79e..ba87d71bf4 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,13 +328,32 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.enabled", "getboolean", None) _process_setting(section, "security.force_complete_disable", "getboolean", None) + _process_setting(section, "security.enable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.accessor_token", "get", None) - _process_setting(section, "security.customer_id", "get", None) + _process_setting(section, "security.customer_id", "getint", None) + _process_setting(section, "security.log_level", "get", None) + _process_setting(section, "security.sec_home_path", "get", None) + _process_setting(section, "security.sec_log_file_name", "get", None) + _process_setting(section, "security.detection.disable_rci", "getboolean", None) + _process_setting(section, "security.detection.disable_rxss", "getboolean", None) + _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) + _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.batchSize", "getint", None) + _process_setting(section, "security.policy.protectionMode.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.attackerIpBlocking", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.ipDetectViaXFF", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.ipBlocking.timeout", "getint", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.enabled", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAllApis", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis", "getboolean", None) + _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAttackedApis", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index e1e1381021..a31a2b53dd 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,6 +278,30 @@ class ApplicationLoggingLocalDecoratingSettings(Settings): class SecurityModuleSettings(Settings): pass +class SecurityDetectionSettings(Settings): + pass + +class SecurityPolicySettings(Settings): + pass + +class SecurityPolicyVulnerabilityScanSettings(Settings): + pass + +class SecurityPolicyIASTSettings(Settings): + pass + +class SecurityPolicyIASTProbingSettings(Settings): + pass + +class SecurityPolicyprotectionModeSettings(Settings): + pass + +class SecurityPolicyIPBlockingSettings(Settings): + pass + +class SecurityPolicyAPIBlockingSettings(Settings): + pass + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -401,7 +425,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.infinite_tracing = InfiniteTracingSettings() _settings.event_harvest_config = EventHarvestConfigSettings() _settings.event_harvest_config.harvest_limits = EventHarvestConfigHarvestLimitSettings() -_settings.security = SecurityModuleSettings() _settings.log_file = os.environ.get("NEW_RELIC_LOG", None) _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) @@ -824,13 +847,122 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.force_complete_disable = False -_settings.security.enable = False -_settings.security.mode = "RASP" -_settings.security.validator_service_endpoint_url = "" -_settings.security.resource_service_endpoint_url = "" -_settings.security.accessor_token = "" -_settings.security.customer_id = "" +_settings.security = SecurityModuleSettings() +_settings.security.detection = SecurityDetectionSettings() +_settings.security.policy = SecurityPolicySettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTSettings() +_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTProbingSettings() +_settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() +_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() +_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyAPIBlockingSettings() + +_settings.security.force_complete_disable = _environ_as_bool( + "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", + default=False +) +_settings.security.enable = _environ_as_bool( + "NEW_RELIC_SECURITY_ENABLE", + default=False +) +_settings.security.mode = os.environ.get( + "NEW_RELIC_SECURITY_MODE", + default="RASP" +) +_settings.security.validator_service_endpoint_url = os.environ.get( + "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", + default=None +) +_settings.security.resource_service_endpoint_url = os.environ.get( + "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", + default=None +) +_settings.security.accessor_token = os.environ.get( + "NEW_RELIC_SECURITY_ACCESSOR_TOKEN", + default=None +) +_settings.security.customer_id = os.environ.get( + "NEW_RELIC_SECURITY_CUSTOMER_ID", + default=None +) +_settings.security.log_level = os.environ.get( + "NEW_RELIC_SECURITY_LOG_LEVEL", + default="INFO" +) +_settings.security.sec_home_path = os.environ.get( + "NEW_RELIC_SECURITY_SEC_HOME_PATH", + default="/opt" +) +_settings.security.sec_log_file_name = os.environ.get( + "NEW_RELIC_SECURITY_SEC_LOG_FILE_NAME", + default="/temp/security_log" +) +_settings.security.detection.disable_rci = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", + default=False +) +_settings.security.detection.disable_rxss = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_RXSS", + default=False +) +_settings.security.detection.disable_deserialization = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DISABLE_DESERIALIZATION", + default=False +) + +_settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", + default=False +) +_settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", + default=False +) +_settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", + default=1 +) +_settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", + default=5 +) +_settings.security.policy.protectionMode.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", + default=False +) +_settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", + default=120 +) +_settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", + default=False +) +_settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", + default=False +) + def global_settings(): """This returns the default global settings. Generally only used diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 1946eb3abd..d985f87a02 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -49,27 +49,52 @@ app_name = Python Application # NEW_RELIC_MONITOR_MODE environment variable. monitor_mode = true +# When enabled, the agent's security module is not loaded +# this would mean changes on soft reloads won't work +security.force_complete_disable = false # When enabled, the agent collects security data about your # application and reports this data to the New Relic. -security.enable = false -security.force_complete_disable = false +security.enable = true # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing # Default mode is RASP security.mode = RASP + # web-protect agent endpoint connection URLs -security.validator_service_endpoint_url = ws://localhost:54321 -security.resource_service_endpoint_url = http://localhost:54322 +security.validator_service_endpoint_url = +security.resource_service_endpoint_url = # web-protect agent accessor token -# security.sec_home_path = -# security.sec_log_file_name = - -# disable vulnerability type flags -# security.detection.disable_rci = -# security.detection.disable_rxss = -# security.detection.disable_desearlization = +security.accessor_token = +security.customer_id = + +# log level +security.log_level = INFO + +# security file settings +# security.sec_home_path = /opt +# security.sec_log_file_name = /tmp/security.log + +# vulnerabilty detection disable +# security.detection.disable_rci = false +# security.detection.disable_rxss = false +# security.detection.disable_deserialization = false + +# policy settings +# security.policy.vulnerabilityScan.enabled = false +# security.policy.vulnerabilityScan.iastScan.enabled = false +# security.policy.vulnerabilityScan.iastScan.probing.interval = 1 +# security.policy.vulnerabilityScan.iastScan.probing.batchSize = 5 +# security.policy.protectionMode.enabled = false +# security.policy.protectionMode.ipBlocking.enabled = false +# security.policy.protectionMode.ipBlocking.attackerIpBlocking = false +# security.policy.protectionMode.ipBlocking.ipDetectViaXFF = false +# security.policy.protectionMode.ipBlocking.timeout = 120 +# security.policy.protectionMode.apiBlocking.enabled = false +# security.policy.protectionMode.apiBlocking.protectAllApis = false +# security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = false +# security.policy.protectionMode.apiBlocking.protectAttackedApis = false # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 52e59e2cb6348d48483576345c59e029548ecad4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 3 Oct 2022 15:05:43 +0530 Subject: [PATCH 015/525] changes to populate dictionary of policy and changes to reflect security.enable flag on startup --- newrelic/config.py | 12 ++++++++++-- newrelic/core/config.py | 30 ++++++++++++++++++++---------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index ba87d71bf4..d598774b63 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3080,6 +3080,10 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _generate_security_policy(): + return _settings.security.policy.to_deep_dict() + + def _generate_security_module_config(): from k2_python_agent import AgentConfig config = AgentConfig() @@ -3098,7 +3102,8 @@ def _generate_security_module_config(): def _update_security_module(agent): config = _generate_security_module_config() agent.refresh_agent(config) - agent.connect() + if _settings.security.enable: + agent.connect() def _setup_security_module(): @@ -3107,16 +3112,19 @@ def _setup_security_module(): """ if _settings.security.force_complete_disable: return - # run security module from k2_python_agent import AgentConfig, ModuleLoadAgent from functools import partial as Partial config =_generate_security_module_config() + policy = _generate_security_policy() security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() + security_module_agent.set_policy(policy) + if not _settings.security.enable: + security_module_agent.disable() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index a31a2b53dd..81a9024d29 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -94,6 +94,16 @@ def __contains__(self, item): return hasattr(self, item) +class DeepDictableSetting: + def to_deep_dict(self): + _dict = self.__dict__ + for attr in _dict.keys(): + value = _dict[attr] + if isinstance(value, DeepDictableSetting): + _dict[attr] = value.to_deep_dict() + return _dict + + def create_settings(nested): return type("Settings", (Settings,), {"nested": nested})() @@ -281,25 +291,25 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings): +class SecurityPolicySettings(Settings, DeepDictableSetting): pass -class SecurityPolicyVulnerabilityScanSettings(Settings): +class SecurityPolicyVulnerabilityScanSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIASTSettings(Settings): +class SecurityPolicyIASTSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIASTProbingSettings(Settings): +class SecurityPolicyIASTProbingSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyprotectionModeSettings(Settings): +class SecurityPolicyprotectionModeSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyIPBlockingSettings(Settings): +class SecurityPolicyIPBlockingSettings(Settings, DeepDictableSetting): pass -class SecurityPolicyAPIBlockingSettings(Settings): +class SecurityPolicyAPIBlockingSettings(Settings, DeepDictableSetting): pass @@ -851,11 +861,11 @@ def default_host(license_key): _settings.security.detection = SecurityDetectionSettings() _settings.security.policy = SecurityPolicySettings() _settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTSettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyIASTProbingSettings() +_settings.security.policy.vulnerabilityScan.iastScan = SecurityPolicyIASTSettings() +_settings.security.policy.vulnerabilityScan.iastScan.probing = SecurityPolicyIASTProbingSettings() _settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() _settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() -_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyAPIBlockingSettings() +_settings.security.policy.protectionMode.apiBlocking = SecurityPolicyAPIBlockingSettings() _settings.security.force_complete_disable = _environ_as_bool( "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", From a609df87cc118ff3b83b9fc9c14f4b3815bc2ec5 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 6 Oct 2022 13:11:11 +0530 Subject: [PATCH 016/525] This contains changes for policy propagation to k2 --- newrelic/config.py | 2 +- newrelic/core/config.py | 50 +++++++++++++++++------------------------ 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index d598774b63..1a14a63e79 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3081,7 +3081,7 @@ def _setup_agent_console(): def _generate_security_policy(): - return _settings.security.policy.to_deep_dict() + return dict(_settings.security.policy) def _generate_security_module_config(): diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 81a9024d29..dc3e41a50d 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -94,16 +94,6 @@ def __contains__(self, item): return hasattr(self, item) -class DeepDictableSetting: - def to_deep_dict(self): - _dict = self.__dict__ - for attr in _dict.keys(): - value = _dict[attr] - if isinstance(value, DeepDictableSetting): - _dict[attr] = value.to_deep_dict() - return _dict - - def create_settings(nested): return type("Settings", (Settings,), {"nested": nested})() @@ -291,25 +281,25 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings, DeepDictableSetting): +class SecurityPolicySettings(Settings): pass -class SecurityPolicyVulnerabilityScanSettings(Settings, DeepDictableSetting): +class SecurityPolicyVulnerabilityScanSettings(Settings): pass -class SecurityPolicyIASTSettings(Settings, DeepDictableSetting): +class SecurityPolicyIASTSettings(Settings): pass -class SecurityPolicyIASTProbingSettings(Settings, DeepDictableSetting): +class SecurityPolicyIASTProbingSettings(Settings): pass -class SecurityPolicyprotectionModeSettings(Settings, DeepDictableSetting): +class SecurityPolicyprotectionModeSettings(Settings): pass -class SecurityPolicyIPBlockingSettings(Settings, DeepDictableSetting): +class SecurityPolicyIPBlockingSettings(Settings): pass -class SecurityPolicyAPIBlockingSettings(Settings, DeepDictableSetting): +class SecurityPolicyAPIBlockingSettings(Settings): pass @@ -922,55 +912,55 @@ def default_host(license_key): _settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", - default=False + default=None ) _settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", - default=False + default=None ) _settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", - default=1 + default=-1 ) _settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", - default=5 + default=-1 ) _settings.security.policy.protectionMode.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", - default=False + default=None ) _settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", - default=120 + default=-1 ) _settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", - default=False + default=None ) _settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", - default=False + default=None ) From 30a280add772c6b6d46647c72b7bc2b589c885e8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 6 Oct 2022 13:11:49 +0530 Subject: [PATCH 017/525] Merge branch 'feature/k2i/policy_propagation' into k2_integration From a5dd4184c43beb22ad9ea2870a86ffce05264554 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 7 Oct 2022 17:52:38 +0530 Subject: [PATCH 018/525] updated config for logs upload and exception handling in security module setup --- newrelic/config.py | 40 +++++++++++++++++++++------------------- newrelic/core/config.py | 4 ---- newrelic/newrelic.ini | 3 +-- 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1a14a63e79..3aa551fe0c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -337,7 +337,6 @@ def _process_configuration(section): _process_setting(section, "security.customer_id", "getint", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) - _process_setting(section, "security.sec_log_file_name", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) _process_setting(section, "security.detection.disable_rxss", "getboolean", None) _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) @@ -3110,24 +3109,27 @@ def _setup_security_module(): """Initiates k2 security module and adds a callback to agent startup to propagate NR config """ - if _settings.security.force_complete_disable: - return - # run security module - from k2_python_agent import AgentConfig, ModuleLoadAgent - from functools import partial as Partial - - config =_generate_security_module_config() - policy = _generate_security_policy() - - security_module_agent = ModuleLoadAgent(config) - security_module_agent.initialise() - - security_module_agent.set_policy(policy) - if not _settings.security.enable: - security_module_agent.disable() - # create a callback to reinitialise the security module - callback = Partial(_update_security_module, security_module_agent) - newrelic.core.agent.Agent.run_on_startup(callback) + try: + if _settings.security.force_complete_disable: + return + # run security module + from k2_python_agent import AgentConfig, ModuleLoadAgent + from functools import partial as Partial + + config =_generate_security_module_config() + policy = _generate_security_policy() + + security_module_agent = ModuleLoadAgent(config) + security_module_agent.initialise() + + security_module_agent.set_policy(policy) + if not _settings.security.enable: + security_module_agent.disable() + # create a callback to reinitialise the security module + callback = Partial(_update_security_module, security_module_agent) + newrelic.core.agent.Agent.run_on_startup(callback) + except Exception as k2error: + _logger.error("K2 Startup failed with error %s", k2error) def initialize( diff --git a/newrelic/core/config.py b/newrelic/core/config.py index dc3e41a50d..1c7b6458d8 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -893,10 +893,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_SEC_HOME_PATH", default="/opt" ) -_settings.security.sec_log_file_name = os.environ.get( - "NEW_RELIC_SECURITY_SEC_LOG_FILE_NAME", - default="/temp/security_log" -) _settings.security.detection.disable_rci = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", default=False diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d985f87a02..02b0de3aa0 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -73,8 +73,7 @@ security.customer_id = security.log_level = INFO # security file settings -# security.sec_home_path = /opt -# security.sec_log_file_name = /tmp/security.log +security.sec_home_path = /tmp # vulnerabilty detection disable # security.detection.disable_rci = false From 5f226e8e1b52d5b1e72471661a14a28c246810c8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 9 Oct 2022 19:00:35 +0530 Subject: [PATCH 019/525] Merge branch 'feature/k2i/logs_config_propagation' into k2_integration From 6d515e845520abf95431f2419faaafdea4d3f18b Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Sun, 9 Oct 2022 19:03:03 +0530 Subject: [PATCH 020/525] Changes for setting the trasnsaction id catcher by handing over a lambda which would fetch transaction_id on run --- newrelic/config.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index 3aa551fe0c..cbac91c845 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3128,6 +3128,11 @@ def _setup_security_module(): # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) + + # set transaction id catcher + security_module_agent.set_transaction_id_catcher( + lambda *args: trace_cache.trace_cache().current_transaction()._transaction_id + ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) From 34d610bff21a8faacc4a707af5e1917055162109 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 10 Oct 2022 14:49:24 +0530 Subject: [PATCH 021/525] changes to send trace metadata long with transaction id from metadata catcher --- newrelic/config.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index cbac91c845..0274197aba 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3079,6 +3079,15 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _get_transaction_metadata_for_security_module(): + trace = trace_cache.trace_cache().current_trace() + transaction = trace and trace.transaction and trace.transaction + metadata = trace and trace._get_trace_linking_metadata() + transaction_id = transaction._transaction_id + metadata['transaction_id'] = transaction_id + return metadata + + def _generate_security_policy(): return dict(_settings.security.policy) @@ -3130,8 +3139,8 @@ def _setup_security_module(): newrelic.core.agent.Agent.run_on_startup(callback) # set transaction id catcher - security_module_agent.set_transaction_id_catcher( - lambda *args: trace_cache.trace_cache().current_transaction()._transaction_id + security_module_agent.set_metadata_catcher( + _get_transaction_metadata_for_security_module ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) From e02eacfa685a0244e55381da3d67620959f717e0 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 10 Oct 2022 14:50:35 +0530 Subject: [PATCH 022/525] Merge 'feature/k2i/logs_config_propagation' into k2_integration From 385b96696914a176d0025743470ab2b2e7d2a544 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 15:14:01 +0530 Subject: [PATCH 023/525] This contains multiple changes: 1. Changes to start security module even before the agent is activated, thus, init on active_agent callback have been removed and instead added along with module load phase. 2. Updated agent refresh with policy and linking metadata along with agent config for security module 3. Removed deprecated security module configs constomer_id and accessor_token --- newrelic/config.py | 52 ++++++++++++++++++++++++----------------- newrelic/core/config.py | 8 ------- newrelic/newrelic.ini | 4 ---- 3 files changed, 30 insertions(+), 34 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 0274197aba..c2f204bbe2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -333,8 +333,6 @@ def _process_configuration(section): _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) _process_setting(section, "security.resource_service_endpoint_url", "get", None) - _process_setting(section, "security.accessor_token", "get", None) - _process_setting(section, "security.customer_id", "getint", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) @@ -3079,16 +3077,19 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) -def _get_transaction_metadata_for_security_module(): +def _get_linking_metadata_for_security_module(): + import newrelic.agent + context = newrelic.agent.get_linking_metadata() + return context + + +def _get_trace_linking_metadata_for_security_module(): trace = trace_cache.trace_cache().current_trace() - transaction = trace and trace.transaction and trace.transaction - metadata = trace and trace._get_trace_linking_metadata() - transaction_id = transaction._transaction_id - metadata['transaction_id'] = transaction_id + metadata = trace and trace._get_trace_linking_metadata() or {} return metadata -def _generate_security_policy(): +def _generate_security_module_policy(): return dict(_settings.security.policy) @@ -3096,22 +3097,27 @@ def _generate_security_module_config(): from k2_python_agent import AgentConfig config = AgentConfig() config.set_base_config(_settings.security) - # propogate app name and id - agent_instance = newrelic.core.agent.agent_instance() - application = agent_instance.application(_settings.app_name) - if application: - configuration = application.configuration - config.application_id = configuration.entity_guid + config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name return config def _update_security_module(agent): + """refreshes the security module with latest config and + linking metadata + """ config = _generate_security_module_config() - agent.refresh_agent(config) - if _settings.security.enable: - agent.connect() + policy = _generate_security_module_policy() + metadata = _get_linking_metadata_for_security_module() + # propogate app name and id + agent_instance = newrelic.core.agent.agent_instance() + application = agent_instance.application(_settings.app_name) + if application: + configuration = application.configuration + metadata["agentRunId"] = configuration.agent_run_id + + agent.refresh_agent(config, policy, metadata) def _setup_security_module(): @@ -3126,21 +3132,23 @@ def _setup_security_module(): from functools import partial as Partial config =_generate_security_module_config() - policy = _generate_security_policy() + policy = _generate_security_module_policy() security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() - security_module_agent.set_policy(policy) if not _settings.security.enable: security_module_agent.disable() + else: + security_module_agent.connect() + # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) newrelic.core.agent.Agent.run_on_startup(callback) - # set transaction id catcher - security_module_agent.set_metadata_catcher( - _get_transaction_metadata_for_security_module + # set trace_linking_metadata catcher + security_module_agent.set_linking_metadata_catcher( + _get_trace_linking_metadata_for_security_module ) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1c7b6458d8..1466b74da9 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -877,14 +877,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", default=None ) -_settings.security.accessor_token = os.environ.get( - "NEW_RELIC_SECURITY_ACCESSOR_TOKEN", - default=None -) -_settings.security.customer_id = os.environ.get( - "NEW_RELIC_SECURITY_CUSTOMER_ID", - default=None -) _settings.security.log_level = os.environ.get( "NEW_RELIC_SECURITY_LOG_LEVEL", default="INFO" diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 02b0de3aa0..55d5913b0c 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -65,10 +65,6 @@ security.mode = RASP security.validator_service_endpoint_url = security.resource_service_endpoint_url = -# web-protect agent accessor token -security.accessor_token = -security.customer_id = - # log level security.log_level = INFO From ded8f4822b065d3bc751a1613dce67ab69bfac9f Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 16:23:06 +0530 Subject: [PATCH 024/525] minor change to reflect k2 module changes --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index c2f204bbe2..e803fa7a18 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3136,6 +3136,7 @@ def _setup_security_module(): security_module_agent = ModuleLoadAgent(config) security_module_agent.initialise() + security_module_agent.set_policy_from_flat_dict(policy) if not _settings.security.enable: security_module_agent.disable() From 52984303cd55f322689c21c7d1ffa443b9cf0780 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 12 Oct 2022 16:24:11 +0530 Subject: [PATCH 025/525] Merge branch 'feature/k2i/add_linking_metadata_deprecate_old_k2_auth_config' into k2_integration From a2f385e869c7896f60e7c520814f20b6a7e947ea Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 26 Oct 2022 14:20:09 +0530 Subject: [PATCH 026/525] Addition of enforce flag --- newrelic/config.py | 1 + newrelic/core/config.py | 4 ++++ newrelic/newrelic.ini | 1 + 3 files changed, 6 insertions(+) diff --git a/newrelic/config.py b/newrelic/config.py index e803fa7a18..f4653441b3 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -338,6 +338,7 @@ def _process_configuration(section): _process_setting(section, "security.detection.disable_rci", "getboolean", None) _process_setting(section, "security.detection.disable_rxss", "getboolean", None) _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) + _process_setting(section, "security.policy.enforce", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 1466b74da9..dbd6856a38 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -898,6 +898,10 @@ def default_host(license_key): default=False ) +_settings.security.policy.enforce = _environ_as_bool( + "NEW_RELIC_SECURITY_POLICY_ENFORCE", + default=False +) _settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", default=None diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 55d5913b0c..937d958b70 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -77,6 +77,7 @@ security.sec_home_path = /tmp # security.detection.disable_deserialization = false # policy settings +# security.policy.enforce = false # security.policy.vulnerabilityScan.enabled = false # security.policy.vulnerabilityScan.iastScan.enabled = false # security.policy.vulnerabilityScan.iastScan.probing.interval = 1 From a8a7a889361b45288456bd95392ce4026e41ca80 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 21 Dec 2022 13:09:50 +0530 Subject: [PATCH 027/525] Addition of account id in linking metadata --- newrelic/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/newrelic/config.py b/newrelic/config.py index d12962b900..5e960fd8ff 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3154,6 +3154,7 @@ def _update_security_module(agent): if application: configuration = application.configuration metadata["agentRunId"] = configuration.agent_run_id + metadata["accountId"] = configuration.account_id agent.refresh_agent(config, policy, metadata) From d925658c362dcda585817a1a440a6e78730ba7a7 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 23 Dec 2022 13:58:47 +0530 Subject: [PATCH 028/525] Temp changes --- newrelic/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e960fd8ff..5e0ba502bf 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3132,8 +3132,8 @@ def _generate_security_module_policy(): def _generate_security_module_config(): - from k2_python_agent import AgentConfig - config = AgentConfig() + from newrelic_security.agent.models.agent_config import NRSecurityAgentConfig + config = NRSecurityAgentConfig() config.set_base_config(_settings.security) config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name @@ -3167,13 +3167,13 @@ def _setup_security_module(): if _settings.security.force_complete_disable: return # run security module - from k2_python_agent import AgentConfig, ModuleLoadAgent + from newrelic_security.agent.nr_security_agent import NRSecurityAgent from functools import partial as Partial config =_generate_security_module_config() policy = _generate_security_module_policy() - security_module_agent = ModuleLoadAgent(config) + security_module_agent = NRSecurityAgent(config) security_module_agent.initialise() security_module_agent.set_policy_from_flat_dict(policy) @@ -3191,6 +3191,8 @@ def _setup_security_module(): _get_trace_linking_metadata_for_security_module ) except Exception as k2error: + import traceback + traceback.print_tb(k2error.__traceback__) _logger.error("K2 Startup failed with error %s", k2error) From a741c2d8d7ea9e776ccc08151e9f68edd2bba5a4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 23 Dec 2022 14:33:41 +0530 Subject: [PATCH 029/525] Let the connect be called with refresh (When linking metadata is available) --- newrelic/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e960fd8ff..310174aef2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3179,8 +3179,6 @@ def _setup_security_module(): if not _settings.security.enable: security_module_agent.disable() - else: - security_module_agent.connect() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) From 0ec4fe4907b6c97e4c9624b6296189a9e2f49eff Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 28 Dec 2022 10:40:44 +0530 Subject: [PATCH 030/525] agent would not connect with startup --- newrelic/config.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 5e0ba502bf..1d06a55048 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3155,7 +3155,7 @@ def _update_security_module(agent): configuration = application.configuration metadata["agentRunId"] = configuration.agent_run_id metadata["accountId"] = configuration.account_id - + print("REFRESH AGENT!!!!") agent.refresh_agent(config, policy, metadata) @@ -3179,8 +3179,6 @@ def _setup_security_module(): if not _settings.security.enable: security_module_agent.disable() - else: - security_module_agent.connect() # create a callback to reinitialise the security module callback = Partial(_update_security_module, security_module_agent) From f5a98bad58e8c981356e419410b9ccb4ae642613 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Wed, 25 Jan 2023 23:06:41 +0530 Subject: [PATCH 031/525] removal of rest server endpoint config, use of SingletonAgentConfig instead of NRSecurityAgentConfig for generating a config. --- newrelic/config.py | 5 ++--- newrelic/core/config.py | 4 ---- newrelic/newrelic.ini | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 1d06a55048..36b6efe0d1 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -332,7 +332,6 @@ def _process_configuration(section): _process_setting(section, "security.enable", "getboolean", None) _process_setting(section, "security.mode", "get", None) _process_setting(section, "security.validator_service_endpoint_url", "get", None) - _process_setting(section, "security.resource_service_endpoint_url", "get", None) _process_setting(section, "security.log_level", "get", None) _process_setting(section, "security.sec_home_path", "get", None) _process_setting(section, "security.detection.disable_rci", "getboolean", None) @@ -3132,8 +3131,8 @@ def _generate_security_module_policy(): def _generate_security_module_config(): - from newrelic_security.agent.models.agent_config import NRSecurityAgentConfig - config = NRSecurityAgentConfig() + from newrelic_security.agent.models.agent_config import SingletonAgentConfig + config = SingletonAgentConfig.instance() config.set_base_config(_settings.security) config.set_acessor_token(_settings.license_key) config.application_name = _settings.app_name diff --git a/newrelic/core/config.py b/newrelic/core/config.py index dbd6856a38..d6363915cf 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -873,10 +873,6 @@ def default_host(license_key): "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", default=None ) -_settings.security.resource_service_endpoint_url = os.environ.get( - "NEW_RELIC_SECURITY_RESOURCE_SERVICE_ENDPOINT_URL", - default=None -) _settings.security.log_level = os.environ.get( "NEW_RELIC_SECURITY_LOG_LEVEL", default="INFO" diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index 937d958b70..f1a589b2e3 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -63,7 +63,6 @@ security.mode = RASP # web-protect agent endpoint connection URLs security.validator_service_endpoint_url = -security.resource_service_endpoint_url = # log level security.log_level = INFO From 524b3154b0a399846cf51102c325091581d7665d Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 26 Jan 2023 00:31:47 +0530 Subject: [PATCH 032/525] removal of all security agent initialization logic --- newrelic/config.py | 73 ++++------------------------------------------ 1 file changed, 5 insertions(+), 68 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 36b6efe0d1..6253fbf417 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3114,50 +3114,6 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) -def _get_linking_metadata_for_security_module(): - import newrelic.agent - context = newrelic.agent.get_linking_metadata() - return context - - -def _get_trace_linking_metadata_for_security_module(): - trace = trace_cache.trace_cache().current_trace() - metadata = trace and trace._get_trace_linking_metadata() or {} - return metadata - - -def _generate_security_module_policy(): - return dict(_settings.security.policy) - - -def _generate_security_module_config(): - from newrelic_security.agent.models.agent_config import SingletonAgentConfig - config = SingletonAgentConfig.instance() - config.set_base_config(_settings.security) - config.set_acessor_token(_settings.license_key) - config.application_name = _settings.app_name - - return config - - -def _update_security_module(agent): - """refreshes the security module with latest config and - linking metadata - """ - config = _generate_security_module_config() - policy = _generate_security_module_policy() - metadata = _get_linking_metadata_for_security_module() - # propogate app name and id - agent_instance = newrelic.core.agent.agent_instance() - application = agent_instance.application(_settings.app_name) - if application: - configuration = application.configuration - metadata["agentRunId"] = configuration.agent_run_id - metadata["accountId"] = configuration.account_id - print("REFRESH AGENT!!!!") - agent.refresh_agent(config, policy, metadata) - - def _setup_security_module(): """Initiates k2 security module and adds a callback to agent startup to propagate NR config @@ -3165,31 +3121,12 @@ def _setup_security_module(): try: if _settings.security.force_complete_disable: return - # run security module - from newrelic_security.agent.nr_security_agent import NRSecurityAgent - from functools import partial as Partial - - config =_generate_security_module_config() - policy = _generate_security_module_policy() - - security_module_agent = NRSecurityAgent(config) - security_module_agent.initialise() - security_module_agent.set_policy_from_flat_dict(policy) - - if not _settings.security.enable: - security_module_agent.disable() - - # create a callback to reinitialise the security module - callback = Partial(_update_security_module, security_module_agent) - newrelic.core.agent.Agent.run_on_startup(callback) - - # set trace_linking_metadata catcher - security_module_agent.set_linking_metadata_catcher( - _get_trace_linking_metadata_for_security_module - ) + from newrelic_security.api.agent import Agent as SecurityAgent + # initialize security agent + security_agent = SecurityAgent() + # create a callback to reinitialise the security module + newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) except Exception as k2error: - import traceback - traceback.print_tb(k2error.__traceback__) _logger.error("K2 Startup failed with error %s", k2error) From 46637fe6f6d864336714108154793bba42a5c2bb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Tue, 14 Mar 2023 22:01:00 +0530 Subject: [PATCH 033/525] updates to newrelic config for security module --- newrelic/config.py | 28 ++------- newrelic/core/config.py | 127 ++++++++-------------------------------- newrelic/newrelic.ini | 56 +++++++----------- 3 files changed, 52 insertions(+), 159 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 6253fbf417..03df8cde5c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -328,29 +328,13 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) - _process_setting(section, "security.force_complete_disable", "getboolean", None) - _process_setting(section, "security.enable", "getboolean", None) + _process_setting(section, "security.agent.enabled", "getboolean", None) + _process_setting(section, "security.enabled", "getboolean", None) _process_setting(section, "security.mode", "get", None) - _process_setting(section, "security.validator_service_endpoint_url", "get", None) - _process_setting(section, "security.log_level", "get", None) - _process_setting(section, "security.sec_home_path", "get", None) - _process_setting(section, "security.detection.disable_rci", "getboolean", None) - _process_setting(section, "security.detection.disable_rxss", "getboolean", None) - _process_setting(section, "security.detection.disable_deserialization", "getboolean", None) - _process_setting(section, "security.policy.enforce", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.enabled", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.enabled", "getboolean", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.interval", "getint", None) - _process_setting(section, "security.policy.vulnerabilityScan.iastScan.probing.batchSize", "getint", None) - _process_setting(section, "security.policy.protectionMode.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.attackerIpBlocking", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.ipDetectViaXFF", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.ipBlocking.timeout", "getint", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.enabled", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAllApis", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis", "getboolean", None) - _process_setting(section, "security.policy.protectionMode.apiBlocking.protectAttackedApis", "getboolean", None) + _process_setting(section, "security.validator_service_url", "get", None) + _process_setting(section, "security.detection.rci.enabled", "getboolean", None) + _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) + _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index d6363915cf..4d7c14d637 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -281,26 +281,18 @@ class SecurityModuleSettings(Settings): class SecurityDetectionSettings(Settings): pass -class SecurityPolicySettings(Settings): +class SecurityAgentSettings(Settings): pass -class SecurityPolicyVulnerabilityScanSettings(Settings): +class SecurityDetectionRCISettings(Settings): pass -class SecurityPolicyIASTSettings(Settings): +class SecurityDetectionRXSSSettings(Settings): pass -class SecurityPolicyIASTProbingSettings(Settings): +class SecurityDetectionDeserializationSettings(Settings): pass -class SecurityPolicyprotectionModeSettings(Settings): - pass - -class SecurityPolicyIPBlockingSettings(Settings): - pass - -class SecurityPolicyAPIBlockingSettings(Settings): - pass class InfiniteTracingSettings(Settings): @@ -848,107 +840,40 @@ def default_host(license_key): ) _settings.security = SecurityModuleSettings() +_settings.security.agent = SecurityAgentSettings() _settings.security.detection = SecurityDetectionSettings() -_settings.security.policy = SecurityPolicySettings() -_settings.security.policy.vulnerabilityScan = SecurityPolicyVulnerabilityScanSettings() -_settings.security.policy.vulnerabilityScan.iastScan = SecurityPolicyIASTSettings() -_settings.security.policy.vulnerabilityScan.iastScan.probing = SecurityPolicyIASTProbingSettings() -_settings.security.policy.protectionMode = SecurityPolicyprotectionModeSettings() -_settings.security.policy.protectionMode.ipBlocking = SecurityPolicyIPBlockingSettings() -_settings.security.policy.protectionMode.apiBlocking = SecurityPolicyAPIBlockingSettings() - -_settings.security.force_complete_disable = _environ_as_bool( - "NEW_RELIC_SECURITY_FORCE_COMPLETE_DISABLE", +_settings.security.detection.rci = SecurityDetectionRCISettings() +_settings.security.detection.rxss = SecurityDetectionRXSSSettings() +_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() + + +_settings.security.agent.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_AGENT_ENABLED", default=False ) -_settings.security.enable = _environ_as_bool( - "NEW_RELIC_SECURITY_ENABLE", +_settings.security.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_ENABLED", default=False ) _settings.security.mode = os.environ.get( "NEW_RELIC_SECURITY_MODE", - default="RASP" -) -_settings.security.validator_service_endpoint_url = os.environ.get( - "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_ENDPOINT_URL", - default=None -) -_settings.security.log_level = os.environ.get( - "NEW_RELIC_SECURITY_LOG_LEVEL", - default="INFO" -) -_settings.security.sec_home_path = os.environ.get( - "NEW_RELIC_SECURITY_SEC_HOME_PATH", - default="/opt" -) -_settings.security.detection.disable_rci = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_RCI", - default=False -) -_settings.security.detection.disable_rxss = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_RXSS", - default=False -) -_settings.security.detection.disable_deserialization = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DISABLE_DESERIALIZATION", - default=False -) - -_settings.security.policy.enforce = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_ENFORCE", - default=False -) -_settings.security.policy.vulnerabilityScan.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_ENABLED", - default=None -) -_settings.security.policy.vulnerabilityScan.iastScan.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_ENABLED", - default=None + default="IAST" ) -_settings.security.policy.vulnerabilityScan.iastScan.probing.interval = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_INTERVAL", - default=-1 -) -_settings.security.policy.vulnerabilityScan.iastScan.probing.batchSize = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_VULNERABILITYSCAN_IASTSCAN_PROBING_BATCHSIZE", - default=-1 -) -_settings.security.policy.protectionMode.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_ENABLED", +_settings.security.validator_service_url = os.environ.get( + "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", default=None ) -_settings.security.policy.protectionMode.ipBlocking.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ENABLED", - default=None +_settings.security.detection.rci.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", + default=True ) -_settings.security.policy.protectionMode.ipBlocking.attackerIpBlocking = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_ATTACKERIPBLOCKING", - default=None +_settings.security.detection.rxss.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", + default=True ) -_settings.security.policy.protectionMode.ipBlocking.ipDetectViaXFF = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_IPDETECTVIAXFF", - default=None -) -_settings.security.policy.protectionMode.ipBlocking.timeout = _environ_as_int( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_IPBLOCKING_TIMEOUT", - default=-1 -) -_settings.security.policy.protectionMode.apiBlocking.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_ENABLED", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectAllApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTALLAPIS", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTKNOWNVULNERABLEAPIS", - default=None -) -_settings.security.policy.protectionMode.apiBlocking.protectAttackedApis = _environ_as_bool( - "NEW_RELIC_SECURITY_POLICY_PROTECTIONMODE_APIBLOCKING_PROTECTATTACKEDAPIS", - default=None +_settings.security.detection.deserialization.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", + default=True ) diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index f1a589b2e3..4e43171a54 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -49,47 +49,31 @@ app_name = Python Application # NEW_RELIC_MONITOR_MODE environment variable. monitor_mode = true -# When enabled, the agent's security module is not loaded -# this would mean changes on soft reloads won't work -security.force_complete_disable = false -# When enabled, the agent collects security data about your -# application and reports this data to the New Relic. -security.enable = true +# Indicates if attack detection security module is to be enabled +security.enabled = false + +# To completely disable security set flag to false If the flag is +# set to false, the security module is not loaded. This property +# is read only once at application start. +security.agent.enabled = false + + # security module provides two modes IAST or RASP # RASP stands for Runtime Application Self Protection # while IAST for Interactive Application Security Testing -# Default mode is RASP -security.mode = RASP +# Default mode is IAST +security.mode = IAST + # web-protect agent endpoint connection URLs -security.validator_service_endpoint_url = - -# log level -security.log_level = INFO - -# security file settings -security.sec_home_path = /tmp - -# vulnerabilty detection disable -# security.detection.disable_rci = false -# security.detection.disable_rxss = false -# security.detection.disable_deserialization = false - -# policy settings -# security.policy.enforce = false -# security.policy.vulnerabilityScan.enabled = false -# security.policy.vulnerabilityScan.iastScan.enabled = false -# security.policy.vulnerabilityScan.iastScan.probing.interval = 1 -# security.policy.vulnerabilityScan.iastScan.probing.batchSize = 5 -# security.policy.protectionMode.enabled = false -# security.policy.protectionMode.ipBlocking.enabled = false -# security.policy.protectionMode.ipBlocking.attackerIpBlocking = false -# security.policy.protectionMode.ipBlocking.ipDetectViaXFF = false -# security.policy.protectionMode.ipBlocking.timeout = 120 -# security.policy.protectionMode.apiBlocking.enabled = false -# security.policy.protectionMode.apiBlocking.protectAllApis = false -# security.policy.protectionMode.apiBlocking.protectKnownVulnerableApis = false -# security.policy.protectionMode.apiBlocking.protectAttackedApis = false +security.validator_service_url = wss://csec.nr-data.net + + +# vulnerabilty detection flags +security.detection.rci.enabled = true +security.detection.rxss.enabled = true +security.detection.deserialization.enabled = true + # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the From 5be183660254dfe9d094eb6ad6041f042e7da9e4 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 23 Mar 2023 10:44:15 +0530 Subject: [PATCH 034/525] minor fix --- newrelic/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/newrelic/config.py b/newrelic/config.py index 03df8cde5c..b2988dc1a2 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3103,7 +3103,7 @@ def _setup_security_module(): callback to agent startup to propagate NR config """ try: - if _settings.security.force_complete_disable: + if not _settings.security.agent.enabled: return from newrelic_security.api.agent import Agent as SecurityAgent # initialize security agent From ac7e712621782ec45f01c327d9b282b2e70bfd7c Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Mon, 17 Apr 2023 14:32:34 +0530 Subject: [PATCH 035/525] point to nr_adaptation with updated newrelic_security package --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f6b8fafb23..c6cb15f394 100644 --- a/setup.py +++ b/setup.py @@ -153,7 +153,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["k2_python_agent @ git+https://github.com/k2io/k2-python-agent@newrelic_integration#egg=k2_python_agent"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@nr_adaptation#egg=newrelic_security"] ) if with_setuptools: From 28d789a7ca2ad0cf4b63e4550712ab08d30e7abb Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 28 Apr 2023 17:30:26 +0530 Subject: [PATCH 036/525] Refactoring and relocation of security settings in core config --- newrelic/core/config.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 23fc44d5f1..3cf9875102 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,7 +278,7 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass -class SecurityModuleSettings(Settings): +class SecuritySettings(Settings): pass class SecurityDetectionSettings(Settings): @@ -414,6 +414,12 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.message_tracer = MessageTracerSettings() _settings.process_host = ProcessHostSettings() _settings.rum = RumSettings() +_settings.security = SecuritySettings() +_settings.security.agent = SecurityAgentSettings() +_settings.security.detection = SecurityDetectionSettings() +_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() +_settings.security.detection.rci = SecurityDetectionRCISettings() +_settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -858,14 +864,6 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security = SecurityModuleSettings() -_settings.security.agent = SecurityAgentSettings() -_settings.security.detection = SecurityDetectionSettings() -_settings.security.detection.rci = SecurityDetectionRCISettings() -_settings.security.detection.rxss = SecurityDetectionRXSSSettings() -_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() - - _settings.security.agent.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_AGENT_ENABLED", default=False From 1f780c2094a0f8e19cf1cee3749ca9bfd1a2c806 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 4 May 2023 10:21:35 +0530 Subject: [PATCH 037/525] initialising security agent before configuring nr apm hooks --- newrelic/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index b1ec8f98d8..c78f4e481c 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3223,6 +3223,8 @@ def initialize( _load_configuration(config_file, environment, ignore_errors, log_file, log_level) + _setup_security_module() + if _settings.monitor_mode or _settings.developer_mode: _settings.enabled = True _setup_instrumentation() @@ -3232,8 +3234,6 @@ def initialize( else: _settings.enabled = False - _setup_security_module() - def filter_app_factory(app, global_conf, config_file, environment=None): initialize(config_file, environment) From 5fb7a423d69bf67bb751951760eb7351f8532b94 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 18 May 2023 00:23:25 +0530 Subject: [PATCH 038/525] fixes in config default values for security config for python 2.7 support --- newrelic/core/config.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 3cf9875102..cb8dccb7bd 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -866,31 +866,31 @@ def default_host(license_key): _settings.security.agent.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_AGENT_ENABLED", - default=False + False ) _settings.security.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_ENABLED", - default=False + False ) _settings.security.mode = os.environ.get( "NEW_RELIC_SECURITY_MODE", - default="IAST" + "IAST" ) _settings.security.validator_service_url = os.environ.get( "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", - default=None + None ) _settings.security.detection.rci.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", - default=True + True ) _settings.security.detection.rxss.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", - default=True + True ) _settings.security.detection.deserialization.enabled = _environ_as_bool( "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", - default=True + True ) From 11c0049413c81ccc53dee285bd03691e2ef92879 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Fri, 26 May 2023 10:24:16 +0530 Subject: [PATCH 039/525] Will use dev branch for security agent --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8eafb356ad..ecc43a64a0 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@nr_adaptation#egg=newrelic_security"] + install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@dev#egg=newrelic_security"] ) if with_setuptools: From 093fd3a576aa65ee0ce1b27eafb4a858ce2dc3f8 Mon Sep 17 00:00:00 2001 From: Anupam Juniwal Date: Thu, 1 Jun 2023 11:45:52 +0530 Subject: [PATCH 040/525] Updated remote for pulling newrelic_security module --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ecc43a64a0..c960cd357a 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/k2io/k2-python-agent@dev#egg=newrelic_security"] + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] ) if with_setuptools: From 88cb32e0829744548fc756bb1125cbd55922d3f4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 11:05:37 -0700 Subject: [PATCH 041/525] Update install requires line. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c960cd357a..44b4957d32 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] + install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] ) if with_setuptools: From 7a43a7fae139795e40cd802f6c319a85383da67b Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 18:08:32 +0000 Subject: [PATCH 042/525] [Mega-Linter] Apply linters fixes --- newrelic/config.py | 3 ++- newrelic/core/config.py | 40 +++++++++++++--------------------------- setup.py | 2 +- 3 files changed, 16 insertions(+), 29 deletions(-) diff --git a/newrelic/config.py b/newrelic/config.py index 8009938c0e..d856118995 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3184,9 +3184,10 @@ def _setup_security_module(): if not _settings.security.agent.enabled: return from newrelic_security.api.agent import Agent as SecurityAgent + # initialize security agent security_agent = SecurityAgent() - # create a callback to reinitialise the security module + # create a callback to reinitialise the security module newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) except Exception as k2error: _logger.error("K2 Startup failed with error %s", k2error) diff --git a/newrelic/core/config.py b/newrelic/core/config.py index cb8dccb7bd..03758fc74a 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -278,26 +278,31 @@ class ApplicationLoggingMetricsSettings(Settings): class ApplicationLoggingLocalDecoratingSettings(Settings): pass + class SecuritySettings(Settings): pass + class SecurityDetectionSettings(Settings): pass + class SecurityAgentSettings(Settings): pass + class SecurityDetectionRCISettings(Settings): pass + class SecurityDetectionRXSSSettings(Settings): pass + class SecurityDetectionDeserializationSettings(Settings): pass - class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -864,33 +869,14 @@ def default_host(license_key): "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) -_settings.security.agent.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_AGENT_ENABLED", - False -) -_settings.security.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_ENABLED", - False -) -_settings.security.mode = os.environ.get( - "NEW_RELIC_SECURITY_MODE", - "IAST" -) -_settings.security.validator_service_url = os.environ.get( - "NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", - None -) -_settings.security.detection.rci.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", - True -) -_settings.security.detection.rxss.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", - True -) +_settings.security.agent.enabled = _environ_as_bool("NEW_RELIC_SECURITY_AGENT_ENABLED", False) +_settings.security.enabled = _environ_as_bool("NEW_RELIC_SECURITY_ENABLED", False) +_settings.security.mode = os.environ.get("NEW_RELIC_SECURITY_MODE", "IAST") +_settings.security.validator_service_url = os.environ.get("NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", None) +_settings.security.detection.rci.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", True) +_settings.security.detection.rxss.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", True) _settings.security.detection.deserialization.enabled = _environ_as_bool( - "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", - True + "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True ) diff --git a/setup.py b/setup.py index 44b4957d32..acb5a4ad9a 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"] + install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], ) if with_setuptools: From 3d44b8c2388d79b15778a3051180664d219eff2f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 12:11:05 -0700 Subject: [PATCH 043/525] Restore install requires. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index acb5a4ad9a..5811e85933 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,7 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], ) if with_setuptools: From fe17e3632239387f7f53bd6d5873b0728c4a37b8 Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 19:12:37 +0000 Subject: [PATCH 044/525] [Mega-Linter] Apply linters fixes --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5811e85933..76340f55e9 100644 --- a/setup.py +++ b/setup.py @@ -152,7 +152,9 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, - install_requires=["newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + install_requires=[ + "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" + ], ) if with_setuptools: From b0e0a5053b3aa3e9e507e582dce8055c8d53bd87 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 13:02:05 -0700 Subject: [PATCH 045/525] switch ordering --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 76340f55e9..59d985b549 100644 --- a/setup.py +++ b/setup.py @@ -151,10 +151,10 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, install_requires=[ "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" ], + extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) if with_setuptools: From dada30bdc7135197223cd60a049ddf86518f0e75 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 13:22:17 -0700 Subject: [PATCH 046/525] Add dependency links. --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 59d985b549..49f83401ff 100644 --- a/setup.py +++ b/setup.py @@ -152,8 +152,9 @@ def build_extension(self, ext): "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, install_requires=[ - "newrelic_security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security" + "newrelic_security" ], + dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) From 78d90b65da16156f2bf46ae3aa594a28a522ed48 Mon Sep 17 00:00:00 2001 From: umaannamalai Date: Thu, 1 Jun 2023 20:23:41 +0000 Subject: [PATCH 047/525] [Mega-Linter] Apply linters fixes --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 49f83401ff..321ffbeb90 100644 --- a/setup.py +++ b/setup.py @@ -151,9 +151,7 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - install_requires=[ - "newrelic_security" - ], + install_requires=["newrelic_security"], dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) From 1b0c1b5cfc2bb94f3dbe02680f6f789d4a98065f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 15:32:25 -0700 Subject: [PATCH 048/525] Add security settings to web framework conftests. --- setup.py | 3 +-- tests/framework_bottle/conftest.py | 4 ++++ tests/framework_django/conftest.py | 4 ++++ tests/framework_flask/conftest.py | 5 ++++- tox.ini | 2 ++ 5 files changed, 15 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 321ffbeb90..c5a564b048 100644 --- a/setup.py +++ b/setup.py @@ -151,8 +151,7 @@ def build_extension(self, ext): package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, - install_requires=["newrelic_security"], - dependency_links=["git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic_security"], + #install_requires=["newrelic-security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) diff --git a/tests/framework_bottle/conftest.py b/tests/framework_bottle/conftest.py index 095a3331f3..f03f6461b5 100644 --- a/tests/framework_bottle/conftest.py +++ b/tests/framework_bottle/conftest.py @@ -23,6 +23,10 @@ 'transaction_tracer.stack_trace_threshold': 0.0, 'debug.log_data_collector_payloads': True, 'debug.record_transaction_failure': True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tests/framework_django/conftest.py b/tests/framework_django/conftest.py index 8a43ef5c90..6556447321 100644 --- a/tests/framework_django/conftest.py +++ b/tests/framework_django/conftest.py @@ -25,6 +25,10 @@ 'debug.record_transaction_failure': True, 'debug.log_autorum_middleware': True, 'feature_flag': set(['django.instrumentation.inclusion-tags.r1']), + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tests/framework_flask/conftest.py b/tests/framework_flask/conftest.py index f90ed9b512..8029ad2868 100644 --- a/tests/framework_flask/conftest.py +++ b/tests/framework_flask/conftest.py @@ -19,7 +19,6 @@ from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - _default_settings = { 'transaction_tracer.explain_threshold': 0.0, 'transaction_tracer.transaction_threshold': 0.0, @@ -27,6 +26,10 @@ 'debug.log_data_collector_payloads': True, 'debug.record_transaction_failure': True, 'debug.log_autorum_middleware': True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( diff --git a/tox.ini b/tox.ini index 147851088c..2059abbe4a 100644 --- a/tox.ini +++ b/tox.ini @@ -173,8 +173,10 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage + git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security WebTest==2.0.35 + # Test Suite Dependencies adapter_cheroot: cheroot adapter_daphne-daphnelatest: daphne From 14d70192a274f1094c639a75ef3e9c7b0ffd5870 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 17:17:29 -0700 Subject: [PATCH 049/525] github ssh url. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 2059abbe4a..a974c9d627 100644 --- a/tox.ini +++ b/tox.ini @@ -173,7 +173,7 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage - git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security + git+ssh://git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security WebTest==2.0.35 From 874fd035b2556779c37cf44d818201e6a3aad38f Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 1 Jun 2023 17:26:14 -0700 Subject: [PATCH 050/525] remove ssh prefix. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a974c9d627..3d4a9298b6 100644 --- a/tox.ini +++ b/tox.ini @@ -173,7 +173,7 @@ deps = {py27,pypy}: pytest==4.6.11 iniconfig coverage - git+ssh://git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security + git+git@github.com:newrelic/csec-python-agent.git#egg=newrelic-security WebTest==2.0.35 From f192c24a178c46b04f3933c2a3eba6d3653ea931 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 12 Jun 2023 16:15:54 -0700 Subject: [PATCH 051/525] Fix Testing Failures (#828) * Fix tastypie tests * Adjust asgiref pinned version * Make aioredis key PID unique * Pin more asgiref versions --- tests/datastore_aioredis/conftest.py | 5 +++ tests/datastore_aioredis/test_get_and_set.py | 14 ++++---- tests/datastore_aioredis/test_transactions.py | 36 +++++++++---------- tox.ini | 2 ++ 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/tests/datastore_aioredis/conftest.py b/tests/datastore_aioredis/conftest.py index d501292555..e1cea4c01b 100644 --- a/tests/datastore_aioredis/conftest.py +++ b/tests/datastore_aioredis/conftest.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import pytest from newrelic.common.package_version_utils import get_package_version_tuple @@ -67,3 +68,7 @@ def client(request, loop): pytest.skip("StrictRedis not implemented.") else: raise NotImplementedError() + +@pytest.fixture(scope="session") +def key(): + return "AIOREDIS-TEST-" + str(os.getpid()) diff --git a/tests/datastore_aioredis/test_get_and_set.py b/tests/datastore_aioredis/test_get_and_set.py index 180f325788..cbddf6091b 100644 --- a/tests/datastore_aioredis/test_get_and_set.py +++ b/tests/datastore_aioredis/test_get_and_set.py @@ -64,9 +64,9 @@ _disable_rollup_metrics.append((_instance_metric_name, None)) -async def exercise_redis(client): - await client.set("key", "value") - await client.get("key") +async def exercise_redis(client, key): + await client.set(key, "value") + await client.get(key) @override_application_settings(_enable_instance_settings) @@ -77,8 +77,8 @@ async def exercise_redis(client): background_task=True, ) @background_task() -def test_redis_client_operation_enable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_enable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) @override_application_settings(_disable_instance_settings) @@ -89,5 +89,5 @@ def test_redis_client_operation_enable_instance(client, loop): background_task=True, ) @background_task() -def test_redis_client_operation_disable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_disable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) diff --git a/tests/datastore_aioredis/test_transactions.py b/tests/datastore_aioredis/test_transactions.py index 0f84ca684e..ced9220225 100644 --- a/tests/datastore_aioredis/test_transactions.py +++ b/tests/datastore_aioredis/test_transactions.py @@ -23,42 +23,46 @@ @background_task() @pytest.mark.parametrize("in_transaction", (True, False)) -def test_pipelines_no_harm(client, in_transaction, loop): +def test_pipelines_no_harm(client, in_transaction, loop, key): async def exercise(): if AIOREDIS_VERSION >= (2,): pipe = client.pipeline(transaction=in_transaction) else: pipe = client.pipeline() # Transaction kwarg unsupported - pipe.set("TXN", 1) + pipe.set(key, 1) return await pipe.execute() status = loop.run_until_complete(exercise()) assert status == [True] -def exercise_transaction_sync(pipe): - pipe.set("TXN", 1) +def exercise_transaction_sync(key): + def _run(pipe): + pipe.set(key, 1) + return _run -async def exercise_transaction_async(pipe): - await pipe.set("TXN", 1) +def exercise_transaction_async(key): + async def _run(pipe): + await pipe.set(key, 1) + return _run @SKIPIF_AIOREDIS_V1 @pytest.mark.parametrize("exercise", (exercise_transaction_sync, exercise_transaction_async)) @background_task() -def test_transactions_no_harm(client, loop, exercise): - status = loop.run_until_complete(client.transaction(exercise)) +def test_transactions_no_harm(client, loop, key, exercise): + status = loop.run_until_complete(client.transaction(exercise(key))) assert status == [True] @SKIPIF_AIOREDIS_V2 @background_task() -def test_multi_exec_no_harm(client, loop): +def test_multi_exec_no_harm(client, loop, key): async def exercise(): pipe = client.multi_exec() - pipe.set("key", "value") + pipe.set(key, "value") status = await pipe.execute() assert status == [True] @@ -67,9 +71,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_pipeline_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_pipeline_immediate_execution_no_harm(client, loop, key): async def exercise(): await client.set(key, 1) @@ -94,9 +96,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_transaction_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_immediate_execution_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) @@ -119,9 +119,7 @@ async def exercise_transaction(pipe): @SKIPIF_AIOREDIS_V1 @validate_transaction_errors([]) @background_task() -def test_transaction_watch_error_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_watch_error_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) diff --git a/tox.ini b/tox.ini index 147851088c..fbfc0ee52a 100644 --- a/tox.ini +++ b/tox.ini @@ -221,8 +221,10 @@ deps = component_tastypie-tastypie0143: django-tastypie<0.14.4 component_tastypie-{py27,pypy}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 + component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie component_tastypie-tastypielatest: django<4.1 + component_tastypie-tastypielatest: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ coroutines_asyncio-{py37,py38,py39,py310,py311}: uvloop cross_agent: mock==1.0.1 cross_agent: requests From 3def8b0d4e753d38a255217f9c712e5561316517 Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 12 Jun 2023 16:53:06 -0700 Subject: [PATCH 052/525] Fix pytest test filtering when running tox (#823) Co-authored-by: Uma Annamalai --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index fbfc0ee52a..7e4f1e3707 100644 --- a/tox.ini +++ b/tox.ini @@ -408,7 +408,7 @@ commands = framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl - coverage run -m pytest -v + coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* From 90ccb4cdbf65f032cd993c488ad7e7daced299e4 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 14 Jun 2023 16:35:57 -0700 Subject: [PATCH 053/525] Validator transfer p3 (#745) * Move validate_transaction_metrics to validators directory * Comment out original validate_transaction_metrics from fixtures.py * Move validate_time_metrics_outside_transaction to validators directory * Move validate_internal_metrics into validators directory and fixed validate_transaction_metrics * Move validate_transaction_errors into validators directory * Move validate_application_errors into validators directory * Move validate_custom_parameters into validators directory * Move validate_synthetics_event into validators directory * Move validate_transaction_event_attributes into validators directory * Move validate_non_transaction_error_event into validators directory * Move validate_application_error_trace_count into validators directory * Move validate_application_error_event_count into validators directory * Move validate_synthetics_transaction_trace into validators directory * Move validate_tt_collector_json to validators directory * Move validate_transaction_trace_attributes into validator directory * Move validate_transaction_error_trace_attributes into validator directory * Move validate_error_trace_collector_json into validator directory * Move validate_error_event_collector_json into validator directory * Move validate_transaction_event_collector_json into validator directory * Move validate_custom_event_collector_json into validator directory * Move validate_tt_parameters into validator directory * Move validate_tt_parameters into validator directory * Move validate_tt_segment_params into validator directory * Move validate_browser_attributes into validators directory * Move validate_error_event_attributes into validators directory * Move validate_error_trace_attributes_outside_transaction into validators directory * Move validate_error_event_attributes_outside_transaction into validators directory * Fix some pylint errors * Redirect check_error_attributes * Fix more Pylint errors * Fix import issues from move * Fix more import shuffle errors * Sort logging JSON test for PY2 consistency * Fix Pylint errors in validators * Fix import error --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/agent_features/test_asgi_browser.py | 2 +- .../test_attributes_in_action.py | 29 +- .../agent_features/test_collector_payloads.py | 8 +- .../test_distributed_tracing.py | 337 +++++++------ .../test_error_group_callback.py | 111 +++-- .../agent_features/test_high_security_mode.py | 4 +- .../test_ignore_expected_errors.py | 6 +- tests/agent_features/test_span_events.py | 6 +- .../test_transaction_trace_segments.py | 4 +- tests/agent_features/test_wsgi_attributes.py | 7 +- tests/cross_agent/test_cat_map.py | 168 ++++--- tests/cross_agent/test_distributed_tracing.py | 214 +++++---- tests/external_boto3/test_boto3_iam.py | 70 +-- tests/external_boto3/test_boto3_sns.py | 111 +++-- .../test_botocore_dynamodb.py | 192 ++++---- tests/external_botocore/test_botocore_ec2.py | 86 ++-- tests/external_httplib/test_httplib.py | 16 +- tests/external_httpx/test_client.py | 12 +- tests/framework_aiohttp/test_server_cat.py | 2 +- .../test_consumer.py | 9 +- .../test_consumer.py | 9 +- .../test_serialization.py | 4 +- tests/testing_support/fixtures.py | 446 +++--------------- .../validators/validate_browser_attributes.py | 74 +++ .../validate_custom_event_collector_json.py | 64 +++ .../validate_error_event_attributes.py | 51 ++ ...or_event_attributes_outside_transaction.py | 48 ++ ...or_trace_attributes_outside_transaction.py | 45 ++ .../validators/validate_tt_parameters.py | 52 ++ .../validators/validate_tt_segment_params.py | 91 ++++ 30 files changed, 1237 insertions(+), 1041 deletions(-) create mode 100644 tests/testing_support/validators/validate_browser_attributes.py create mode 100644 tests/testing_support/validators/validate_custom_event_collector_json.py create mode 100644 tests/testing_support/validators/validate_error_event_attributes.py create mode 100644 tests/testing_support/validators/validate_error_event_attributes_outside_transaction.py create mode 100644 tests/testing_support/validators/validate_error_trace_attributes_outside_transaction.py create mode 100644 tests/testing_support/validators/validate_tt_parameters.py create mode 100644 tests/testing_support/validators/validate_tt_segment_params.py diff --git a/tests/agent_features/test_asgi_browser.py b/tests/agent_features/test_asgi_browser.py index 1e718e1e0d..281d08b967 100644 --- a/tests/agent_features/test_asgi_browser.py +++ b/tests/agent_features/test_asgi_browser.py @@ -111,7 +111,7 @@ def test_footer_attributes(): obfuscation_key = settings.license_key[:13] - type_transaction_data = unicode if six.PY2 else str # noqa: F821 + type_transaction_data = unicode if six.PY2 else str # noqa: F821, pylint: disable=E0602 assert isinstance(data["transactionName"], type_transaction_data) txn_name = deobfuscate(data["transactionName"], obfuscation_key) diff --git a/tests/agent_features/test_attributes_in_action.py b/tests/agent_features/test_attributes_in_action.py index e56994d0a1..08601fccf6 100644 --- a/tests/agent_features/test_attributes_in_action.py +++ b/tests/agent_features/test_attributes_in_action.py @@ -20,14 +20,22 @@ override_application_settings, reset_core_stats_engine, validate_attributes, +) +from testing_support.validators.validate_browser_attributes import ( validate_browser_attributes, +) +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( validate_error_event_attributes_outside_transaction, - validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( + validate_error_trace_attributes_outside_transaction, +) from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_error_trace_attributes import ( validate_transaction_error_trace_attributes, @@ -43,7 +51,7 @@ from newrelic.api.background_task import background_task from newrelic.api.message_transaction import message_transaction from newrelic.api.time_trace import notice_error -from newrelic.api.transaction import add_custom_attribute, current_transaction, set_user_id +from newrelic.api.transaction import add_custom_attribute, set_user_id from newrelic.api.wsgi_application import wsgi_application from newrelic.common.object_names import callable_name @@ -930,16 +938,21 @@ def test_none_type_routing_key_agent_attribute(): _forgone_agent_attributes = [] -@pytest.mark.parametrize('input_user_id, reported_user_id, high_security',( +@pytest.mark.parametrize( + "input_user_id, reported_user_id, high_security", + ( ("1234", "1234", True), - ("a" * 260, "a" * 255, False), -)) + ("a" * 260, "a" * 255, False), + ), +) def test_enduser_id_attribute_api_valid_types(input_user_id, reported_user_id, high_security): @reset_core_stats_engine() @validate_error_trace_attributes( callable_name(ValueError), exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}} ) - @validate_error_event_attributes(exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}}) + @validate_error_event_attributes( + exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}} + ) @validate_attributes("agent", _required_agent_attributes, _forgone_agent_attributes) @background_task() @override_application_settings({"high_security": high_security}) @@ -950,10 +963,11 @@ def _test(): raise ValueError() except Exception: notice_error() + _test() -@pytest.mark.parametrize('input_user_id',(None, '', 123)) +@pytest.mark.parametrize("input_user_id", (None, "", 123)) def test_enduser_id_attribute_api_invalid_types(input_user_id): @reset_core_stats_engine() @validate_attributes("agent", [], ["enduser.id"]) @@ -965,4 +979,5 @@ def _test(): raise ValueError() except Exception: notice_error() + _test() diff --git a/tests/agent_features/test_collector_payloads.py b/tests/agent_features/test_collector_payloads.py index 0c1b2367ce..42510e5c74 100644 --- a/tests/agent_features/test_collector_payloads.py +++ b/tests/agent_features/test_collector_payloads.py @@ -14,15 +14,15 @@ import pytest import webtest -from testing_support.fixtures import ( - override_application_settings, - validate_custom_event_collector_json, -) +from testing_support.fixtures import override_application_settings from testing_support.sample_applications import ( simple_app, simple_custom_event_app, simple_exceptional_app, ) +from testing_support.validators.validate_custom_event_collector_json import ( + validate_custom_event_collector_json, +) from testing_support.validators.validate_error_event_collector_json import ( validate_error_event_collector_json, ) diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 7f795573a6..4db6d2dab9 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -12,71 +12,82 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import json + import pytest import webtest -import copy +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_error_event_attributes import ( + validate_error_event_attributes, +) +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.application import application_instance -from newrelic.api.background_task import background_task, BackgroundTask -from newrelic.api.transaction import (current_transaction, current_trace_id, - current_span_id) +from newrelic.api.background_task import BackgroundTask, background_task from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import ( + current_span_id, + current_trace_id, + current_transaction, +) from newrelic.api.web_transaction import WSGIWebTransaction from newrelic.api.wsgi_application import wsgi_application -from testing_support.fixtures import (override_application_settings, - validate_attributes, - validate_error_event_attributes) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes - -distributed_trace_intrinsics = ['guid', 'traceId', 'priority', 'sampled'] -inbound_payload_intrinsics = ['parent.type', 'parent.app', 'parent.account', - 'parent.transportType', 'parent.transportDuration'] +distributed_trace_intrinsics = ["guid", "traceId", "priority", "sampled"] +inbound_payload_intrinsics = [ + "parent.type", + "parent.app", + "parent.account", + "parent.transportType", + "parent.transportDuration", +] payload = { - 'v': [0, 1], - 'd': { - 'ac': '1', - 'ap': '2827902', - 'id': '7d3efb1b173fecfa', - 'pa': '5e5733a911cfbc73', - 'pr': 10.001, - 'sa': True, - 'ti': 1518469636035, - 'tr': 'd6b4ba0c3a712ca', - 'ty': 'App', - } + "v": [0, 1], + "d": { + "ac": "1", + "ap": "2827902", + "id": "7d3efb1b173fecfa", + "pa": "5e5733a911cfbc73", + "pr": 10.001, + "sa": True, + "ti": 1518469636035, + "tr": "d6b4ba0c3a712ca", + "ty": "App", + }, } -parent_order = ['parent_type', 'parent_account', - 'parent_app', 'parent_transport_type'] +parent_order = ["parent_type", "parent_account", "parent_app", "parent_transport_type"] parent_info = { - 'parent_type': payload['d']['ty'], - 'parent_account': payload['d']['ac'], - 'parent_app': payload['d']['ap'], - 'parent_transport_type': 'HTTP' + "parent_type": payload["d"]["ty"], + "parent_account": payload["d"]["ac"], + "parent_app": payload["d"]["ap"], + "parent_transport_type": "HTTP", } @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' - output = b'hello world' - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + status = "200 OK" + output = b"hello world" + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] txn = current_transaction() # Make assertions on the WSGIWebTransaction object assert txn._distributed_trace_state - assert txn.parent_type == 'App' - assert txn.parent_app == '2827902' - assert txn.parent_account == '1' - assert txn.parent_span == '7d3efb1b173fecfa' - assert txn.parent_transport_type == 'HTTP' + assert txn.parent_type == "App" + assert txn.parent_app == "2827902" + assert txn.parent_account == "1" + assert txn.parent_span == "7d3efb1b173fecfa" + assert txn.parent_transport_type == "HTTP" assert isinstance(txn.parent_transport_duration, float) - assert txn._trace_id == 'd6b4ba0c3a712ca' + assert txn._trace_id == "d6b4ba0c3a712ca" assert txn.priority == 10.001 assert txn.sampled @@ -87,90 +98,75 @@ def target_wsgi_application(environ, start_response): test_application = webtest.TestApp(target_wsgi_application) _override_settings = { - 'trusted_account_key': '1', - 'distributed_tracing.enabled': True, + "trusted_account_key": "1", + "distributed_tracing.enabled": True, } _metrics = [ - ('Supportability/DistributedTrace/AcceptPayload/Success', 1), - ('Supportability/TraceContext/Accept/Success', None) + ("Supportability/DistributedTrace/AcceptPayload/Success", 1), + ("Supportability/TraceContext/Accept/Success", None), ] @override_application_settings(_override_settings) -@validate_transaction_metrics( - '', - group='Uri', - rollup_metrics=_metrics) +@validate_transaction_metrics("", group="Uri", rollup_metrics=_metrics) def test_distributed_tracing_web_transaction(): - headers = {'newrelic': json.dumps(payload)} - response = test_application.get('/', headers=headers) - assert 'X-NewRelic-App-Data' not in response.headers + headers = {"newrelic": json.dumps(payload)} + response = test_application.get("/", headers=headers) + assert "X-NewRelic-App-Data" not in response.headers -@pytest.mark.parametrize('span_events', (True, False)) -@pytest.mark.parametrize('accept_payload', (True, False)) +@pytest.mark.parametrize("span_events", (True, False)) +@pytest.mark.parametrize("accept_payload", (True, False)) def test_distributed_trace_attributes(span_events, accept_payload): if accept_payload: - _required_intrinsics = ( - distributed_trace_intrinsics + inbound_payload_intrinsics) + _required_intrinsics = distributed_trace_intrinsics + inbound_payload_intrinsics _forgone_txn_intrinsics = [] _forgone_error_intrinsics = [] _exact_intrinsics = { - 'parent.type': 'Mobile', - 'parent.app': '2827902', - 'parent.account': '1', - 'parent.transportType': 'HTTP', - 'traceId': 'd6b4ba0c3a712ca', + "parent.type": "Mobile", + "parent.app": "2827902", + "parent.account": "1", + "parent.transportType": "HTTP", + "traceId": "d6b4ba0c3a712ca", } - _exact_txn_attributes = {'agent': {}, 'user': {}, - 'intrinsic': _exact_intrinsics.copy()} - _exact_error_attributes = {'agent': {}, 'user': {}, - 'intrinsic': _exact_intrinsics.copy()} - _exact_txn_attributes['intrinsic']['parentId'] = '7d3efb1b173fecfa' - _exact_txn_attributes['intrinsic']['parentSpanId'] = 'c86df80de2e6f51c' - - _forgone_error_intrinsics.append('parentId') - _forgone_error_intrinsics.append('parentSpanId') - _forgone_txn_intrinsics.append('grandparentId') - _forgone_error_intrinsics.append('grandparentId') - - _required_attributes = { - 'intrinsic': _required_intrinsics, 'agent': [], 'user': []} - _forgone_txn_attributes = {'intrinsic': _forgone_txn_intrinsics, - 'agent': [], 'user': []} - _forgone_error_attributes = {'intrinsic': _forgone_error_intrinsics, - 'agent': [], 'user': []} + _exact_txn_attributes = {"agent": {}, "user": {}, "intrinsic": _exact_intrinsics.copy()} + _exact_error_attributes = {"agent": {}, "user": {}, "intrinsic": _exact_intrinsics.copy()} + _exact_txn_attributes["intrinsic"]["parentId"] = "7d3efb1b173fecfa" + _exact_txn_attributes["intrinsic"]["parentSpanId"] = "c86df80de2e6f51c" + + _forgone_error_intrinsics.append("parentId") + _forgone_error_intrinsics.append("parentSpanId") + _forgone_txn_intrinsics.append("grandparentId") + _forgone_error_intrinsics.append("grandparentId") + + _required_attributes = {"intrinsic": _required_intrinsics, "agent": [], "user": []} + _forgone_txn_attributes = {"intrinsic": _forgone_txn_intrinsics, "agent": [], "user": []} + _forgone_error_attributes = {"intrinsic": _forgone_error_intrinsics, "agent": [], "user": []} else: _required_intrinsics = distributed_trace_intrinsics - _forgone_txn_intrinsics = _forgone_error_intrinsics = \ - inbound_payload_intrinsics + ['grandparentId', 'parentId', - 'parentSpanId'] - - _required_attributes = { - 'intrinsic': _required_intrinsics, 'agent': [], 'user': []} - _forgone_txn_attributes = {'intrinsic': _forgone_txn_intrinsics, - 'agent': [], 'user': []} - _forgone_error_attributes = {'intrinsic': _forgone_error_intrinsics, - 'agent': [], 'user': []} + _forgone_txn_intrinsics = _forgone_error_intrinsics = inbound_payload_intrinsics + [ + "grandparentId", + "parentId", + "parentSpanId", + ] + + _required_attributes = {"intrinsic": _required_intrinsics, "agent": [], "user": []} + _forgone_txn_attributes = {"intrinsic": _forgone_txn_intrinsics, "agent": [], "user": []} + _forgone_error_attributes = {"intrinsic": _forgone_error_intrinsics, "agent": [], "user": []} _exact_txn_attributes = _exact_error_attributes = None _forgone_trace_intrinsics = _forgone_error_intrinsics test_settings = _override_settings.copy() - test_settings['span_events.enabled'] = span_events + test_settings["span_events.enabled"] = span_events @override_application_settings(test_settings) - @validate_transaction_event_attributes( - _required_attributes, _forgone_txn_attributes, - _exact_txn_attributes) - @validate_error_event_attributes( - _required_attributes, _forgone_error_attributes, - _exact_error_attributes) - @validate_attributes('intrinsic', - _required_intrinsics, _forgone_trace_intrinsics) - @background_task(name='test_distributed_trace_attributes') + @validate_transaction_event_attributes(_required_attributes, _forgone_txn_attributes, _exact_txn_attributes) + @validate_error_event_attributes(_required_attributes, _forgone_error_attributes, _exact_error_attributes) + @validate_attributes("intrinsic", _required_intrinsics, _forgone_trace_intrinsics) + @background_task(name="test_distributed_trace_attributes") def _test(): txn = current_transaction() @@ -183,10 +179,10 @@ def _test(): "id": "c86df80de2e6f51c", "tr": "d6b4ba0c3a712ca", "ti": 1518469636035, - "tx": "7d3efb1b173fecfa" - } + "tx": "7d3efb1b173fecfa", + }, } - payload['d']['pa'] = "5e5733a911cfbc73" + payload["d"]["pa"] = "5e5733a911cfbc73" if accept_payload: result = txn.accept_distributed_trace_payload(payload) @@ -195,7 +191,7 @@ def _test(): txn._create_distributed_trace_payload() try: - raise ValueError('cookies') + raise ValueError("cookies") except ValueError: txn.notice_error() @@ -203,33 +199,30 @@ def _test(): _forgone_attributes = { - 'agent': [], - 'user': [], - 'intrinsic': (inbound_payload_intrinsics + ['grandparentId']), + "agent": [], + "user": [], + "intrinsic": (inbound_payload_intrinsics + ["grandparentId"]), } @override_application_settings(_override_settings) -@validate_transaction_event_attributes( - {}, _forgone_attributes) -@validate_error_event_attributes( - {}, _forgone_attributes) -@validate_attributes('intrinsic', - {}, _forgone_attributes['intrinsic']) -@background_task(name='test_distributed_trace_attrs_omitted') +@validate_transaction_event_attributes({}, _forgone_attributes) +@validate_error_event_attributes({}, _forgone_attributes) +@validate_attributes("intrinsic", {}, _forgone_attributes["intrinsic"]) +@background_task(name="test_distributed_trace_attrs_omitted") def test_distributed_trace_attrs_omitted(): txn = current_transaction() try: - raise ValueError('cookies') + raise ValueError("cookies") except ValueError: txn.notice_error() # test our distributed_trace metrics by creating a transaction and then forcing # it to process a distributed trace payload -@pytest.mark.parametrize('web_transaction', (True, False)) -@pytest.mark.parametrize('gen_error', (True, False)) -@pytest.mark.parametrize('has_parent', (True, False)) +@pytest.mark.parametrize("web_transaction", (True, False)) +@pytest.mark.parametrize("gen_error", (True, False)) +@pytest.mark.parametrize("has_parent", (True, False)) def test_distributed_tracing_metrics(web_transaction, gen_error, has_parent): def _make_dt_tag(pi): return "%s/%s/%s/%s/all" % tuple(pi[x] for x in parent_order) @@ -237,11 +230,11 @@ def _make_dt_tag(pi): # figure out which metrics we'll see based on the test params # note: we'll always see DurationByCaller if the distributed # tracing flag is turned on - metrics = ['DurationByCaller'] + metrics = ["DurationByCaller"] if gen_error: - metrics.append('ErrorsByCaller') + metrics.append("ErrorsByCaller") if has_parent: - metrics.append('TransportDuration') + metrics.append("TransportDuration") tag = None dt_payload = copy.deepcopy(payload) @@ -251,15 +244,14 @@ def _make_dt_tag(pi): if has_parent: tag = _make_dt_tag(parent_info) else: - tag = _make_dt_tag(dict((x, 'Unknown') for x in parent_info.keys())) - del dt_payload['d']['tr'] + # tag = _make_dt_tag(dict((x, "Unknown") for x in parent_order)) + tag = _make_dt_tag(dict((x, "Unknown") for x in parent_info.keys())) + del dt_payload["d"]["tr"] # now run the test - transaction_name = "test_dt_metrics_%s" % '_'.join(metrics) + transaction_name = "test_dt_metrics_%s" % "_".join(metrics) _rollup_metrics = [ - ("%s/%s%s" % (x, tag, bt), 1) - for x in metrics - for bt in ['', 'Web' if web_transaction else 'Other'] + ("%s/%s%s" % (x, tag, bt), 1) for x in metrics for bt in ["", "Web" if web_transaction else "Other"] ] def _make_test_transaction(): @@ -268,16 +260,15 @@ def _make_test_transaction(): if not web_transaction: return BackgroundTask(application, transaction_name) - environ = {'REQUEST_URI': '/trace_ends_after_txn'} + environ = {"REQUEST_URI": "/trace_ends_after_txn"} tn = WSGIWebTransaction(application, environ) tn.set_transaction_name(transaction_name) return tn @override_application_settings(_override_settings) @validate_transaction_metrics( - transaction_name, - background_task=not(web_transaction), - rollup_metrics=_rollup_metrics) + transaction_name, background_task=not (web_transaction), rollup_metrics=_rollup_metrics + ) def _test(): with _make_test_transaction() as transaction: transaction.accept_distributed_trace_payload(dt_payload) @@ -291,54 +282,56 @@ def _test(): _test() -NEW_RELIC_ACCEPTED = \ - [('Supportability/DistributedTrace/AcceptPayload/Success', 1), - ('Supportability/TraceContext/Accept/Success', None), - ('Supportability/TraceContext/TraceParent/Accept/Success', None), - ('Supportability/TraceContext/Accept/Success', None)] -TRACE_CONTEXT_ACCEPTED = \ - [('Supportability/TraceContext/Accept/Success', 1), - ('Supportability/TraceContext/TraceParent/Accept/Success', 1), - ('Supportability/TraceContext/Accept/Success', 1), - ('Supportability/DistributedTrace/AcceptPayload/Success', None)] -NO_HEADERS_ACCEPTED = \ - [('Supportability/DistributedTrace/AcceptPayload/Success', None), - ('Supportability/TraceContext/Accept/Success', None), - ('Supportability/TraceContext/TraceParent/Accept/Success', None), - ('Supportability/TraceContext/Accept/Success', None)] -TRACEPARENT = '00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01' -TRACESTATE = 'rojo=f06a0ba902b7,congo=t61rcWkgMzE' - - -@pytest.mark.parametrize('traceparent,tracestate,newrelic,metrics', - [(False, False, False, NO_HEADERS_ACCEPTED), - (False, False, True, NEW_RELIC_ACCEPTED), - (False, True, True, NEW_RELIC_ACCEPTED), - (False, True, False, NO_HEADERS_ACCEPTED), - (True, True, True, TRACE_CONTEXT_ACCEPTED), - (True, False, False, TRACE_CONTEXT_ACCEPTED), - (True, False, True, TRACE_CONTEXT_ACCEPTED), - (True, True, False, TRACE_CONTEXT_ACCEPTED)] - ) +NEW_RELIC_ACCEPTED = [ + ("Supportability/DistributedTrace/AcceptPayload/Success", 1), + ("Supportability/TraceContext/Accept/Success", None), + ("Supportability/TraceContext/TraceParent/Accept/Success", None), + ("Supportability/TraceContext/Accept/Success", None), +] +TRACE_CONTEXT_ACCEPTED = [ + ("Supportability/TraceContext/Accept/Success", 1), + ("Supportability/TraceContext/TraceParent/Accept/Success", 1), + ("Supportability/TraceContext/Accept/Success", 1), + ("Supportability/DistributedTrace/AcceptPayload/Success", None), +] +NO_HEADERS_ACCEPTED = [ + ("Supportability/DistributedTrace/AcceptPayload/Success", None), + ("Supportability/TraceContext/Accept/Success", None), + ("Supportability/TraceContext/TraceParent/Accept/Success", None), + ("Supportability/TraceContext/Accept/Success", None), +] +TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01" +TRACESTATE = "rojo=f06a0ba902b7,congo=t61rcWkgMzE" + + +@pytest.mark.parametrize( + "traceparent,tracestate,newrelic,metrics", + [ + (False, False, False, NO_HEADERS_ACCEPTED), + (False, False, True, NEW_RELIC_ACCEPTED), + (False, True, True, NEW_RELIC_ACCEPTED), + (False, True, False, NO_HEADERS_ACCEPTED), + (True, True, True, TRACE_CONTEXT_ACCEPTED), + (True, False, False, TRACE_CONTEXT_ACCEPTED), + (True, False, True, TRACE_CONTEXT_ACCEPTED), + (True, True, False, TRACE_CONTEXT_ACCEPTED), + ], +) @override_application_settings(_override_settings) -def test_distributed_tracing_backwards_compatibility(traceparent, - tracestate, - newrelic, - metrics): +def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, newrelic, metrics): headers = [] if traceparent: - headers.append(('traceparent', TRACEPARENT)) + headers.append(("traceparent", TRACEPARENT)) if tracestate: - headers.append(('tracestate', TRACESTATE)) + headers.append(("tracestate", TRACESTATE)) if newrelic: - headers.append(('newrelic', json.dumps(payload))) + headers.append(("newrelic", json.dumps(payload))) @validate_transaction_metrics( - "test_distributed_tracing_backwards_compatibility", - background_task=True, - rollup_metrics=metrics) - @background_task(name='test_distributed_tracing_backwards_compatibility') + "test_distributed_tracing_backwards_compatibility", background_task=True, rollup_metrics=metrics + ) + @background_task(name="test_distributed_tracing_backwards_compatibility") def _test(): transaction = current_transaction() transaction.accept_distributed_trace_headers(headers) @@ -346,7 +339,7 @@ def _test(): _test() -@background_task(name='test_current_trace_id_api_inside_transaction') +@background_task(name="test_current_trace_id_api_inside_transaction") def test_current_trace_id_api_inside_transaction(): trace_id = current_trace_id() assert len(trace_id) == 32 @@ -358,7 +351,7 @@ def test_current_trace_id_api_outside_transaction(): assert trace_id is None -@background_task(name='test_current_span_id_api_inside_transaction') +@background_task(name="test_current_span_id_api_inside_transaction") def test_current_span_id_inside_transaction(): span_id = current_span_id() assert span_id == current_trace().guid diff --git a/tests/agent_features/test_error_group_callback.py b/tests/agent_features/test_error_group_callback.py index 742391162c..2fe2fc68c7 100644 --- a/tests/agent_features/test_error_group_callback.py +++ b/tests/agent_features/test_error_group_callback.py @@ -12,35 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys import threading import traceback -import sys import pytest - from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, +) +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( validate_error_event_attributes_outside_transaction, - validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( + validate_error_trace_attributes_outside_transaction, +) from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task +from newrelic.api.settings import set_error_group_callback from newrelic.api.time_trace import notice_error from newrelic.api.transaction import current_transaction -from newrelic.api.settings import set_error_group_callback from newrelic.api.web_transaction import web_transaction from newrelic.common.object_names import callable_name - _callback_called = threading.Event() _truncated_value = "A" * 300 + def error_group_callback(exc, data): _callback_called.set() @@ -64,12 +69,9 @@ def test_clear_error_group_callback(): assert settings.error_collector.error_group_callback is None, "Failed to clear callback." -@pytest.mark.parametrize("callback,accepted", [ - (error_group_callback, True), - (lambda x, y: None, True), - (None, False), - ("string", False) -]) +@pytest.mark.parametrize( + "callback,accepted", [(error_group_callback, True), (lambda x, y: None, True), (None, False), ("string", False)] +) def test_set_error_group_callback(callback, accepted): try: set_error_group_callback(callback) @@ -82,15 +84,19 @@ def test_set_error_group_callback(callback, accepted): set_error_group_callback(None) -@pytest.mark.parametrize("exc_class,group_name,high_security", [ - (ValueError, "value", False), - (ValueError, "value", True), - (TypeError, None, False), - (RuntimeError, None, False), - (IndexError, None, False), - (LookupError, None, False), - (ZeroDivisionError, _truncated_value[:255], False), -], ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value")) +@pytest.mark.parametrize( + "exc_class,group_name,high_security", + [ + (ValueError, "value", False), + (ValueError, "value", True), + (TypeError, None, False), + (RuntimeError, None, False), + (IndexError, None, False), + (LookupError, None, False), + (ZeroDivisionError, _truncated_value[:255], False), + ], + ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value"), +) @reset_core_stats_engine() def test_error_group_name_callback(exc_class, group_name, high_security): _callback_called.clear() @@ -102,9 +108,7 @@ def test_error_group_name_callback(exc_class, group_name, high_security): exact = None forgone = {"user": [], "intrinsic": [], "agent": ["error.group.name"]} - @validate_error_trace_attributes( - callable_name(exc_class), forgone_params=forgone, exact_attrs=exact - ) + @validate_error_trace_attributes(callable_name(exc_class), forgone_params=forgone, exact_attrs=exact) @validate_error_event_attributes(forgone_params=forgone, exact_attrs=exact) @override_application_settings({"high_security": high_security}) @background_task() @@ -124,15 +128,19 @@ def _test(): set_error_group_callback(None) -@pytest.mark.parametrize("exc_class,group_name,high_security", [ - (ValueError, "value", False), - (ValueError, "value", True), - (TypeError, None, False), - (RuntimeError, None, False), - (IndexError, None, False), - (LookupError, None, False), - (ZeroDivisionError, _truncated_value[:255], False), -], ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value")) +@pytest.mark.parametrize( + "exc_class,group_name,high_security", + [ + (ValueError, "value", False), + (ValueError, "value", True), + (TypeError, None, False), + (RuntimeError, None, False), + (IndexError, None, False), + (LookupError, None, False), + (ZeroDivisionError, _truncated_value[:255], False), + ], + ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value"), +) @reset_core_stats_engine() def test_error_group_name_callback_outside_transaction(exc_class, group_name, high_security): _callback_called.clear() @@ -155,7 +163,7 @@ def _test(): except Exception: app = application() notice_error(application=app) - + assert _callback_called.is_set() try: @@ -165,11 +173,22 @@ def _test(): set_error_group_callback(None) -@pytest.mark.parametrize("transaction_decorator", [ - background_task(name="TestBackgroundTask"), - web_transaction(name="TestWebTransaction", host="localhost", port=1234, request_method="GET", request_path="/", headers=[],), - None, -], ids=("background_task", "web_transation", "outside_transaction")) +@pytest.mark.parametrize( + "transaction_decorator", + [ + background_task(name="TestBackgroundTask"), + web_transaction( + name="TestWebTransaction", + host="localhost", + port=1234, + request_method="GET", + request_path="/", + headers=[], + ), + None, + ], + ids=("background_task", "web_transation", "outside_transaction"), +) @reset_core_stats_engine() def test_error_group_name_callback_attributes(transaction_decorator): callback_errors = [] @@ -178,6 +197,7 @@ def test_error_group_name_callback_attributes(transaction_decorator): def callback(error, data): def _callback(): import types + _data.append(data) txn = current_transaction() @@ -191,23 +211,23 @@ def _callback(): # All attributes should always be included, but set to None when not relevant. if txn is None: # Outside transaction assert data["transactionName"] is None - assert data["custom_params"] == {'notice_error_attribute': 1} + assert data["custom_params"] == {"notice_error_attribute": 1} assert data["response.status"] is None assert data["request.method"] is None assert data["request.uri"] is None elif txn.background_task: # Background task assert data["transactionName"] == "TestBackgroundTask" - assert data["custom_params"] == {'notice_error_attribute': 1, 'txn_attribute': 2} + assert data["custom_params"] == {"notice_error_attribute": 1, "txn_attribute": 2} assert data["response.status"] is None assert data["request.method"] is None assert data["request.uri"] is None else: # Web transaction assert data["transactionName"] == "TestWebTransaction" - assert data["custom_params"] == {'notice_error_attribute': 1, 'txn_attribute': 2} + assert data["custom_params"] == {"notice_error_attribute": 1, "txn_attribute": 2} assert data["response.status"] == 200 assert data["request.method"] == "GET" assert data["request.uri"] == "/" - + try: _callback() except Exception: @@ -225,8 +245,11 @@ def _test(): except Exception: app = application() if transaction_decorator is None else None # Only set outside transaction notice_error(application=app, attributes={"notice_error_attribute": 1}) - - assert not callback_errors, "Callback inputs failed to validate.\nerror: %s\ndata: %s" % (traceback.format_exception(*callback_errors[0]), str(_data[0])) + + assert not callback_errors, "Callback inputs failed to validate.\nerror: %s\ndata: %s" % ( + traceback.format_exception(*callback_errors[0]), + str(_data[0]), + ) if transaction_decorator is not None: _test = transaction_decorator(_test) # Manually decorate test function diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index dad7edc295..20d9978373 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -25,7 +25,6 @@ validate_custom_event_count, validate_custom_event_in_application_stats_engine, validate_request_params_omitted, - validate_tt_segment_params, ) from testing_support.validators.validate_custom_parameters import ( validate_custom_parameters, @@ -36,6 +35,9 @@ from testing_support.validators.validate_transaction_errors import ( validate_transaction_errors, ) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task diff --git a/tests/agent_features/test_ignore_expected_errors.py b/tests/agent_features/test_ignore_expected_errors.py index 93595aa35b..ee26245c5d 100644 --- a/tests/agent_features/test_ignore_expected_errors.py +++ b/tests/agent_features/test_ignore_expected_errors.py @@ -16,8 +16,12 @@ from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, - validate_error_event_attributes_outside_transaction, validate_error_event_sample_data, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( + validate_error_event_attributes_outside_transaction, +) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_time_metrics_outside_transaction import ( diff --git a/tests/agent_features/test_span_events.py b/tests/agent_features/test_span_events.py index 655efee8ce..b9c04a8c86 100644 --- a/tests/agent_features/test_span_events.py +++ b/tests/agent_features/test_span_events.py @@ -19,7 +19,6 @@ dt_enabled, function_not_called, override_application_settings, - validate_tt_segment_params, ) from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_event_attributes import ( @@ -28,6 +27,9 @@ from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.api.database_trace import DatabaseTrace @@ -725,7 +727,7 @@ def test_span_event_notice_error_overrides_observed(trace_type, args): raise ERROR except Exception: notice_error() - raise ValueError # pylint: disable + raise ValueError # pylint: disable (Py2/Py3 compatibility) except ValueError: pass diff --git a/tests/agent_features/test_transaction_trace_segments.py b/tests/agent_features/test_transaction_trace_segments.py index b205afc3ce..8318c0fca7 100644 --- a/tests/agent_features/test_transaction_trace_segments.py +++ b/tests/agent_features/test_transaction_trace_segments.py @@ -13,8 +13,8 @@ # limitations under the License. import pytest -from testing_support.fixtures import ( - override_application_settings, +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_tt_segment_params import ( validate_tt_segment_params, ) diff --git a/tests/agent_features/test_wsgi_attributes.py b/tests/agent_features/test_wsgi_attributes.py index e90410b6db..db9fc807a4 100644 --- a/tests/agent_features/test_wsgi_attributes.py +++ b/tests/agent_features/test_wsgi_attributes.py @@ -13,12 +13,11 @@ # limitations under the License. import webtest -from testing_support.fixtures import ( - dt_enabled, - override_application_settings, +from testing_support.fixtures import dt_enabled, override_application_settings +from testing_support.sample_applications import fully_featured_app +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, ) -from testing_support.sample_applications import fully_featured_app from testing_support.validators.validate_transaction_error_trace_attributes import ( validate_transaction_error_trace_attributes, ) diff --git a/tests/cross_agent/test_cat_map.py b/tests/cross_agent/test_cat_map.py index 67c5ab8151..6e7ac63d6d 100644 --- a/tests/cross_agent/test_cat_map.py +++ b/tests/cross_agent/test_cat_map.py @@ -18,42 +18,58 @@ can be found in test/framework_tornado_r3/test_cat_map.py """ -import webtest -import pytest import json import os +import pytest +import webtest + try: from urllib2 import urlopen # Py2.X except ImportError: - from urllib.request import urlopen # Py3.X - -from newrelic.packages import six + from urllib.request import urlopen # Py3.X + +from testing_support.fixtures import ( + make_cross_agent_headers, + override_application_name, + override_application_settings, + validate_analytics_catmap_data, +) +from testing_support.mock_external_http_server import ( + MockExternalHTTPHResponseHeadersServer, +) +from testing_support.validators.validate_tt_parameters import validate_tt_parameters from newrelic.api.external_trace import ExternalTrace -from newrelic.api.transaction import (get_browser_timing_header, - set_transaction_name, get_browser_timing_footer, set_background_task, - current_transaction) +from newrelic.api.transaction import ( + current_transaction, + get_browser_timing_footer, + get_browser_timing_header, + set_background_task, + set_transaction_name, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.encoding_utils import obfuscate, json_encode - -from testing_support.fixtures import (override_application_settings, - override_application_name, validate_tt_parameters, - make_cross_agent_headers, validate_analytics_catmap_data) -from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) +from newrelic.common.encoding_utils import json_encode, obfuscate +from newrelic.packages import six -ENCODING_KEY = '1234567890123456789012345678901234567890' +ENCODING_KEY = "1234567890123456789012345678901234567890" CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures')) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures")) OUTBOUD_REQUESTS = {} -_parameters_list = ["name", "appName", "transactionName", "transactionGuid", - "inboundPayload", "outboundRequests", "expectedIntrinsicFields", - "nonExpectedIntrinsicFields"] +_parameters_list = [ + "name", + "appName", + "transactionName", + "transactionGuid", + "inboundPayload", + "outboundRequests", + "expectedIntrinsicFields", + "nonExpectedIntrinsicFields", +] -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server @@ -61,8 +77,8 @@ def server(): def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'cat_map.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "cat_map.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: @@ -77,57 +93,52 @@ def load_tests(): @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' + status = "200 OK" - txn_name = environ.get('txn') + txn_name = environ.get("txn") if six.PY2: - txn_name = txn_name.decode('UTF-8') - txn_name = txn_name.split('/', 3) + txn_name = txn_name.decode("UTF-8") + txn_name = txn_name.split("/", 3) - guid = environ.get('guid') - old_cat = environ.get('old_cat') == 'True' + guid = environ.get("guid") + old_cat = environ.get("old_cat") == "True" txn = current_transaction() txn.guid = guid for req in OUTBOUD_REQUESTS: # Change the transaction name before making an outbound call. - outgoing_name = req['outboundTxnName'].split('/', 3) - if outgoing_name[0] != 'WebTransaction': + outgoing_name = req["outboundTxnName"].split("/", 3) + if outgoing_name[0] != "WebTransaction": set_background_task(True) set_transaction_name(outgoing_name[2], group=outgoing_name[1]) - expected_outbound_header = obfuscate( - json_encode(req['expectedOutboundPayload']), ENCODING_KEY) - generated_outbound_header = dict( - ExternalTrace.generate_request_headers(txn)) + expected_outbound_header = obfuscate(json_encode(req["expectedOutboundPayload"]), ENCODING_KEY) + generated_outbound_header = dict(ExternalTrace.generate_request_headers(txn)) # A 500 error is returned because 'assert' statements in the wsgi app # are ignored. if old_cat: - if (expected_outbound_header != - generated_outbound_header['X-NewRelic-Transaction']): - status = '500 Outbound Headers Check Failed.' + if expected_outbound_header != generated_outbound_header["X-NewRelic-Transaction"]: + status = "500 Outbound Headers Check Failed." else: - if 'X-NewRelic-Transaction' in generated_outbound_header: - status = '500 Outbound Headers Check Failed.' - r = urlopen(environ['server_url']) + if "X-NewRelic-Transaction" in generated_outbound_header: + status = "500 Outbound Headers Check Failed." + r = urlopen(environ["server_url"]) # nosec B310 r.read(10) # Set the final transaction name. - if txn_name[0] != 'WebTransaction': + if txn_name[0] != "WebTransaction": set_background_task(True) set_transaction_name(txn_name[2], group=txn_name[1]) - text = '%s

RESPONSE

%s' + text = "%s

RESPONSE

%s" - output = (text % (get_browser_timing_header(), - get_browser_timing_footer())).encode('UTF-8') + output = (text % (get_browser_timing_header(), get_browser_timing_footer())).encode("UTF-8") - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] start_response(status, response_headers) return [output] @@ -137,26 +148,35 @@ def target_wsgi_application(environ, start_response): @pytest.mark.parametrize(_parameters, load_tests()) -@pytest.mark.parametrize('old_cat', (True, False)) -def test_cat_map(name, appName, transactionName, transactionGuid, - inboundPayload, outboundRequests, expectedIntrinsicFields, - nonExpectedIntrinsicFields, old_cat, server): +@pytest.mark.parametrize("old_cat", (True, False)) +def test_cat_map( + name, + appName, + transactionName, + transactionGuid, + inboundPayload, + outboundRequests, + expectedIntrinsicFields, + nonExpectedIntrinsicFields, + old_cat, + server, +): global OUTBOUD_REQUESTS OUTBOUD_REQUESTS = outboundRequests or {} _custom_settings = { - 'cross_process_id': '1#1', - 'encoding_key': ENCODING_KEY, - 'trusted_account_ids': [1], - 'cross_application_tracer.enabled': True, - 'distributed_tracing.enabled': not old_cat, - 'transaction_tracer.transaction_threshold': 0.0, + "cross_process_id": "1#1", + "encoding_key": ENCODING_KEY, + "trusted_account_ids": [1], + "cross_application_tracer.enabled": True, + "distributed_tracing.enabled": not old_cat, + "transaction_tracer.transaction_threshold": 0.0, } if expectedIntrinsicFields and old_cat: _external_node_params = { - 'path_hash': expectedIntrinsicFields['nr.pathHash'], - 'trip_id': expectedIntrinsicFields['nr.tripId'], + "path_hash": expectedIntrinsicFields["nr.pathHash"], + "trip_id": expectedIntrinsicFields["nr.tripId"], } else: _external_node_params = [] @@ -167,16 +187,16 @@ def test_cat_map(name, appName, transactionName, transactionGuid, expectedIntrinsicFields = {} @validate_tt_parameters(required_params=_external_node_params) - @validate_analytics_catmap_data(transactionName, - expected_attributes=expectedIntrinsicFields, - non_expected_attributes=nonExpectedIntrinsicFields) + @validate_analytics_catmap_data( + transactionName, expected_attributes=expectedIntrinsicFields, non_expected_attributes=nonExpectedIntrinsicFields + ) @override_application_settings(_custom_settings) @override_application_name(appName) def run_cat_test(): if six.PY2: - txn_name = transactionName.encode('UTF-8') - guid = transactionGuid.encode('UTF-8') + txn_name = transactionName.encode("UTF-8") + guid = transactionGuid.encode("UTF-8") else: txn_name = transactionName guid = transactionGuid @@ -185,20 +205,26 @@ def run_cat_test(): # are properly ignoring these headers when the agent is using better # cat. - headers = make_cross_agent_headers(inboundPayload, ENCODING_KEY, '1#1') - response = target_application.get('/', headers=headers, - extra_environ={'txn': txn_name, 'guid': guid, - 'old_cat': str(old_cat), - 'server_url': 'http://localhost:%d' % server.port}) + headers = make_cross_agent_headers(inboundPayload, ENCODING_KEY, "1#1") + response = target_application.get( + "/", + headers=headers, + extra_environ={ + "txn": txn_name, + "guid": guid, + "old_cat": str(old_cat), + "server_url": "http://localhost:%d" % server.port, + }, + ) # Validation of analytic data happens in the decorator. - assert response.status == '200 OK' + assert response.status == "200 OK" content = response.html.html.body.p.string # Validate actual body content as sanity check. - assert content == 'RESPONSE' + assert content == "RESPONSE" run_cat_test() diff --git a/tests/cross_agent/test_distributed_tracing.py b/tests/cross_agent/test_distributed_tracing.py index 0ff46eea24..060fe8a864 100644 --- a/tests/cross_agent/test_distributed_tracing.py +++ b/tests/cross_agent/test_distributed_tracing.py @@ -14,54 +14,70 @@ import json import os + import pytest import webtest +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_error_event_attributes import ( + validate_error_event_attributes, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.transaction import current_transaction from newrelic.api.wsgi_application import wsgi_application from newrelic.common.encoding_utils import DistributedTracePayload from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.fixtures import (override_application_settings, - validate_error_event_attributes, validate_attributes) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes - CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ['account_id', 'comment', 'expected_metrics', - 'force_sampled_true', 'inbound_payloads', 'intrinsics', - 'major_version', 'minor_version', 'outbound_payloads', - 'raises_exception', 'span_events_enabled', 'test_name', - 'transport_type', 'trusted_account_key', 'web_transaction'] -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = [ + "account_id", + "comment", + "expected_metrics", + "force_sampled_true", + "inbound_payloads", + "intrinsics", + "major_version", + "minor_version", + "outbound_payloads", + "raises_exception", + "span_events_enabled", + "test_name", + "transport_type", + "trusted_account_key", + "web_transaction", +] +_parameters = ",".join(_parameters_list) def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'distributed_tracing.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "distributed_tracing.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @@ -70,58 +86,54 @@ def assert_payload(payload, payload_assertions, major_version, minor_version): # flatten payload so it matches the test: # payload['d']['ac'] -> payload['d.ac'] - d = payload.pop('d') + d = payload.pop("d") for key, value in d.items(): - payload['d.%s' % key] = value + payload["d.%s" % key] = value - for expected in payload_assertions.get('expected', []): + for expected in payload_assertions.get("expected", []): assert expected in payload - for unexpected in payload_assertions.get('unexpected', []): + for unexpected in payload_assertions.get("unexpected", []): assert unexpected not in payload - for key, value in payload_assertions.get('exact', {}).items(): + for key, value in payload_assertions.get("exact", {}).items(): assert key in payload if isinstance(value, list): value = tuple(value) assert payload[key] == value - assert payload['v'][0] == major_version - assert payload['v'][1] == minor_version + assert payload["v"][0] == major_version + assert payload["v"][1] == minor_version @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' - output = b'hello world' - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + status = "200 OK" + output = b"hello world" + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] txn = current_transaction() - txn.set_transaction_name(test_settings['test_name']) + txn.set_transaction_name(test_settings["test_name"]) - if not test_settings['web_transaction']: + if not test_settings["web_transaction"]: txn.background_task = True - if test_settings['raises_exception']: + if test_settings["raises_exception"]: try: 1 / 0 except ZeroDivisionError: txn.notice_error() - extra_inbound_payloads = test_settings['extra_inbound_payloads'] + extra_inbound_payloads = test_settings["extra_inbound_payloads"] for payload, expected_result in extra_inbound_payloads: - result = txn.accept_distributed_trace_payload(payload, - test_settings['transport_type']) + result = txn.accept_distributed_trace_payload(payload, test_settings["transport_type"]) assert result is expected_result - outbound_payloads = test_settings['outbound_payloads'] + outbound_payloads = test_settings["outbound_payloads"] if outbound_payloads: for payload_assertions in outbound_payloads: payload = txn._create_distributed_trace_payload() - assert_payload(payload, payload_assertions, - test_settings['major_version'], - test_settings['minor_version']) + assert_payload(payload, payload_assertions, test_settings["major_version"], test_settings["minor_version"]) start_response(status, response_headers) return [output] @@ -131,14 +143,26 @@ def target_wsgi_application(environ, start_response): @pytest.mark.parametrize(_parameters, load_tests()) -def test_distributed_tracing(account_id, comment, expected_metrics, - force_sampled_true, inbound_payloads, intrinsics, major_version, - minor_version, outbound_payloads, raises_exception, - span_events_enabled, test_name, transport_type, trusted_account_key, - web_transaction): +def test_distributed_tracing( + account_id, + comment, + expected_metrics, + force_sampled_true, + inbound_payloads, + intrinsics, + major_version, + minor_version, + outbound_payloads, + raises_exception, + span_events_enabled, + test_name, + transport_type, + trusted_account_key, + web_transaction, +): extra_inbound_payloads = [] - if transport_type != 'HTTP': + if transport_type != "HTTP": # Since wsgi_application calls accept_distributed_trace_payload # automatically with transport_type='HTTP', we must defer this call # until we can specify the transport type. @@ -153,78 +177,68 @@ def test_distributed_tracing(account_id, comment, expected_metrics, global test_settings test_settings = { - 'test_name': test_name, - 'web_transaction': web_transaction, - 'raises_exception': raises_exception, - 'extra_inbound_payloads': extra_inbound_payloads, - 'outbound_payloads': outbound_payloads, - 'transport_type': transport_type, - 'major_version': major_version, - 'minor_version': minor_version, + "test_name": test_name, + "web_transaction": web_transaction, + "raises_exception": raises_exception, + "extra_inbound_payloads": extra_inbound_payloads, + "outbound_payloads": outbound_payloads, + "transport_type": transport_type, + "major_version": major_version, + "minor_version": minor_version, } override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } - common_required = intrinsics['common']['expected'] - common_forgone = intrinsics['common']['unexpected'] - common_exact = intrinsics['common'].get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common_required = intrinsics["common"]["expected"] + common_forgone = intrinsics["common"]["unexpected"] + common_exact = intrinsics["common"].get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) headers = {} if inbound_payloads: payload = DistributedTracePayload(inbound_payloads[0]) - headers['newrelic'] = payload.http_safe() - - @validate_transaction_metrics(test_name, - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + headers["newrelic"] = payload.http_safe() + + @validate_transaction_metrics(test_name, rollup_metrics=expected_metrics, background_task=not web_transaction) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_compute_sampled(force_sampled_true) @override_application_settings(override_settings) def _test(): - response = test_application.get('/', headers=headers) - assert 'X-NewRelic-App-Data' not in response.headers + response = test_application.get("/", headers=headers) + assert "X-NewRelic-App-Data" not in response.headers - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) if raises_exception: - error_event_required = {'agent': [], 'user': [], - 'intrinsic': common_required} - error_event_forgone = {'agent': [], 'user': [], - 'intrinsic': common_forgone} - error_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': common_exact} - _test = validate_error_event_attributes(error_event_required, - error_event_forgone, error_event_exact)(_test) + error_event_required = {"agent": [], "user": [], "intrinsic": common_required} + error_event_forgone = {"agent": [], "user": [], "intrinsic": common_forgone} + error_event_exact = {"agent": {}, "user": {}, "intrinsic": common_exact} + _test = validate_error_event_attributes(error_event_required, error_event_forgone, error_event_exact)(_test) _test() diff --git a/tests/external_boto3/test_boto3_iam.py b/tests/external_boto3/test_boto3_iam.py index ac49214f44..a2237dc936 100644 --- a/tests/external_boto3/test_boto3_iam.py +++ b/tests/external_boto3/test_boto3_iam.py @@ -17,69 +17,73 @@ import boto3 import moto +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) # patch earlier versions of moto to support py37 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): import re + moto.packages.responses.responses.re._pattern_type = re.Pattern -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) -TEST_USER = 'python-agent-test-%s' % uuid.uuid4() +TEST_USER = "python-agent-test-%s" % uuid.uuid4() _iam_scoped_metrics = [ - ('External/iam.amazonaws.com/botocore/POST', 3), + ("External/iam.amazonaws.com/botocore/POST", 3), ] _iam_rollup_metrics = [ - ('External/all', 3), - ('External/allOther', 3), - ('External/iam.amazonaws.com/all', 3), - ('External/iam.amazonaws.com/botocore/POST', 3), + ("External/all", 3), + ("External/allOther", 3), + ("External/iam.amazonaws.com/all", 3), + ("External/iam.amazonaws.com/botocore/POST", 3), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events( - exact_agents={'http.url': 'https://iam.amazonaws.com/'}, count=3) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'CreateUser'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'GetUser'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteUser'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(exact_agents={"http.url": "https://iam.amazonaws.com/"}, count=3) +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "CreateUser"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetUser"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteUser"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) @validate_transaction_metrics( - 'test_boto3_iam:test_iam', - scoped_metrics=_iam_scoped_metrics, - rollup_metrics=_iam_rollup_metrics, - background_task=True) + "test_boto3_iam:test_iam", + scoped_metrics=_iam_scoped_metrics, + rollup_metrics=_iam_rollup_metrics, + background_task=True, +) @background_task() @moto.mock_iam def test_iam(): iam = boto3.client( - 'iam', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + "iam", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, ) # Create user resp = iam.create_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 # Get the user resp = iam.get_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['User']['UserName'] == TEST_USER + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["User"]["UserName"] == TEST_USER # Delete the user resp = iam.delete_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 diff --git a/tests/external_boto3/test_boto3_sns.py b/tests/external_boto3/test_boto3_sns.py index 3718d52924..bafe68611d 100644 --- a/tests/external_boto3/test_boto3_sns.py +++ b/tests/external_boto3/test_boto3_sns.py @@ -13,80 +13,91 @@ # limitations under the License. import sys + import boto3 import moto import pytest +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) # patch earlier versions of moto to support py37 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): import re + moto.packages.responses.responses.re._pattern_type = re.Pattern -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION_NAME = 'us-east-1' -SNS_URL = 'sns-us-east-1.amazonaws.com' -TOPIC = 'arn:aws:sns:us-east-1:123456789012:some-topic' -sns_metrics = [ - ('MessageBroker/SNS/Topic' - '/Produce/Named/%s' % TOPIC, 1)] -sns_metrics_phone = [ - ('MessageBroker/SNS/Topic' - '/Produce/Named/PhoneNumber', 1)] +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION_NAME = "us-east-1" +SNS_URL = "sns-us-east-1.amazonaws.com" +TOPIC = "arn:aws:sns:us-east-1:123456789012:some-topic" +sns_metrics = [("MessageBroker/SNS/Topic" "/Produce/Named/%s" % TOPIC, 1)] +sns_metrics_phone = [("MessageBroker/SNS/Topic" "/Produce/Named/PhoneNumber", 1)] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=2) -@validate_span_events(exact_agents={'aws.operation': 'CreateTopic'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Publish'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) -@pytest.mark.parametrize('topic_argument', ('TopicArn', 'TargetArn')) -@validate_transaction_metrics('test_boto3_sns:test_publish_to_sns_topic', - scoped_metrics=sns_metrics, rollup_metrics=sns_metrics, - background_task=True) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(expected_agents=("aws.requestId",), count=2) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) +@pytest.mark.parametrize("topic_argument", ("TopicArn", "TargetArn")) +@validate_transaction_metrics( + "test_boto3_sns:test_publish_to_sns_topic", + scoped_metrics=sns_metrics, + rollup_metrics=sns_metrics, + background_task=True, +) @background_task() @moto.mock_sns def test_publish_to_sns_topic(topic_argument): - conn = boto3.client('sns', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION_NAME) + conn = boto3.client( + "sns", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION_NAME, + ) - topic_arn = conn.create_topic(Name='some-topic')['TopicArn'] + topic_arn = conn.create_topic(Name="some-topic")["TopicArn"] kwargs = {topic_argument: topic_arn} - published_message = conn.publish(Message='my msg', **kwargs) - assert 'MessageId' in published_message + published_message = conn.publish(Message="my msg", **kwargs) + assert "MessageId" in published_message -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'CreateTopic'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Subscribe'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Publish'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) -@validate_transaction_metrics('test_boto3_sns:test_publish_to_sns_phone', - scoped_metrics=sns_metrics_phone, rollup_metrics=sns_metrics_phone, - background_task=True) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Subscribe"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) +@validate_transaction_metrics( + "test_boto3_sns:test_publish_to_sns_phone", + scoped_metrics=sns_metrics_phone, + rollup_metrics=sns_metrics_phone, + background_task=True, +) @background_task() @moto.mock_sns def test_publish_to_sns_phone(): - conn = boto3.client('sns', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION_NAME) + conn = boto3.client( + "sns", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION_NAME, + ) - topic_arn = conn.create_topic(Name='some-topic')['TopicArn'] - conn.subscribe(TopicArn=topic_arn, Protocol='sms', Endpoint='5555555555') + topic_arn = conn.create_topic(Name="some-topic")["TopicArn"] + conn.subscribe(TopicArn=topic_arn, Protocol="sms", Endpoint="5555555555") - published_message = conn.publish( - PhoneNumber='5555555555', Message='my msg') - assert 'MessageId' in published_message + published_message = conn.publish(PhoneNumber="5555555555", Message="my msg") + assert "MessageId" in published_message diff --git a/tests/external_botocore/test_botocore_dynamodb.py b/tests/external_botocore/test_botocore_dynamodb.py index 44862d827d..30114d53b1 100644 --- a/tests/external_botocore/test_botocore_dynamodb.py +++ b/tests/external_botocore/test_botocore_dynamodb.py @@ -17,91 +17,96 @@ import botocore.session import moto +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) # patch earlier versions of moto to support py37 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): import re + moto.packages.responses.responses.re._pattern_type = re.Pattern -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION = 'us-east-1' +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION = "us-east-1" -TEST_TABLE = 'python-agent-test-%s' % uuid.uuid4() +TEST_TABLE = "python-agent-test-%s" % uuid.uuid4() _dynamodb_scoped_metrics = [ - ('Datastore/statement/DynamoDB/%s/create_table' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/put_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/get_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/update_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/query' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/scan' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/delete_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/delete_table' % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/create_table" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/put_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/get_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/update_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/query" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/scan" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/delete_item" % TEST_TABLE, 1), + ("Datastore/statement/DynamoDB/%s/delete_table" % TEST_TABLE, 1), ] _dynamodb_rollup_metrics = [ - ('Datastore/all', 8), - ('Datastore/allOther', 8), - ('Datastore/DynamoDB/all', 8), - ('Datastore/DynamoDB/allOther', 8), + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/DynamoDB/all", 8), + ("Datastore/DynamoDB/allOther", 8), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=8) -@validate_span_events(exact_agents={'aws.operation': 'PutItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'GetItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'CreateTable'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteTable'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Query'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Scan'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(expected_agents=("aws.requestId",), count=8) +@validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) @validate_transaction_metrics( - 'test_botocore_dynamodb:test_dynamodb', - scoped_metrics=_dynamodb_scoped_metrics, - rollup_metrics=_dynamodb_rollup_metrics, - background_task=True) + "test_botocore_dynamodb:test_dynamodb", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, +) @background_task() @moto.mock_dynamodb2 def test_dynamodb(): session = botocore.session.get_session() client = session.create_client( - 'dynamodb', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "dynamodb", + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, ) # Create table resp = client.create_table( - TableName=TEST_TABLE, - AttributeDefinitions=[ - {'AttributeName': 'Id', 'AttributeType': 'N'}, - {'AttributeName': 'Foo', 'AttributeType': 'S'}, - ], - KeySchema=[ - {'AttributeName': 'Id', 'KeyType': 'HASH'}, - {'AttributeName': 'Foo', 'KeyType': 'RANGE'}, - ], - ProvisionedThroughput={ - 'ReadCapacityUnits': 5, - 'WriteCapacityUnits': 5, - }, + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "Id", "KeyType": "HASH"}, + {"AttributeName": "Foo", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={ + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, ) - assert resp['TableDescription']['TableName'] == TEST_TABLE + assert resp["TableDescription"]["TableName"] == TEST_TABLE # moto response is ACTIVE, AWS response is CREATING # assert resp['TableDescription']['TableStatus'] == 'ACTIVE' @@ -111,73 +116,70 @@ def test_dynamodb(): # Put item resp = client.put_item( - TableName=TEST_TABLE, - Item={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - } + TableName=TEST_TABLE, + Item={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + "SomeValue": {"S": "some_random_attribute"}, + }, ) # No checking response, due to inconsistent return values. # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] # Get item resp = client.get_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - } + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + "SomeValue": {"S": "some_random_attribute"}, + }, ) - assert resp['Item']['SomeValue']['S'] == 'some_random_attribute' + assert resp["Item"]["SomeValue"]["S"] == "some_random_attribute" # Update item resp = client.update_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - }, - AttributeUpdates={ - 'Foo2': { - 'Value': {'S': 'hello_world2'}, - 'Action': 'PUT' - }, - }, - ReturnValues='ALL_NEW', + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + "SomeValue": {"S": "some_random_attribute"}, + }, + AttributeUpdates={ + "Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}, + }, + ReturnValues="ALL_NEW", ) - assert resp['Attributes']['Foo2'] + assert resp["Attributes"]["Foo2"] # Query for item resp = client.query( - TableName=TEST_TABLE, - Select='ALL_ATTRIBUTES', - KeyConditionExpression='#Id = :v_id', - ExpressionAttributeNames={'#Id': 'Id'}, - ExpressionAttributeValues={':v_id': {'N': '101'}}, + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, ) - assert len(resp['Items']) == 1 - assert resp['Items'][0]['SomeValue']['S'] == 'some_random_attribute' + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["SomeValue"]["S"] == "some_random_attribute" # Scan resp = client.scan(TableName=TEST_TABLE) - assert len(resp['Items']) == 1 + assert len(resp["Items"]) == 1 # Delete item resp = client.delete_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - }, + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, ) # No checking response, due to inconsistent return values. # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] # Delete table resp = client.delete_table(TableName=TEST_TABLE) - assert resp['TableDescription']['TableName'] == TEST_TABLE + assert resp["TableDescription"]["TableName"] == TEST_TABLE # moto response is ACTIVE, AWS response is DELETING # assert resp['TableDescription']['TableStatus'] == 'DELETING' diff --git a/tests/external_botocore/test_botocore_ec2.py b/tests/external_botocore/test_botocore_ec2.py index 0cfd09b6fd..28a8ff63ae 100644 --- a/tests/external_botocore/test_botocore_ec2.py +++ b/tests/external_botocore/test_botocore_ec2.py @@ -17,81 +17,81 @@ import botocore.session import moto +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) # patch earlier versions of moto to support py37 if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): import re + moto.packages.responses.responses.re._pattern_type = re.Pattern -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION = 'us-east-1' -UBUNTU_14_04_PARAVIRTUAL_AMI = 'ami-c65be9ae' +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION = "us-east-1" +UBUNTU_14_04_PARAVIRTUAL_AMI = "ami-c65be9ae" -TEST_INSTANCE = 'python-agent-test-%s' % uuid.uuid4() +TEST_INSTANCE = "python-agent-test-%s" % uuid.uuid4() _ec2_scoped_metrics = [ - ('External/ec2.us-east-1.amazonaws.com/botocore/POST', 3), + ("External/ec2.us-east-1.amazonaws.com/botocore/POST", 3), ] _ec2_rollup_metrics = [ - ('External/all', 3), - ('External/allOther', 3), - ('External/ec2.us-east-1.amazonaws.com/all', 3), - ('External/ec2.us-east-1.amazonaws.com/botocore/POST', 3), + ("External/all", 3), + ("External/allOther", 3), + ("External/ec2.us-east-1.amazonaws.com/all", 3), + ("External/ec2.us-east-1.amazonaws.com/botocore/POST", 3), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'RunInstances'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'DescribeInstances'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'TerminateInstances'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) +@override_application_settings({"distributed_tracing.enabled": True}) +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "RunInstances"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DescribeInstances"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "TerminateInstances"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) @validate_transaction_metrics( - 'test_botocore_ec2:test_ec2', - scoped_metrics=_ec2_scoped_metrics, - rollup_metrics=_ec2_rollup_metrics, - background_task=True) + "test_botocore_ec2:test_ec2", + scoped_metrics=_ec2_scoped_metrics, + rollup_metrics=_ec2_rollup_metrics, + background_task=True, +) @background_task() @moto.mock_ec2 def test_ec2(): session = botocore.session.get_session() client = session.create_client( - 'ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "ec2", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY ) # Create instance resp = client.run_instances( - ImageId=UBUNTU_14_04_PARAVIRTUAL_AMI, - InstanceType='m1.small', - MinCount=1, - MaxCount=1, + ImageId=UBUNTU_14_04_PARAVIRTUAL_AMI, + InstanceType="m1.small", + MinCount=1, + MaxCount=1, ) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert len(resp['Instances']) == 1 - instance_id = resp['Instances'][0]['InstanceId'] + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert len(resp["Instances"]) == 1 + instance_id = resp["Instances"][0]["InstanceId"] # Describe instance resp = client.describe_instances(InstanceIds=[instance_id]) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['Reservations'][0]['Instances'][0]['InstanceId'] == instance_id + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["Reservations"][0]["Instances"][0]["InstanceId"] == instance_id # Delete instance resp = client.terminate_instances(InstanceIds=[instance_id]) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['TerminatingInstances'][0]['InstanceId'] == instance_id + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["TerminatingInstances"][0]["InstanceId"] == instance_id diff --git a/tests/external_httplib/test_httplib.py b/tests/external_httplib/test_httplib.py index c7747f8ff7..f67e68dc29 100644 --- a/tests/external_httplib/test_httplib.py +++ b/tests/external_httplib/test_httplib.py @@ -23,12 +23,7 @@ cache_outgoing_headers, insert_incoming_headers, ) -from testing_support.fixtures import ( - cat_enabled, - override_application_settings, - validate_tt_segment_params, -) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.fixtures import cat_enabled, override_application_settings from testing_support.validators.validate_cross_process_headers import ( validate_cross_process_headers, ) @@ -36,6 +31,12 @@ validate_external_node_params, ) from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.common.encoding_utils import DistributedTracePayload @@ -104,7 +105,8 @@ def test_httplib_https_request(server): ) @background_task(name="test_httplib:test_httplib_https_request") def _test(): - connection = httplib.HTTPSConnection("localhost", server.port) + # fix HTTPSConnection: https://wiki.openstack.org/wiki/OSSN/OSSN-0033 + connection = httplib.HTTPSConnection("localhost", server.port) # nosec # It doesn't matter that a SSL exception is raised here because the # agent still records this as an external request try: diff --git a/tests/external_httpx/test_client.py b/tests/external_httpx/test_client.py index 87a1bc7d01..b4760a38f0 100644 --- a/tests/external_httpx/test_client.py +++ b/tests/external_httpx/test_client.py @@ -19,7 +19,6 @@ dt_enabled, override_application_settings, override_generic_settings, - validate_tt_segment_params, ) from testing_support.mock_external_http_server import ( MockExternalHTTPHResponseHeadersServer, @@ -28,8 +27,15 @@ validate_cross_process_headers, ) from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.api.time_trace import current_trace diff --git a/tests/framework_aiohttp/test_server_cat.py b/tests/framework_aiohttp/test_server_cat.py index 44b5c72174..28af90d8df 100644 --- a/tests/framework_aiohttp/test_server_cat.py +++ b/tests/framework_aiohttp/test_server_cat.py @@ -37,7 +37,7 @@ def record_aiohttp1_raw_headers(raw_headers): try: - import aiohttp.protocol # noqa: F401 + import aiohttp.protocol # noqa: F401, pylint: disable=W0611 except ImportError: def pass_through(function): diff --git a/tests/messagebroker_confluentkafka/test_consumer.py b/tests/messagebroker_confluentkafka/test_consumer.py index 5478b7c804..31f9478b30 100644 --- a/tests/messagebroker_confluentkafka/test_consumer.py +++ b/tests/messagebroker_confluentkafka/test_consumer.py @@ -14,14 +14,13 @@ import pytest from conftest import cache_kafka_consumer_headers -from testing_support.fixtures import ( - reset_core_stats_engine, - validate_attributes, - validate_error_event_attributes_outside_transaction, -) +from testing_support.fixtures import reset_core_stats_engine, validate_attributes from testing_support.validators.validate_distributed_trace_accepted import ( validate_distributed_trace_accepted, ) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( + validate_error_event_attributes_outside_transaction, +) from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) diff --git a/tests/messagebroker_kafkapython/test_consumer.py b/tests/messagebroker_kafkapython/test_consumer.py index 47e42d6c93..78ba086c6e 100644 --- a/tests/messagebroker_kafkapython/test_consumer.py +++ b/tests/messagebroker_kafkapython/test_consumer.py @@ -14,14 +14,13 @@ import pytest from conftest import cache_kafka_consumer_headers -from testing_support.fixtures import ( - reset_core_stats_engine, - validate_attributes, - validate_error_event_attributes_outside_transaction, -) +from testing_support.fixtures import reset_core_stats_engine, validate_attributes from testing_support.validators.validate_distributed_trace_accepted import ( validate_distributed_trace_accepted, ) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( + validate_error_event_attributes_outside_transaction, +) from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) diff --git a/tests/messagebroker_kafkapython/test_serialization.py b/tests/messagebroker_kafkapython/test_serialization.py index f58d082ec7..0b2bee74df 100644 --- a/tests/messagebroker_kafkapython/test_serialization.py +++ b/tests/messagebroker_kafkapython/test_serialization.py @@ -15,8 +15,8 @@ import json import pytest -from testing_support.fixtures import ( - reset_core_stats_engine, +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( validate_error_event_attributes_outside_transaction, ) from testing_support.validators.validate_transaction_errors import ( diff --git a/tests/testing_support/fixtures.py b/tests/testing_support/fixtures.py index 07de22cf0a..ce6166f0b8 100644 --- a/tests/testing_support/fixtures.py +++ b/tests/testing_support/fixtures.py @@ -40,13 +40,7 @@ register_application, ) from newrelic.common.agent_http import DeveloperModeClient -from newrelic.common.encoding_utils import ( - deobfuscate, - json_decode, - json_encode, - obfuscate, - unpack_field, -) +from newrelic.common.encoding_utils import json_encode, obfuscate from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import ( ObjectProxy, @@ -63,10 +57,6 @@ AttributeFilter, ) from newrelic.core.config import apply_config_setting, flatten_settings, global_settings -from newrelic.core.database_utils import SQLConnections - -# from newrelic.core.internal_metrics import InternalTraceContext -# from newrelic.core.stats_engine import CustomMetrics from newrelic.network.exceptions import RetryDataForRequest from newrelic.packages import six @@ -434,383 +424,6 @@ def check_event_attributes(event_data, required_params=None, forgone_params=None assert intrinsics[param] == value, ((param, value), intrinsics) -def check_error_attributes( - parameters, required_params=None, forgone_params=None, exact_attrs=None, is_transaction=True -): - required_params = required_params or {} - forgone_params = forgone_params or {} - exact_attrs = exact_attrs or {} - - parameter_fields = ["userAttributes"] - if is_transaction: - parameter_fields.extend(["stack_trace", "agentAttributes", "intrinsics"]) - - for field in parameter_fields: - assert field in parameters - - # we can remove this after agent attributes transition is all over - assert "parameter_groups" not in parameters - assert "custom_params" not in parameters - assert "request_params" not in parameters - assert "request_uri" not in parameters - - check_attributes(parameters, required_params, forgone_params, exact_attrs) - - -def check_attributes(parameters, required_params=None, forgone_params=None, exact_attrs=None): - required_params = required_params or {} - forgone_params = forgone_params or {} - exact_attrs = exact_attrs or {} - - intrinsics = parameters.get("intrinsics", {}) - user_attributes = parameters.get("userAttributes", {}) - agent_attributes = parameters.get("agentAttributes", {}) - - if required_params: - for param in required_params["agent"]: - assert param in agent_attributes, (param, agent_attributes) - for param in required_params["user"]: - assert param in user_attributes, (param, user_attributes) - for param in required_params["intrinsic"]: - assert param in intrinsics, (param, intrinsics) - - if forgone_params: - for param in forgone_params["agent"]: - assert param not in agent_attributes, (param, agent_attributes) - for param in forgone_params["user"]: - assert param not in user_attributes, (param, user_attributes) - for param in forgone_params["intrinsic"]: - assert param not in intrinsics, (param, intrinsics) - - if exact_attrs: - for param, value in exact_attrs["agent"].items(): - assert agent_attributes[param] == value, ((param, value), agent_attributes) - for param, value in exact_attrs["user"].items(): - assert user_attributes[param] == value, ((param, value), user_attributes) - for param, value in exact_attrs["intrinsic"].items(): - assert intrinsics[param] == value, ((param, value), intrinsics) - - -def validate_custom_event_collector_json(num_events=1): - """Validate the format, types and number of custom events.""" - - @transient_function_wrapper("newrelic.core.application", "Application.record_transaction") - def _validate_custom_event_collector_json(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - stats = instance._stats_engine - settings = stats.settings - - agent_run_id = 666 - sampling_info = stats.custom_events.sampling_info - samples = list(stats.custom_events) - - # Emulate the payload used in data_collector.py - - payload = (agent_run_id, sampling_info, samples) - collector_json = json_encode(payload) - - decoded_json = json.loads(collector_json) - - decoded_agent_run_id = decoded_json[0] - decoded_sampling_info = decoded_json[1] - decoded_events = decoded_json[2] - - assert decoded_agent_run_id == agent_run_id - assert decoded_sampling_info == sampling_info - - max_setting = settings.event_harvest_config.harvest_limits.custom_event_data - assert decoded_sampling_info["reservoir_size"] == max_setting - - assert decoded_sampling_info["events_seen"] == num_events - assert len(decoded_events) == num_events - - for intrinsics, attributes in decoded_events: - assert isinstance(intrinsics, dict) - assert isinstance(attributes, dict) - - return result - - return _validate_custom_event_collector_json - - -def validate_tt_parameters(required_params=None, forgone_params=None): - required_params = required_params or {} - forgone_params = forgone_params or {} - - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") - def _validate_tt_parameters(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - # Now that transaction has been recorded, generate - # a transaction trace - - connections = SQLConnections() - trace_data = instance.transaction_trace_data(connections) - pack_data = unpack_field(trace_data[0][4]) - tt_intrinsics = pack_data[0][4]["intrinsics"] - - for name in required_params: - assert name in tt_intrinsics, "name=%r, intrinsics=%r" % (name, tt_intrinsics) - assert tt_intrinsics[name] == required_params[name], "name=%r, value=%r, intrinsics=%r" % ( - name, - required_params[name], - tt_intrinsics, - ) - - for name in forgone_params: - assert name not in tt_intrinsics, "name=%r, intrinsics=%r" % (name, tt_intrinsics) - - return result - - return _validate_tt_parameters - - -def validate_tt_segment_params(forgone_params=(), present_params=(), exact_params=None): - exact_params = exact_params or {} - recorded_traces = [] - - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") - def _extract_trace(wrapped, instance, args, kwargs): - result = wrapped(*args, **kwargs) - - # Now that transaction has been recorded, generate - # a transaction trace - - connections = SQLConnections() - trace_data = instance.transaction_trace_data(connections) - # Save the recorded traces - recorded_traces.extend(trace_data) - - return result - - @function_wrapper - def validator(wrapped, instance, args, kwargs): - new_wrapper = _extract_trace(wrapped) - result = new_wrapper(*args, **kwargs) - - # Verify that traces have been recorded - assert recorded_traces - - # Extract the first transaction trace - transaction_trace = recorded_traces[0] - pack_data = unpack_field(transaction_trace[4]) - - # Extract the root segment from the root node - root_segment = pack_data[0][3] - - recorded_params = {} - - def _validate_segment_params(segment): - segment_params = segment[3] - - # Translate from the string cache - for key, value in segment_params.items(): - if hasattr(value, "startswith") and value.startswith("`"): - try: - index = int(value[1:]) - value = pack_data[1][index] - except ValueError: - pass - segment_params[key] = value - - recorded_params.update(segment_params) - - for child_segment in segment[4]: - _validate_segment_params(child_segment) - - _validate_segment_params(root_segment) - - recorded_params_set = set(recorded_params.keys()) - - # Verify that the params in present params have been recorded - present_params_set = set(present_params) - assert recorded_params_set.issuperset(present_params_set) - - # Verify that all forgone params are omitted - recorded_forgone_params = recorded_params_set & set(forgone_params) - assert not recorded_forgone_params - - # Verify that all exact params are correct - for key, value in exact_params.items(): - assert recorded_params[key] == value - - return result - - return validator - - -def validate_browser_attributes(required_params=None, forgone_params=None): - required_params = required_params or {} - forgone_params = forgone_params or {} - - @transient_function_wrapper("newrelic.api.web_transaction", "WSGIWebTransaction.browser_timing_footer") - def _validate_browser_attributes(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - - # pick out attributes from footer string_types - - footer_data = result.split("NREUM.info=")[1] - footer_data = footer_data.split("")[0] - footer_data = json.loads(footer_data) - - if "intrinsic" in required_params: - for attr in required_params["intrinsic"]: - assert attr in footer_data - - if "atts" in footer_data: - obfuscation_key = instance._settings.license_key[:13] - attributes = json_decode(deobfuscate(footer_data["atts"], obfuscation_key)) - else: - # if there are no user or agent attributes, there will be no dict - # for them in the browser data - - attributes = None - - if "user" in required_params: - for attr in required_params["user"]: - assert attr in attributes["u"] - - if "agent" in required_params: - for attr in required_params["agent"]: - assert attr in attributes["a"] - - if "user" in forgone_params: - if attributes: - if "u" in attributes: - for attr in forgone_params["user"]: - assert attr not in attributes["u"] - - if "agent" in forgone_params: - if attributes: - if "a" in attributes: - for attr in forgone_params["agent"]: - assert attr not in attributes["a"] - - return result - - return _validate_browser_attributes - - -def validate_error_event_attributes(required_params=None, forgone_params=None, exact_attrs=None): - """Check the error event for attributes, expect only one error to be - present in the transaction. - """ - required_params = required_params or {} - forgone_params = forgone_params or {} - exact_attrs = exact_attrs or {} - error_data_samples = [] - - @function_wrapper - def _validate_wrapper(wrapped, instance, args, kwargs): - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") - def _validate_error_event_attributes(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - event_data = instance.error_events - for sample in event_data: - error_data_samples.append(sample) - - check_event_attributes(event_data, required_params, forgone_params, exact_attrs) - - return result - - _new_wrapper = _validate_error_event_attributes(wrapped) - val = _new_wrapper(*args, **kwargs) - assert error_data_samples - return val - - return _validate_wrapper - - -def validate_error_trace_attributes_outside_transaction( - err_name, required_params=None, forgone_params=None, exact_attrs=None -): - required_params = required_params or {} - forgone_params = forgone_params or {} - exact_attrs = exact_attrs or {} - - target_error = [] - - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.notice_error") - def _validate_error_trace_attributes_outside_transaction(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - target_error.append(core_application_stats_engine_error(err_name)) - - return result - - - @function_wrapper - def _validator_wrapper(wrapped, instance, args, kwargs): - result = _validate_error_trace_attributes_outside_transaction(wrapped)(*args, **kwargs) - - assert target_error and target_error[0] is not None, "No error found with name %s" % err_name - check_error_attributes(target_error[0].parameters, required_params, forgone_params, exact_attrs) - - return result - - - return _validator_wrapper - - -def validate_error_event_attributes_outside_transaction( - required_params=None, forgone_params=None, exact_attrs=None, num_errors=None -): - required_params = required_params or {} - forgone_params = forgone_params or {} - - event_data = [] - - @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.notice_error") - def _validate_error_event_attributes_outside_transaction(wrapped, instance, args, kwargs): - try: - result = wrapped(*args, **kwargs) - except: - raise - else: - for event in instance.error_events: - event_data.append(event) - - return result - - @function_wrapper - def wrapper(wrapped, instance, args, kwargs): - try: - result = _validate_error_event_attributes_outside_transaction(wrapped)(*args, **kwargs) - except: - raise - else: - if num_errors is not None: - exc_message = ( - "Expected: %d, Got: %d. Verify StatsEngine is being reset before using this validator." - % (num_errors, len(event_data)) - ) - assert num_errors == len(event_data), exc_message - - for event in event_data: - check_event_attributes([event], required_params, forgone_params, exact_attrs=exact_attrs) - - return result - - return wrapper - - def validate_request_params_omitted(): @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") def _validate_request_params(wrapped, instance, args, kwargs): @@ -1713,6 +1326,63 @@ def _bind_params(method, *args, **kwargs): return send_request_wrapper +def check_attributes(parameters, required_params=None, forgone_params=None, exact_attrs=None): + required_params = required_params or {} + forgone_params = forgone_params or {} + exact_attrs = exact_attrs or {} + + intrinsics = parameters.get("intrinsics", {}) + user_attributes = parameters.get("userAttributes", {}) + agent_attributes = parameters.get("agentAttributes", {}) + + if required_params: + for param in required_params["agent"]: + assert param in agent_attributes, (param, agent_attributes) + for param in required_params["user"]: + assert param in user_attributes, (param, user_attributes) + for param in required_params["intrinsic"]: + assert param in intrinsics, (param, intrinsics) + + if forgone_params: + for param in forgone_params["agent"]: + assert param not in agent_attributes, (param, agent_attributes) + for param in forgone_params["user"]: + assert param not in user_attributes, (param, user_attributes) + for param in forgone_params["intrinsic"]: + assert param not in intrinsics, (param, intrinsics) + + if exact_attrs: + for param, value in exact_attrs["agent"].items(): + assert agent_attributes[param] == value, ((param, value), agent_attributes) + for param, value in exact_attrs["user"].items(): + assert user_attributes[param] == value, ((param, value), user_attributes) + for param, value in exact_attrs["intrinsic"].items(): + assert intrinsics[param] == value, ((param, value), intrinsics) + + +def check_error_attributes( + parameters, required_params=None, forgone_params=None, exact_attrs=None, is_transaction=True +): + required_params = required_params or {} + forgone_params = forgone_params or {} + exact_attrs = exact_attrs or {} + + parameter_fields = ["userAttributes"] + if is_transaction: + parameter_fields.extend(["stack_trace", "agentAttributes", "intrinsics"]) + + for field in parameter_fields: + assert field in parameters + + # we can remove this after agent attributes transition is all over + assert "parameter_groups" not in parameters + assert "custom_params" not in parameters + assert "request_params" not in parameters + assert "request_uri" not in parameters + + check_attributes(parameters, required_params, forgone_params, exact_attrs) + + class Environ(object): """Context manager for setting environment variables temporarily.""" diff --git a/tests/testing_support/validators/validate_browser_attributes.py b/tests/testing_support/validators/validate_browser_attributes.py new file mode 100644 index 0000000000..bf8c82ccc3 --- /dev/null +++ b/tests/testing_support/validators/validate_browser_attributes.py @@ -0,0 +1,74 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from newrelic.common.encoding_utils import deobfuscate, json_decode +from newrelic.common.object_wrapper import transient_function_wrapper + + +def validate_browser_attributes(required_params=None, forgone_params=None): + required_params = required_params or {} + forgone_params = forgone_params or {} + + @transient_function_wrapper("newrelic.api.web_transaction", "WSGIWebTransaction.browser_timing_footer") + def _validate_browser_attributes(wrapped, instance, args, kwargs): + try: + result = wrapped(*args, **kwargs) + except: + raise + + # pick out attributes from footer string_types + + footer_data = result.split("NREUM.info=")[1] + footer_data = footer_data.split("")[0] + footer_data = json.loads(footer_data) + + if "intrinsic" in required_params: + for attr in required_params["intrinsic"]: + assert attr in footer_data + + if "atts" in footer_data: + obfuscation_key = instance._settings.license_key[:13] + attributes = json_decode(deobfuscate(footer_data["atts"], obfuscation_key)) + else: + + # if there are no user or agent attributes, there will be no dict + # for them in the browser data + + attributes = None + + if "user" in required_params: + for attr in required_params["user"]: + assert attr in attributes["u"] + + if "agent" in required_params: + for attr in required_params["agent"]: + assert attr in attributes["a"] + + if "user" in forgone_params: + if attributes: + if "u" in attributes: + for attr in forgone_params["user"]: + assert attr not in attributes["u"] + + if "agent" in forgone_params: + if attributes: + if "a" in attributes: + for attr in forgone_params["agent"]: + assert attr not in attributes["a"] + + return result + + return _validate_browser_attributes diff --git a/tests/testing_support/validators/validate_custom_event_collector_json.py b/tests/testing_support/validators/validate_custom_event_collector_json.py new file mode 100644 index 0000000000..54711ef5fd --- /dev/null +++ b/tests/testing_support/validators/validate_custom_event_collector_json.py @@ -0,0 +1,64 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json + +from newrelic.common.encoding_utils import json_encode +from newrelic.common.object_wrapper import transient_function_wrapper + + +def validate_custom_event_collector_json(num_events=1): + """Validate the format, types and number of custom events.""" + + @transient_function_wrapper("newrelic.core.application", "Application.record_transaction") + def _validate_custom_event_collector_json(wrapped, instance, args, kwargs): + try: + result = wrapped(*args, **kwargs) + except: + raise + + stats = instance._stats_engine + settings = stats.settings + + agent_run_id = 666 + sampling_info = stats.custom_events.sampling_info + samples = list(stats.custom_events) + + # Emulate the payload used in data_collector.py + + payload = (agent_run_id, sampling_info, samples) + collector_json = json_encode(payload) + + decoded_json = json.loads(collector_json) + + decoded_agent_run_id = decoded_json[0] + decoded_sampling_info = decoded_json[1] + decoded_events = decoded_json[2] + + assert decoded_agent_run_id == agent_run_id + assert decoded_sampling_info == sampling_info + + max_setting = settings.event_harvest_config.harvest_limits.custom_event_data + assert decoded_sampling_info["reservoir_size"] == max_setting + + assert decoded_sampling_info["events_seen"] == num_events + assert len(decoded_events) == num_events + + for (intrinsics, attributes) in decoded_events: + assert isinstance(intrinsics, dict) + assert isinstance(attributes, dict) + + return result + + return _validate_custom_event_collector_json diff --git a/tests/testing_support/validators/validate_error_event_attributes.py b/tests/testing_support/validators/validate_error_event_attributes.py new file mode 100644 index 0000000000..6c68085910 --- /dev/null +++ b/tests/testing_support/validators/validate_error_event_attributes.py @@ -0,0 +1,51 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import check_event_attributes + +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper + + +def validate_error_event_attributes(required_params=None, forgone_params=None, exact_attrs=None): + """Check the error event for attributes, expect only one error to be + present in the transaction. + """ + required_params = required_params or {} + forgone_params = forgone_params or {} + exact_attrs = exact_attrs or {} + error_data_samples = [] + + @function_wrapper + def _validate_wrapper(wrapped, instance, args, kwargs): + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + def _validate_error_event_attributes(wrapped, instance, args, kwargs): + try: + result = wrapped(*args, **kwargs) + except: + raise + + event_data = instance.error_events + for sample in event_data: + error_data_samples.append(sample) + + check_event_attributes(event_data, required_params, forgone_params, exact_attrs) + + return result + + _new_wrapper = _validate_error_event_attributes(wrapped) + val = _new_wrapper(*args, **kwargs) + assert error_data_samples + return val + + return _validate_wrapper diff --git a/tests/testing_support/validators/validate_error_event_attributes_outside_transaction.py b/tests/testing_support/validators/validate_error_event_attributes_outside_transaction.py new file mode 100644 index 0000000000..74fbe9361a --- /dev/null +++ b/tests/testing_support/validators/validate_error_event_attributes_outside_transaction.py @@ -0,0 +1,48 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import check_event_attributes + +from newrelic.common.object_wrapper import transient_function_wrapper + + +def validate_error_event_attributes_outside_transaction( + required_params=None, forgone_params=None, exact_attrs=None, num_errors=None +): + required_params = required_params or {} + forgone_params = forgone_params or {} + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.notice_error") + def _validate_error_event_attributes_outside_transaction(wrapped, instance, args, kwargs): + + try: + result = wrapped(*args, **kwargs) + except: + raise + + event_data = list(instance.error_events) + + if num_errors is not None: + exc_message = "Expected: %d, Got: %d. Verify StatsEngine is being reset before using this validator." % ( + num_errors, + len(event_data), + ) + assert num_errors == len(event_data), exc_message + + for event in event_data: + check_event_attributes([event], required_params, forgone_params, exact_attrs=exact_attrs) + + return result + + return _validate_error_event_attributes_outside_transaction diff --git a/tests/testing_support/validators/validate_error_trace_attributes_outside_transaction.py b/tests/testing_support/validators/validate_error_trace_attributes_outside_transaction.py new file mode 100644 index 0000000000..08c22eceaf --- /dev/null +++ b/tests/testing_support/validators/validate_error_trace_attributes_outside_transaction.py @@ -0,0 +1,45 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( + check_error_attributes, + core_application_stats_engine_error, +) + +from newrelic.common.object_wrapper import transient_function_wrapper + + +def validate_error_trace_attributes_outside_transaction( + err_name, required_params=None, forgone_params=None, exact_attrs=None +): + required_params = required_params or {} + forgone_params = forgone_params or {} + exact_attrs = exact_attrs or {} + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.notice_error") + def _validate_error_trace_attributes_outside_transaction(wrapped, instance, args, kwargs): + try: + result = wrapped(*args, **kwargs) + except: + raise + + target_error = core_application_stats_engine_error(err_name) + + check_error_attributes( + target_error.parameters, required_params, forgone_params, exact_attrs, is_transaction=False + ) + + return result + + return _validate_error_trace_attributes_outside_transaction diff --git a/tests/testing_support/validators/validate_tt_parameters.py b/tests/testing_support/validators/validate_tt_parameters.py new file mode 100644 index 0000000000..7e8d8bd89c --- /dev/null +++ b/tests/testing_support/validators/validate_tt_parameters.py @@ -0,0 +1,52 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.common.encoding_utils import unpack_field +from newrelic.common.object_wrapper import transient_function_wrapper +from newrelic.core.database_utils import SQLConnections + + +def validate_tt_parameters(required_params=None, forgone_params=None): + required_params = required_params or {} + forgone_params = forgone_params or {} + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + def _validate_tt_parameters(wrapped, instance, args, kwargs): + try: + result = wrapped(*args, **kwargs) + except: + raise + + # Now that transaction has been recorded, generate + # a transaction trace + + connections = SQLConnections() + trace_data = instance.transaction_trace_data(connections) + pack_data = unpack_field(trace_data[0][4]) + tt_intrinsics = pack_data[0][4]["intrinsics"] + + for name in required_params: + assert name in tt_intrinsics, "name=%r, intrinsics=%r" % (name, tt_intrinsics) + assert tt_intrinsics[name] == required_params[name], "name=%r, value=%r, intrinsics=%r" % ( + name, + required_params[name], + tt_intrinsics, + ) + + for name in forgone_params: + assert name not in tt_intrinsics, "name=%r, intrinsics=%r" % (name, tt_intrinsics) + + return result + + return _validate_tt_parameters diff --git a/tests/testing_support/validators/validate_tt_segment_params.py b/tests/testing_support/validators/validate_tt_segment_params.py new file mode 100644 index 0000000000..5355a50f8a --- /dev/null +++ b/tests/testing_support/validators/validate_tt_segment_params.py @@ -0,0 +1,91 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.common.encoding_utils import unpack_field +from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper +from newrelic.core.database_utils import SQLConnections + + +def validate_tt_segment_params(forgone_params=(), present_params=(), exact_params=None): + exact_params = exact_params or {} + recorded_traces = [] + + @transient_function_wrapper("newrelic.core.stats_engine", "StatsEngine.record_transaction") + def _extract_trace(wrapped, instance, args, kwargs): + result = wrapped(*args, **kwargs) + + # Now that transaction has been recorded, generate + # a transaction trace + + connections = SQLConnections() + trace_data = instance.transaction_trace_data(connections) + # Save the recorded traces + recorded_traces.extend(trace_data) + + return result + + @function_wrapper + def validator(wrapped, instance, args, kwargs): + new_wrapper = _extract_trace(wrapped) + result = new_wrapper(*args, **kwargs) + + # Verify that traces have been recorded + assert recorded_traces + + # Extract the first transaction trace + transaction_trace = recorded_traces[0] + pack_data = unpack_field(transaction_trace[4]) + + # Extract the root segment from the root node + root_segment = pack_data[0][3] + + recorded_params = {} + + def _validate_segment_params(segment): + segment_params = segment[3] + + # Translate from the string cache + for key, value in segment_params.items(): + if hasattr(value, "startswith") and value.startswith("`"): + try: + index = int(value[1:]) + value = pack_data[1][index] + except ValueError: + pass + segment_params[key] = value + + recorded_params.update(segment_params) + + for child_segment in segment[4]: + _validate_segment_params(child_segment) + + _validate_segment_params(root_segment) + + recorded_params_set = set(recorded_params.keys()) + + # Verify that the params in present params have been recorded + present_params_set = set(present_params) + assert recorded_params_set.issuperset(present_params_set) + + # Verify that all forgone params are omitted + recorded_forgone_params = recorded_params_set & set(forgone_params) + assert not recorded_forgone_params + + # Verify that all exact params are correct + for key, value in exact_params.items(): + assert recorded_params[key] == value + + return result + + return validator From 668b0a9a86ac70282ae0df7115959e0f1750b225 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:07:04 -0700 Subject: [PATCH 054/525] Fix set output warning using new GHA syntax (#833) * Fix set output warning using new GHA syntax * Fix quoting --- .github/workflows/tests.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc73168ebf..5d396c9a77 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -126,7 +126,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -163,7 +163,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -206,7 +206,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -266,7 +266,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -321,7 +321,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -371,7 +371,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -423,7 +423,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -473,7 +473,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -524,7 +524,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -596,7 +596,7 @@ jobs: # - name: Get Environments # id: get-envs # run: | - # echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT # env: # GROUP_NUMBER: ${{ matrix.group-number }} @@ -646,7 +646,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -698,7 +698,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -751,7 +751,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -800,7 +800,7 @@ jobs: - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} From abb6405d2bfd629ed83f48e8a17b4a28e3a3c352 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 21 Jun 2023 15:03:38 -0700 Subject: [PATCH 055/525] Remove Python 2.7 and pypy2 testing (#835) * Change setup-python to @v2 for py2.7 * Remove py27 and pypy testing * Fix syntax errors * Fix comma related syntax errors * Fix more issues in tox * Remove gearman test --- .../actions/setup-python-matrix/action.yml | 16 +-- .github/workflows/tests.yml | 98 ++++++++-------- newrelic/config.py | 30 ++--- tox.ini | 109 ++++++++---------- 4 files changed, 119 insertions(+), 134 deletions(-) diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml index 299dd2e7bd..a11e2197c2 100644 --- a/.github/actions/setup-python-matrix/action.yml +++ b/.github/actions/setup-python-matrix/action.yml @@ -8,10 +8,10 @@ runs: python-version: "pypy-3.7" architecture: x64 - - uses: actions/setup-python@v4 - with: - python-version: "pypy-2.7" - architecture: x64 + # - uses: actions/setup-python@v4 + # with: + # python-version: "pypy-2.7" + # architecture: x64 - uses: actions/setup-python@v4 with: @@ -38,10 +38,10 @@ runs: python-version: "3.11" architecture: x64 - - uses: actions/setup-python@v4 - with: - python-version: "2.7" - architecture: x64 + # - uses: actions/setup-python@v4 + # with: + # python-version: "2.7" + # architecture: x64 - name: Install Dependencies shell: bash diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5d396c9a77..b5ab093f19 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - - gearman + # - gearman - grpc #- kafka - libcurl @@ -769,51 +769,51 @@ jobs: path: ./**/.coverage.* retention-days: 1 - gearman: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - services: - gearman: - image: artefactual/gearmand - ports: - - 4730:4730 - # Set health checks to wait until gearman has started - options: >- - --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - - name: Get Environments - id: get-envs - run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 + # gearman: + # env: + # TOTAL_GROUPS: 1 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1] + + # runs-on: ubuntu-20.04 + # timeout-minutes: 30 + + # services: + # gearman: + # image: artefactual/gearmand + # ports: + # - 4730:4730 + # # Set health checks to wait until gearman has started + # options: >- + # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + + # steps: + # - uses: actions/checkout@v3 + # - uses: ./.github/actions/setup-python-matrix + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index df95db0290..5c3961b026 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - _process_module_definition( - "gearman.client", - "newrelic.hooks.application_gearman", - "instrument_gearman_client", - ) - _process_module_definition( - "gearman.connection_manager", - "newrelic.hooks.application_gearman", - "instrument_gearman_connection_manager", - ) - _process_module_definition( - "gearman.worker", - "newrelic.hooks.application_gearman", - "instrument_gearman_worker", - ) + # _process_module_definition( + # "gearman.client", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_client", + # ) + # _process_module_definition( + # "gearman.connection_manager", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_connection_manager", + # ) + # _process_module_definition( + # "gearman.worker", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_worker", + # ) _process_module_definition( "botocore.endpoint", diff --git a/tox.ini b/tox.ini index 7e4f1e3707..6a625b2dbd 100644 --- a/tox.ini +++ b/tox.ini @@ -42,10 +42,10 @@ [tox] setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py27,py37,py38,py310,py311}, + python-adapter_gevent-{py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -54,95 +54,81 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy,pypy37}-without_extensions, - python-agent_streaming-py27-grpc0125-{with,without}_extensions, + python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy37}-without_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy,pypy37}-without_extensions, - python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - gearman-application_gearman-{py27,pypy}, - python-component_djangorestframework-py27-djangorestframework0300, + python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy37}-without_extensions, + python-application_celery-{py37,py38,py39,py310,py311,pypy37}, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, - python-component_flask_rest-{py27,pypy}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, - python-component_tastypie-{py27,pypy}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-cross_agent-pypy-without_extensions, + python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{pypy,py27,py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-memcached01, - mysql-datastore_mysql-mysql080023-py27, + memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest - postgres-datastore_psycopg2cffi-{py27,pypy,py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest - memcached-datastore_pylibmc-{py27,py37}, - memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy,pypy37}-pymongo04, - mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - redis-datastore_redis-{py27,py37,py38,pypy,pypy37}-redis03, + postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, + postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, + memcached-datastore_pylibmc-{py37}, + memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, + mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, + solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, + redis-datastore_redis-{py37,py38,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - solr-datastore_solrpy-{py27,pypy}-solrpy{00,01}, - python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, + python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, + python-external_boto3-{py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_feedparser-py27-feedparser{05,06}, - python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, - python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_http-{py37,py38,py39,py310,py311}, + python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, + python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_urllib3-{py27,py37,pypy}-urllib3{0109}, - python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-urllib3latest, + python-external_requests-{py37,py38,py39,py310,py311,pypy37}, + python-external_urllib3-{py37}-urllib3{0109}, + python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-py27-bottle{0008,0009,0010}, - python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, - python-framework_bottle-pypy-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{pypy,py27}-Django0103, - python-framework_django-{pypy,py27,py37}-Django0108, + python-framework_django-{py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py27,py37,py38,py39,pypy,pypy37}-falcon0103, + python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{pypy,py27}-flask0012, - python-framework_flask-{pypy,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-graphql02, + python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, - grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{pypy,py27,py38}-Pyramid0104, - python-framework_pyramid-{pypy,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{py38}-Pyramid0104, + python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -150,27 +136,26 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy,pypy37}-pika0.13, + rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{pypy,py27,py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py27,py37,py311}-genshilatest - python-template_mako-{py27,py37,py310,py311} + kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py37,py311}-genshilatest, + python-template_mako-{py37,py310,py311}, [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 - {py27,pypy}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -202,7 +187,7 @@ deps = agent_features: beautifulsoup4 application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 - application_gearman: gearman<3.0.0 + ; application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -430,7 +415,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - application_gearman: tests/application_gearman + ; application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From ab92dafcb8a970be58e352cc6d131340bac23d43 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 22 Jun 2023 10:17:48 -0700 Subject: [PATCH 056/525] Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs --- .../actions/setup-python-matrix/action.yml | 50 ----- .github/containers/Dockerfile | 81 ++++++++ .github/containers/Makefile | 43 ++++ .github/containers/install-python.sh | 69 +++++++ .github/containers/requirements.txt | 5 + .github/scripts/retry.sh | 14 ++ .github/workflows/build-ci-image.yml | 68 +++++++ .github/workflows/get-envs.py | 14 ++ .github/workflows/tests.yml | 184 +++++++----------- newrelic/config.py | 30 +-- tox.ini | 121 +++++++----- 11 files changed, 449 insertions(+), 230 deletions(-) delete mode 100644 .github/actions/setup-python-matrix/action.yml create mode 100644 .github/containers/Dockerfile create mode 100644 .github/containers/Makefile create mode 100755 .github/containers/install-python.sh create mode 100644 .github/containers/requirements.txt create mode 100644 .github/workflows/build-ci-image.yml diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml deleted file mode 100644 index a11e2197c2..0000000000 --- a/.github/actions/setup-python-matrix/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "setup-python-matrix" -description: "Sets up all versions of python required for matrix testing in this repo." -runs: - using: "composite" - steps: - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "pypy-2.7" - # architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "2.7" - # architecture: x64 - - - name: Install Dependencies - shell: bash - run: | - python3.10 -m pip install -U pip - python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile new file mode 100644 index 0000000000..06b610f492 --- /dev/null +++ b/.github/containers/Dockerfile @@ -0,0 +1,81 @@ + +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +# Install OS packages +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + bash \ + build-essential \ + curl \ + expat \ + gcc \ + git \ + libbz2-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + libgmp-dev \ + liblzma-dev \ + libmpfr-dev \ + libncurses-dev \ + libpq-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + locales \ + make \ + openssl \ + python2-dev \ + python3-dev \ + python3-pip \ + odbc-postgresql \ + unzip \ + wget \ + zip \ + zlib1g \ + zlib1g-dev && \ + rm -rf /var/lib/apt/lists/* + +# Setup ODBC config +RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini && \ + sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini && \ + sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + +# Set the locale +RUN locale-gen --no-purge en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 + +# Set user to non-root +ENV HOME /home/github +RUN groupadd -g 1000 github && \ + useradd -m -d "${HOME}" -s /bin/bash -u 1000 -g 1000 github +USER 1000:1000 +WORKDIR "${HOME}" + +# Install pyenv +ENV PYENV_ROOT="${HOME}/.pyenv" +RUN curl https://pyenv.run/ | /bin/bash +ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:${PATH}" +RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ + pyenv update + +# Install Python +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7 pypy3.7" +COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh +COPY ./requirements.txt /requirements.txt +RUN /tmp/install-python.sh && \ + rm /tmp/install-python.sh diff --git a/.github/containers/Makefile b/.github/containers/Makefile new file mode 100644 index 0000000000..c84cb95dc9 --- /dev/null +++ b/.github/containers/Makefile @@ -0,0 +1,43 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Repository root for mounting into container. +REPO_ROOT:=$(realpath $(dir $(realpath $(firstword $(MAKEFILE_LIST))))../../) + +.PHONY: default +default: test + +.PHONY: build +build: + @# Perform a shortened build for testing + @docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' . -t ghcr.io/newrelic/python-agent-ci:local + +.PHONY: test +test: build + @# Ensure python versions are usable + @docker run --rm ghcr.io/newrelic/python-agent-ci:local /bin/bash -c '\ + python3.10 --version && \ + python2.7 --version && \ + touch tox.ini && tox --version && \ + echo "Success! Python versions installed."' + +.PHONY: run +run: build + @docker run --rm -it \ + --mount type=bind,source="$(REPO_ROOT)",target=/home/github/python-agent \ + --workdir=/home/github/python-agent \ + -e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \ + -e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \ + -e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \ + ghcr.io/newrelic/python-agent-ci:local /bin/bash diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh new file mode 100755 index 0000000000..51ff41cd61 --- /dev/null +++ b/.github/containers/install-python.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +SED=$(which gsed || which sed) + +SCRIPT_DIR=$(dirname "$0") +PIP_REQUIREMENTS=$(cat /requirements.txt) + +main() { + # Coerce space separated string to array + if [[ ${#PYTHON_VERSIONS[@]} -eq 1 ]]; then + PYTHON_VERSIONS=($PYTHON_VERSIONS) + fi + + if [[ -z "${PYTHON_VERSIONS[@]}" ]]; then + echo "No python versions specified. Make sure PYTHON_VERSIONS is set." 1>&2 + exit 1 + fi + + # Find all latest pyenv supported versions for requested python versions + PYENV_VERSIONS=() + for v in "${PYTHON_VERSIONS[@]}"; do + LATEST=$(pyenv latest -k "$v" || get_latest_patch_version "$v") + if [[ -z "$LATEST" ]]; then + echo "Latest version could not be found for ${v}." 1>&2 + exit 1 + fi + PYENV_VERSIONS+=($LATEST) + done + + # Install each specific version + for v in "${PYENV_VERSIONS[@]}"; do + pyenv install "$v" & + done + wait + + # Set all installed versions as globally accessible + pyenv global ${PYENV_VERSIONS[@]} + + # Install dependencies for main python installation + pyenv exec pip install --upgrade $PIP_REQUIREMENTS +} + +get_latest_patch_version() { + pyenv install --list | # Get all python versions + $SED 's/^ *//g' | # Remove leading whitespace + grep -E "^$1" | # Find specified version by matching start of line + grep -v -- "-c-jit-latest" | # Filter out pypy JIT versions + $SED -E '/(-[a-zA-Z]+$)|(a[0-9]+)|(b[0-9]+)|(rc[0-9]+)/!{s/$/_/}' | # Append trailing _ to any non development versions to place them lower when sorted + sort -V | # Sort using version sorting + $SED 's/_$//' | # Remove any added trailing underscores to correct version names + tail -1 # Grab last result as latest version +} + +main diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt new file mode 100644 index 0000000000..54f3c39d00 --- /dev/null +++ b/.github/containers/requirements.txt @@ -0,0 +1,5 @@ +pip +setuptools +wheel +virtualenv<20.22.1 +tox \ No newline at end of file diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh index 1cb17836eb..b5d51f77b5 100755 --- a/.github/scripts/retry.sh +++ b/.github/scripts/retry.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Time in seconds to backoff after the initial attempt. INITIAL_BACKOFF=10 diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml new file mode 100644 index 0000000000..887c25e8ae --- /dev/null +++ b/.github/workflows/build-ci-image.yml @@ -0,0 +1,68 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Build CI Image + +on: + workflow_dispatch: # Allow manual trigger + +concurrency: + group: ${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - name: Generate Docker Metadata (Tags and Labels) + id: meta + uses: docker/metadata-action@v4 + with: + images: ghcr.io/${{ github.repository }} + flavor: | + prefix= + suffix= + latest=false + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=schedule,pattern={{date 'YYYY-MM-DD'}} + type=sha,format=short,prefix=sha- + type=sha,format=long,prefix=sha- + + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image + uses: docker/build-push-action@v3 + with: + push: ${{ github.event_name != 'pull_request' }} + context: .github/containers + platforms: linux/amd64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/get-envs.py b/.github/workflows/get-envs.py index 576cbeb5c8..4fcba6aa78 100755 --- a/.github/workflows/get-envs.py +++ b/.github/workflows/get-envs.py @@ -1,4 +1,18 @@ #!/usr/bin/env python3.8 +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import fileinput import os diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5ab093f19..889da3d7df 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,9 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - # - gearman + - gearman - grpc #- kafka - - libcurl - memcached - mongodb - mysql @@ -117,11 +116,12 @@ jobs: ] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -154,11 +154,12 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -181,49 +182,6 @@ jobs: path: ./**/.coverage.* retention-days: 1 - libcurl: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - # Special case packages - - name: Install libcurl-dev - run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev - - - name: Get Environments - id: get-envs - run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 - postgres: env: TOTAL_GROUPS: 2 @@ -234,6 +192,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -253,15 +213,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - - name: Install odbc driver for postgresql - run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini - name: Get Environments id: get-envs @@ -294,6 +245,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -316,7 +269,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -349,6 +301,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -366,7 +320,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -399,6 +352,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -418,7 +373,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -451,6 +405,8 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -468,7 +424,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -501,6 +456,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -519,7 +476,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -624,6 +580,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -641,7 +599,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -674,6 +631,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -693,7 +652,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -726,6 +684,8 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest timeout-minutes: 30 services: @@ -746,7 +706,6 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -769,51 +728,52 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # gearman: - # env: - # TOTAL_GROUPS: 1 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # gearman: - # image: artefactual/gearmand - # ports: - # - 4730:4730 - # # Set health checks to wait until gearman has started - # options: >- - # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + gearman: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/python-agent-ci:latest + timeout-minutes: 30 + + services: + gearman: + image: artefactual/gearmand + ports: + - 4730:4730 + # Set health checks to wait until gearman has started + options: >- + --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index 5c3961b026..df95db0290 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - # _process_module_definition( - # "gearman.client", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_client", - # ) - # _process_module_definition( - # "gearman.connection_manager", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_connection_manager", - # ) - # _process_module_definition( - # "gearman.worker", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_worker", - # ) + _process_module_definition( + "gearman.client", + "newrelic.hooks.application_gearman", + "instrument_gearman_client", + ) + _process_module_definition( + "gearman.connection_manager", + "newrelic.hooks.application_gearman", + "instrument_gearman_connection_manager", + ) + _process_module_definition( + "gearman.worker", + "newrelic.hooks.application_gearman", + "instrument_gearman_worker", + ) _process_module_definition( "botocore.endpoint", diff --git a/tox.ini b/tox.ini index 6a625b2dbd..0703d39968 100644 --- a/tox.ini +++ b/tox.ini @@ -42,10 +42,10 @@ [tox] setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py37,py38,py310,py311}, + python-adapter_gevent-{py27,py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -54,81 +54,95 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy37}-without_extensions, + python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy,pypy37}-without_extensions, + python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy37}-without_extensions, - python-application_celery-{py37,py38,py39,py310,py311,pypy37}, + python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + gearman-application_gearman-{py27,pypy}, + python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py27,pypy}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, + python-component_tastypie-{py27,pypy}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-pypy-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-memcached01, + mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, - postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, - memcached-datastore_pylibmc-{py37}, - memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, - mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, - solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, - redis-datastore_redis-{py37,py38,pypy37}-redis03, + postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest + postgres-datastore_psycopg2cffi-{py27,pypy,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest + memcached-datastore_pylibmc-{py27,py37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, - python-external_boto3-{py37,py38,py39,py310,py311}-boto01, + solr-datastore_solrpy-{py27,pypy}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_http-{py37,py38,py39,py310,py311}, - python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, - python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, + python-external_feedparser-py27-feedparser{05,06}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py37,py38,py39,py310,py311,pypy37}, - python-external_urllib3-{py37}-urllib3{0109}, - python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_urllib3-{py27,py37,pypy}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-py27-bottle{0008,0009,0010}, + python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, + python-framework_bottle-pypy-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{py37}-Django0108, + python-framework_django-{pypy,py27}-Django0103, + python-framework_django-{pypy,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy,py27}-flask0012, + python-framework_flask-{pypy,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{py38}-Pyramid0104, - python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -136,26 +150,27 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, + python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py37,py311}-genshilatest, - python-template_mako-{py37,py310,py311}, + kafka-messagebroker_kafkapython-{pypy,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py27,py37,py311}-genshilatest + python-template_mako-{py27,py37,py310,py311} [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py27,pypy}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -187,7 +202,7 @@ deps = agent_features: beautifulsoup4 application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 - ; application_gearman: gearman<3.0.0 + application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -375,9 +390,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - libcurl: PYCURL_SSL_LIBRARY=openssl - libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib - libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado: PYCURL_SSL_LIBRARY=openssl + framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -392,7 +407,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -415,7 +430,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - ; application_gearman: tests/application_gearman + application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 4422b9579400efd819c5e55148ed534c831595e9 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 22 Jun 2023 11:28:23 -0700 Subject: [PATCH 057/525] Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting --- .github/containers/Dockerfile | 2 +- .github/containers/Makefile | 7 ++++--- .github/containers/install-python.sh | 15 +-------------- .github/workflows/build-ci-image.yml | 2 +- .github/workflows/tests.yml | 24 ++++++++++++------------ 5 files changed, 19 insertions(+), 31 deletions(-) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 06b610f492..65fe441047 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -74,7 +74,7 @@ RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ pyenv update # Install Python -ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7 pypy3.7" +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7" COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh COPY ./requirements.txt /requirements.txt RUN /tmp/install-python.sh && \ diff --git a/.github/containers/Makefile b/.github/containers/Makefile index c84cb95dc9..8a72f4c458 100644 --- a/.github/containers/Makefile +++ b/.github/containers/Makefile @@ -13,7 +13,8 @@ # limitations under the License. # Repository root for mounting into container. -REPO_ROOT:=$(realpath $(dir $(realpath $(firstword $(MAKEFILE_LIST))))../../) +MAKEFILE_DIR:=$(dir $(realpath $(firstword $(MAKEFILE_LIST)))) +REPO_ROOT:=$(realpath $(MAKEFILE_DIR)../../) .PHONY: default default: test @@ -21,7 +22,7 @@ default: test .PHONY: build build: @# Perform a shortened build for testing - @docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' . -t ghcr.io/newrelic/python-agent-ci:local + @docker build --build-arg='PYTHON_VERSIONS=3.10 2.7' $(MAKEFILE_DIR) -t ghcr.io/newrelic/newrelic-python-agent-ci:local .PHONY: test test: build @@ -40,4 +41,4 @@ run: build -e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \ -e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \ -e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \ - ghcr.io/newrelic/python-agent-ci:local /bin/bash + ghcr.io/newrelic/newrelic-python-agent-ci:local /bin/bash diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh index 51ff41cd61..92184df3a9 100755 --- a/.github/containers/install-python.sh +++ b/.github/containers/install-python.sh @@ -15,8 +15,6 @@ set -e -SED=$(which gsed || which sed) - SCRIPT_DIR=$(dirname "$0") PIP_REQUIREMENTS=$(cat /requirements.txt) @@ -34,7 +32,7 @@ main() { # Find all latest pyenv supported versions for requested python versions PYENV_VERSIONS=() for v in "${PYTHON_VERSIONS[@]}"; do - LATEST=$(pyenv latest -k "$v" || get_latest_patch_version "$v") + LATEST=$(pyenv latest -k "$v" || pyenv latest -k "$v-dev") if [[ -z "$LATEST" ]]; then echo "Latest version could not be found for ${v}." 1>&2 exit 1 @@ -55,15 +53,4 @@ main() { pyenv exec pip install --upgrade $PIP_REQUIREMENTS } -get_latest_patch_version() { - pyenv install --list | # Get all python versions - $SED 's/^ *//g' | # Remove leading whitespace - grep -E "^$1" | # Find specified version by matching start of line - grep -v -- "-c-jit-latest" | # Filter out pypy JIT versions - $SED -E '/(-[a-zA-Z]+$)|(a[0-9]+)|(b[0-9]+)|(rc[0-9]+)/!{s/$/_/}' | # Append trailing _ to any non development versions to place them lower when sorted - sort -V | # Sort using version sorting - $SED 's/_$//' | # Remove any added trailing underscores to correct version names - tail -1 # Grab last result as latest version -} - main diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml index 887c25e8ae..5bd0e6f692 100644 --- a/.github/workflows/build-ci-image.yml +++ b/.github/workflows/build-ci-image.yml @@ -39,7 +39,7 @@ jobs: id: meta uses: docker/metadata-action@v4 with: - images: ghcr.io/${{ github.repository }} + images: ghcr.io/${{ github.repository }}-ci flavor: | prefix= suffix= diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 889da3d7df..fbfae2e552 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -117,7 +117,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 steps: @@ -155,7 +155,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 steps: @@ -193,7 +193,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -246,7 +246,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -302,7 +302,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -353,7 +353,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -406,7 +406,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -457,7 +457,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -581,7 +581,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -632,7 +632,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -685,7 +685,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -739,7 +739,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/python-agent-ci:latest + image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: From 4da4612ff76632c029953947e86ccb8d206ed838 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:05:18 -0700 Subject: [PATCH 058/525] Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python --- .../actions/setup-python-matrix/action.yml | 50 +++++ .github/workflows/tests.yml | 184 +++++++++++------- newrelic/config.py | 30 +-- tox.ini | 121 +++++------- 4 files changed, 230 insertions(+), 155 deletions(-) create mode 100644 .github/actions/setup-python-matrix/action.yml diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml new file mode 100644 index 0000000000..a11e2197c2 --- /dev/null +++ b/.github/actions/setup-python-matrix/action.yml @@ -0,0 +1,50 @@ +name: "setup-python-matrix" +description: "Sets up all versions of python required for matrix testing in this repo." +runs: + using: "composite" + steps: + - uses: actions/setup-python@v4 + with: + python-version: "pypy-3.7" + architecture: x64 + + # - uses: actions/setup-python@v4 + # with: + # python-version: "pypy-2.7" + # architecture: x64 + + - uses: actions/setup-python@v4 + with: + python-version: "3.7" + architecture: x64 + + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + architecture: x64 + + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + architecture: x64 + + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + architecture: x64 + + - uses: actions/setup-python@v4 + with: + python-version: "3.11" + architecture: x64 + + # - uses: actions/setup-python@v4 + # with: + # python-version: "2.7" + # architecture: x64 + + - name: Install Dependencies + shell: bash + run: | + python3.10 -m pip install -U pip + python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fbfae2e552..b5ab093f19 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,9 +36,10 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - - gearman + # - gearman - grpc #- kafka + - libcurl - memcached - mongodb - mysql @@ -116,12 +117,11 @@ jobs: ] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -154,12 +154,11 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -182,6 +181,49 @@ jobs: path: ./**/.coverage.* retention-days: 1 + libcurl: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-20.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix + + # Special case packages + - name: Install libcurl-dev + run: | + sudo apt-get update + sudo apt-get install libcurl4-openssl-dev + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 + postgres: env: TOTAL_GROUPS: 2 @@ -192,8 +234,6 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -213,6 +253,15 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix + + - name: Install odbc driver for postgresql + run: | + sudo apt-get update + sudo sudo apt-get install odbc-postgresql + sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini + sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini + sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini - name: Get Environments id: get-envs @@ -245,8 +294,6 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -269,6 +316,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -301,8 +349,6 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -320,6 +366,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -352,8 +399,6 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -373,6 +418,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -405,8 +451,6 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -424,6 +468,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -456,8 +501,6 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -476,6 +519,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -580,8 +624,6 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -599,6 +641,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -631,8 +674,6 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -652,6 +693,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -684,8 +726,6 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci timeout-minutes: 30 services: @@ -706,6 +746,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-python-matrix - name: Get Environments id: get-envs @@ -728,52 +769,51 @@ jobs: path: ./**/.coverage.* retention-days: 1 - gearman: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - container: - image: ghcr.io/${{ github.repository }}-ci - timeout-minutes: 30 - - services: - gearman: - image: artefactual/gearmand - ports: - - 4730:4730 - # Set health checks to wait until gearman has started - options: >- - --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@v3 - - - name: Get Environments - id: get-envs - run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 + # gearman: + # env: + # TOTAL_GROUPS: 1 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1] + + # runs-on: ubuntu-20.04 + # timeout-minutes: 30 + + # services: + # gearman: + # image: artefactual/gearmand + # ports: + # - 4730:4730 + # # Set health checks to wait until gearman has started + # options: >- + # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + # --health-interval 10s + # --health-timeout 5s + # --health-retries 5 + + # steps: + # - uses: actions/checkout@v3 + # - uses: ./.github/actions/setup-python-matrix + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index df95db0290..5c3961b026 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - _process_module_definition( - "gearman.client", - "newrelic.hooks.application_gearman", - "instrument_gearman_client", - ) - _process_module_definition( - "gearman.connection_manager", - "newrelic.hooks.application_gearman", - "instrument_gearman_connection_manager", - ) - _process_module_definition( - "gearman.worker", - "newrelic.hooks.application_gearman", - "instrument_gearman_worker", - ) + # _process_module_definition( + # "gearman.client", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_client", + # ) + # _process_module_definition( + # "gearman.connection_manager", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_connection_manager", + # ) + # _process_module_definition( + # "gearman.worker", + # "newrelic.hooks.application_gearman", + # "instrument_gearman_worker", + # ) _process_module_definition( "botocore.endpoint", diff --git a/tox.ini b/tox.ini index 0703d39968..6a625b2dbd 100644 --- a/tox.ini +++ b/tox.ini @@ -42,10 +42,10 @@ [tox] setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py27,py37,py38,py310,py311}, + python-adapter_gevent-{py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -54,95 +54,81 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy,pypy37}-without_extensions, - python-agent_streaming-py27-grpc0125-{with,without}_extensions, + python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy37}-without_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy,pypy37}-without_extensions, - python-application_celery-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - gearman-application_gearman-{py27,pypy}, - python-component_djangorestframework-py27-djangorestframework0300, + python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy37}-without_extensions, + python-application_celery-{py37,py38,py39,py310,py311,pypy37}, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, - python-component_flask_rest-{py27,pypy}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, - python-component_tastypie-{py27,pypy}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-cross_agent-pypy-without_extensions, + python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{pypy,py27,py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-memcached01, - mysql-datastore_mysql-mysql080023-py27, + memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest - postgres-datastore_psycopg2cffi-{py27,pypy,py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest - memcached-datastore_pylibmc-{py27,py37}, - memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy,pypy37}-pymongo04, - mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - redis-datastore_redis-{py27,py37,py38,pypy,pypy37}-redis03, + postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, + postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, + memcached-datastore_pylibmc-{py37}, + memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, + mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, + solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, + redis-datastore_redis-{py37,py38,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - solr-datastore_solrpy-{py27,pypy}-solrpy{00,01}, - python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, + python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, + python-external_boto3-{py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_feedparser-py27-feedparser{05,06}, - python-external_http-{py27,py37,py38,py39,py310,py311,pypy}, - python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-external_http-{py37,py38,py39,py310,py311}, + python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, + python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, - python-external_urllib3-{py27,py37,pypy}-urllib3{0109}, - python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-urllib3latest, + python-external_requests-{py37,py38,py39,py310,py311,pypy37}, + python-external_urllib3-{py37}-urllib3{0109}, + python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-py27-bottle{0008,0009,0010}, - python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, - python-framework_bottle-pypy-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{pypy,py27}-Django0103, - python-framework_django-{pypy,py27,py37}-Django0108, + python-framework_django-{py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py27,py37,py38,py39,pypy,pypy37}-falcon0103, + python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{pypy,py27}-flask0012, - python-framework_flask-{pypy,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy,pypy37}-graphql02, + python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, - grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{pypy,py27,py38}-Pyramid0104, - python-framework_pyramid-{pypy,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{py38}-Pyramid0104, + python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -150,27 +136,26 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy,pypy37}, + python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy,pypy37}-pika0.13, + libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{pypy,py27,py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py27,py37,py311}-genshilatest - python-template_mako-{py27,py37,py310,py311} + kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py37,py311}-genshilatest, + python-template_mako-{py37,py310,py311}, [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 - {py27,pypy}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -202,7 +187,7 @@ deps = agent_features: beautifulsoup4 application_celery: celery<6.0 application_celery-py{py37,37}: importlib-metadata<5.0 - application_gearman: gearman<3.0.0 + ; application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -390,9 +375,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - framework_tornado: PYCURL_SSL_LIBRARY=openssl - framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib - framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include + libcurl: PYCURL_SSL_LIBRARY=openssl + libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib + libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -407,7 +392,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -430,7 +415,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - application_gearman: tests/application_gearman + ; application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver From 658f8186832e44dd1c09fe03e3f10b534b0b201b Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:38:52 -0700 Subject: [PATCH 059/525] Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires --- .github/containers/Dockerfile | 7 ++----- tox.ini | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 65fe441047..8809d87c24 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -59,11 +59,8 @@ RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psq RUN locale-gen --no-purge en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 -# Set user to non-root -ENV HOME /home/github -RUN groupadd -g 1000 github && \ - useradd -m -d "${HOME}" -s /bin/bash -u 1000 -g 1000 github -USER 1000:1000 +# Use root user +ENV HOME /root WORKDIR "${HOME}" # Install pyenv diff --git a/tox.ini b/tox.ini index 6a625b2dbd..58f3815bac 100644 --- a/tox.ini +++ b/tox.ini @@ -40,6 +40,7 @@ ; - python-adapter_gevent-py27 [tox] +requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = python-adapter_cheroot-{py37,py38,py39,py310,py311}, From 57720fde8e24487894638e3c2861647460b1337c Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Fri, 23 Jun 2023 10:45:21 -0700 Subject: [PATCH 060/525] Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino --- .github/containers/Dockerfile | 7 +- .github/containers/requirements.txt | 2 +- .github/workflows/tests.yml | 403 ++++++++++-------- newrelic/config.py | 30 +- tests/application_gearman/test_gearman.py | 6 +- .../test_connection.py | 2 +- tests/testing_support/db_settings.py | 152 +++---- .../validators/validate_tt_collector_json.py | 2 +- tox.ini | 134 +++--- 9 files changed, 392 insertions(+), 346 deletions(-) diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 8809d87c24..3b4b0a7f87 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -38,11 +38,13 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ libssl-dev \ locales \ make \ + odbc-postgresql \ openssl \ python2-dev \ python3-dev \ python3-pip \ - odbc-postgresql \ + tzdata \ + unixodbc-dev \ unzip \ wget \ zip \ @@ -58,6 +60,9 @@ RUN sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psq # Set the locale RUN locale-gen --no-purge en_US.UTF-8 ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 +ENV TZ="Etc/UTC" +RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata # Use root user ENV HOME /root diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt index 54f3c39d00..27fa6624b1 100644 --- a/.github/containers/requirements.txt +++ b/.github/containers/requirements.txt @@ -1,5 +1,5 @@ pip setuptools wheel -virtualenv<20.22.1 +virtualenv<20.22.0 tox \ No newline at end of file diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b5ab093f19..f8e5182434 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,10 +36,9 @@ jobs: - python - elasticsearchserver07 - elasticsearchserver08 - # - gearman + - gearman - grpc #- kafka - - libcurl - memcached - mongodb - mysql @@ -61,6 +60,7 @@ jobs: steps: - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 with: python-version: "3.10" @@ -117,48 +117,19 @@ jobs: ] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Get Environments - id: get-envs + - name: Fetch git tags run: | - echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - env: - GROUP_NUMBER: ${{ matrix.group-number }} - - - name: Test - run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - env: - TOX_PARALLEL_NO_SPINNER: 1 - PY_COLORS: 0 - - - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 - with: - name: coverage-${{ github.job }}-${{ strategy.job-index }} - path: ./**/.coverage.* - retention-days: 1 - - grpc: - env: - TOTAL_GROUPS: 1 - - strategy: - fail-fast: false - matrix: - group-number: [1] - - runs-on: ubuntu-20.04 - timeout-minutes: 30 - - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -169,7 +140,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -181,7 +152,7 @@ jobs: path: ./**/.coverage.* retention-days: 1 - libcurl: + grpc: env: TOTAL_GROUPS: 1 @@ -191,17 +162,19 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - # Special case packages - - name: Install libcurl-dev + - name: Fetch git tags run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -212,7 +185,7 @@ jobs: - name: Test run: | - tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + tox -vv -e ${{ steps.get-envs.outputs.envs }} env: TOX_PARALLEL_NO_SPINNER: 1 PY_COLORS: 0 @@ -234,6 +207,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -253,15 +230,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix - - name: Install odbc driver for postgresql + - name: Fetch git tags run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -294,6 +267,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -316,7 +293,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -349,6 +330,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -366,7 +351,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -399,6 +388,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -418,7 +411,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -451,6 +448,10 @@ jobs: group-number: [1, 2] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -468,7 +469,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -501,6 +506,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -519,7 +528,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -542,77 +555,85 @@ jobs: path: ./**/.coverage.* retention-days: 1 - #kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + # kafka: + # env: + # TOTAL_GROUPS: 4 + + # strategy: + # fail-fast: false + # matrix: + # group-number: [1, 2, 3, 4] + + # runs-on: ubuntu-20.04 + # container: + # image: ghcr.io/${{ github.repository }}-ci:latest + # options: >- + # --add-host=host.docker.internal:host-gateway + # timeout-minutes: 30 + + # services: + # zookeeper: + # image: bitnami/zookeeper:3.7 + # env: + # ALLOW_ANONYMOUS_LOGIN: yes + + # ports: + # - 2181:2181 + + # kafka: + # image: bitnami/kafka:3.2 + # ports: + # - 8080:8080 + # - 8081:8081 + # env: + # ALLOW_PLAINTEXT_LISTENER: yes + # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 + # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, + # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT + # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 + + # steps: + # - uses: actions/checkout@v3 + + # - name: Fetch git tags + # run: | + # git config --global --add safe.directory "$GITHUB_WORKSPACE" + # git fetch --tags origin + + # # Special case packages + # - name: Install librdkafka-dev + # run: | + # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name + # sudo apt-get update + # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime + # sudo apt-get install -y wget gnupg2 software-properties-common + # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - + # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" + # sudo apt-get update + # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) + + # - name: Get Environments + # id: get-envs + # run: | + # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + # env: + # GROUP_NUMBER: ${{ matrix.group-number }} + + # - name: Test + # run: | + # tox -vv -e ${{ steps.get-envs.outputs.envs }} + # env: + # TOX_PARALLEL_NO_SPINNER: 1 + # PY_COLORS: 0 + + # - name: Upload Coverage Artifacts + # uses: actions/upload-artifact@v3 + # with: + # name: coverage-${{ github.job }}-${{ strategy.job-index }} + # path: ./**/.coverage.* + # retention-days: 1 mongodb: env: @@ -624,6 +645,10 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: @@ -641,7 +666,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -674,10 +703,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es07: + elasticsearch: image: elasticsearch:7.17.8 env: "discovery.type": "single-node" @@ -693,7 +726,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -726,10 +763,14 @@ jobs: group-number: [1] runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 services: - es08: + elasticsearch: image: elasticsearch:8.6.0 env: "xpack.security.enabled": "false" @@ -746,7 +787,11 @@ jobs: steps: - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - name: Get Environments id: get-envs @@ -769,51 +814,59 @@ jobs: path: ./**/.coverage.* retention-days: 1 - # gearman: - # env: - # TOTAL_GROUPS: 1 + gearman: + env: + TOTAL_GROUPS: 1 - # strategy: - # fail-fast: false - # matrix: - # group-number: [1] + strategy: + fail-fast: false + matrix: + group-number: [1] - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 + runs-on: ubuntu-20.04 + container: + image: ghcr.io/${{ github.repository }}-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 - # services: - # gearman: - # image: artefactual/gearmand - # ports: - # - 4730:4730 - # # Set health checks to wait until gearman has started - # options: >- - # --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 + services: + gearman: + image: artefactual/gearmand + ports: + - 8080:4730 + # Set health checks to wait until gearman has started + options: >- + --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-interval 10s + --health-timeout 5s + --health-retries 5 - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix + steps: + - uses: actions/checkout@v3 - # - name: Get Environments - # id: get-envs - # run: | - # echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 diff --git a/newrelic/config.py b/newrelic/config.py index 5c3961b026..df95db0290 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -3029,21 +3029,21 @@ def _process_module_builtin_defaults(): _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") - # _process_module_definition( - # "gearman.client", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_client", - # ) - # _process_module_definition( - # "gearman.connection_manager", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_connection_manager", - # ) - # _process_module_definition( - # "gearman.worker", - # "newrelic.hooks.application_gearman", - # "instrument_gearman_worker", - # ) + _process_module_definition( + "gearman.client", + "newrelic.hooks.application_gearman", + "instrument_gearman_client", + ) + _process_module_definition( + "gearman.connection_manager", + "newrelic.hooks.application_gearman", + "instrument_gearman_connection_manager", + ) + _process_module_definition( + "gearman.worker", + "newrelic.hooks.application_gearman", + "instrument_gearman_worker", + ) _process_module_definition( "botocore.endpoint", diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index 7ddc13fdc3..5dda4ef47e 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -20,14 +20,16 @@ import gearman from newrelic.api.background_task import background_task +from testing_support.db_settings import gearman_settings worker_thread = None worker_event = threading.Event() gm_client = None -GEARMAND_HOST = os.environ.get("GEARMAND_PORT_4730_TCP_ADDR", "localhost") -GEARMAND_PORT = os.environ.get("GEARMAND_PORT_4730_TCP_PORT", "4730") +GEARMAND_SETTINGS = gearman_settings()[0] +GEARMAND_HOST = GEARMAND_SETTINGS["host"] +GEARMAND_PORT = GEARMAND_SETTINGS["port"] GEARMAND_ADDR = "%s:%s" % (GEARMAND_HOST, GEARMAND_PORT) diff --git a/tests/datastore_elasticsearch/test_connection.py b/tests/datastore_elasticsearch/test_connection.py index 2e888af9b5..9e8f17b4c1 100644 --- a/tests/datastore_elasticsearch/test_connection.py +++ b/tests/datastore_elasticsearch/test_connection.py @@ -36,7 +36,7 @@ def test_connection_default(): else: conn = Connection(**HOST) - assert conn._nr_host_port == ("localhost", ES_SETTINGS["port"]) + assert conn._nr_host_port == (ES_SETTINGS["host"], ES_SETTINGS["port"]) @SKIP_IF_V7 diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index c7c35935f8..bda3180622 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -29,25 +29,15 @@ def postgresql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "postgres" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 5432 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "localhost", - "port": base_port + instance_num, + "user": "postgres", + "password": "postgres", + "name": "postgres", + "host": host, + "port": 8080 + instance_num, "table_name": "postgres_table_" + str(os.getpid()), } for instance_num in range(instances) @@ -66,25 +56,15 @@ def mysql_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - - user = password = db = "python_agent" - base_port = 8080 - else: - instances = 1 - - user = db = USER - password = "" - base_port = 3306 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 1 settings = [ { - "user": user, - "password": password, - "name": db, - "host": "127.0.0.1", - "port": base_port + instance_num, + "user": "python_agent", + "password": "python_agent", + "name": "python_agent", + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -103,17 +83,12 @@ def redis_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 6379 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] @@ -131,17 +106,12 @@ def memcached_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 11211 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -160,15 +130,10 @@ def mongodb_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 27017 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ - {"host": "127.0.0.1", "port": base_port + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} + {"host": host, "port": 8080 + instance_num, "collection": "mongodb_collection_" + str(os.getpid())} for instance_num in range(instances) ] return settings @@ -185,17 +150,12 @@ def elasticsearch_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9200 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 settings = [ { - "host": "localhost", - "port": str(base_port + instance_num), + "host": host, + "port": str(8080 + instance_num), "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -214,17 +174,12 @@ def solr_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 8983 - + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 2 settings = [ { - "host": "127.0.0.1", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, "namespace": str(os.getpid()), } for instance_num in range(instances) @@ -243,13 +198,12 @@ def rabbitmq_settings(): 2. Github Actions """ + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" instances = 1 - base_port = 5672 - settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 5672 + instance_num, } for instance_num in range(instances) ] @@ -267,17 +221,35 @@ def kafka_settings(): 2. Github Actions """ - if "GITHUB_ACTIONS" in os.environ: - instances = 2 - base_port = 8080 - else: - instances = 1 - base_port = 9092 + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 2 + settings = [ + { + "host": host, + "port": 8080 + instance_num, + } + for instance_num in range(instances) + ] + return settings + +def gearman_settings(): + """Return a list of dict of settings for connecting to kafka. + + Will return the correct settings, depending on which of the environments it + is running in. It attempts to set variables in the following order, where + later environments override earlier ones. + + 1. Local + 2. Github Actions + """ + + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "localhost" + instances = 1 settings = [ { - "host": "localhost", - "port": base_port + instance_num, + "host": host, + "port": 8080 + instance_num, } for instance_num in range(instances) ] diff --git a/tests/testing_support/validators/validate_tt_collector_json.py b/tests/testing_support/validators/validate_tt_collector_json.py index 85e3932806..28c9e93a39 100644 --- a/tests/testing_support/validators/validate_tt_collector_json.py +++ b/tests/testing_support/validators/validate_tt_collector_json.py @@ -135,7 +135,7 @@ def _check_params_and_start_time(node): if segment_name.startswith("Datastore"): for key in datastore_params: assert key in params, key - assert params[key] == datastore_params[key] + assert params[key] == datastore_params[key], "Expected %s. Got %s." % (datastore_params[key], params[key]) for key in datastore_forgone_params: assert key not in params, key diff --git a/tox.ini b/tox.ini index 58f3815bac..0ac7331298 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy37 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -43,10 +43,10 @@ requires = virtualenv<20.22.0 setupdir = {toxinidir} envlist = - python-adapter_cheroot-{py37,py38,py39,py310,py311}, + python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, python-adapter_daphne-py38-daphne{0204,0205}, - python-adapter_gevent-{py37,py38,py310,py311}, + python-adapter_gevent-{py27,py37,py38,py310,py311}, python-adapter_gunicorn-{py37,py38,py39,py310,py311}-aiohttp3-gunicornlatest, python-adapter_hypercorn-{py37,py38,py39,py310,py311}-hypercornlatest, python-adapter_hypercorn-py38-hypercorn{0010,0011,0012,0013}, @@ -55,81 +55,94 @@ envlist = python-adapter_waitress-{py37,py38,py39}-waitress010404, python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, - python-agent_features-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy37}-without_extensions, + python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, - python-agent_unittests-{py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy37}-without_extensions, - python-application_celery-{py37,py38,py39,py310,py311,pypy37}, + python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-agent_unittests-{pypy27,pypy37}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + gearman-application_gearman-{py27,pypy27}, + python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, + python-component_tastypie-{py27,pypy27}-tastypie0143, python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, - python-cross_agent-{py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, + python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, - memcached-datastore_bmemcached-{py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch07, + memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py37,py38,py39,py310,py311,pypy37}-memcached01, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, - postgres-datastore_psycopg2-{py37,py38,py39,py310,py311}-psycopg2latest, - postgres-datastore_psycopg2cffi-{py37,py38,py39,py310,py311}-psycopg2cffilatest, - postgres-datastore_pyodbc-{py37,py311}-pyodbclatest, - memcached-datastore_pylibmc-{py37}, - memcached-datastore_pymemcache-{py37,py38,py39,py310,py311,pypy37}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy37}-pymongo04, - mysql-datastore_pymysql-{py37,py38,py39,py310,py311,pypy37}, - solr-datastore_pysolr-{py37,py38,py39,py310,py311,pypy37}, - redis-datastore_redis-{py37,py38,pypy37}-redis03, + postgres-datastore_psycopg2-{py27,py37,py38,py39,py310,py311}-psycopg2latest + postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, + postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest + memcached-datastore_pylibmc-{py27,py37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py310}-aioredis01, redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, - python-datastore_sqlite-{py37,py38,py39,py310,py311,pypy37}, - python-external_boto3-{py37,py38,py39,py310,py311}-boto01, + solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, - python-external_http-{py37,py38,py39,py310,py311}, - python-external_httplib-{py37,py38,py39,py310,py311,pypy37}, - python-external_httplib2-{py37,py38,py39,py310,py311,pypy37}, + python-external_feedparser-py27-feedparser{05,06}, + python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py37,py38,py39,py310,py311,pypy37}, - python-external_urllib3-{py37}-urllib3{0109}, - python-external_urllib3-{py37,py38,py39,py310,py311,pypy37}-urllib3latest, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, - python-framework_bottle-{py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-py27-bottle{0008,0009,0010}, + python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, + python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, - python-framework_django-{py37}-Django0108, + python-framework_django-{pypy27,py27}-Django0103, + python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py37,py38,py39,pypy37}-falcon0103, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27}-flask0012, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, ; temporarily disabling flaskmaster tests python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py37,py38,py39,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql02, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, + grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, - python-framework_pyramid-{py38}-Pyramid0104, - python-framework_pyramid-{pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, + python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, @@ -137,26 +150,27 @@ envlist = python-framework_starlette-{py37,py38}-starlette{002001}, python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py37,py38,py39,py310,py311,pypy37}, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - libcurl-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, - libcurl-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py37,py38,py39,pypy37}-pika0.13, + python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, - kafka-messagebroker_confluentkafka-{py37,py38,py39,py310,py311}-confluentkafkalatest, - kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0107,0106}, + kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, + kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{py37,py38,pypy37}-kafkapythonlatest, - kafka-messagebroker_kafkapython-{py38}-kafkapython{020001,020000,0104}, - python-template_genshi-{py37,py311}-genshilatest, - python-template_mako-{py37,py310,py311}, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, + python-template_genshi-{py27,py37,py311}-genshilatest + python-template_mako-{py27,py37,py310,py311} [testenv] deps = # Base Dependencies {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py27,pypy27}: pytest==4.6.11 iniconfig coverage WebTest==2.0.35 @@ -187,8 +201,8 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-py{py37,37}: importlib-metadata<5.0 - ; application_gearman: gearman<3.0.0 + application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 component_djangorestframework-djangorestframeworklatest: Django @@ -205,7 +219,7 @@ deps = component_graphqlserver: markupsafe<2.1 component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 - component_tastypie-{py27,pypy}-tastypie0143: django<1.12 + component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie @@ -237,10 +251,9 @@ deps = datastore_redis-redislatest: redis datastore_redis-redis0400: redis<4.1 datastore_redis-redis03: redis<4.0 - datastore_redis-{py27,pypy}: rb + datastore_redis-{py27,pypy27}: rb datastore_aioredis-redislatest: redis datastore_aioredis-aioredislatest: aioredis - datastore_aioredis-aioredis01: aioredis<2 datastore_aredis-aredislatest: aredis datastore_solrpy-solrpy00: solrpy<1.0 datastore_solrpy-solrpy01: solrpy<2.0 @@ -355,7 +368,7 @@ deps = messagebroker_pika-pika0.13: pika<0.14 messagebroker_pika-pikalatest: pika messagebroker_pika: tornado<5 - messagebroker_pika-{py27,pypy}: enum34 + messagebroker_pika-{py27,pypy27}: enum34 messagebroker_confluentkafka-confluentkafkalatest: confluent-kafka messagebroker_confluentkafka-confluentkafka0108: confluent-kafka<1.9 messagebroker_confluentkafka-confluentkafka0107: confluent-kafka<1.8 @@ -376,9 +389,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - libcurl: PYCURL_SSL_LIBRARY=openssl - libcurl: LDFLAGS=-L/usr/local/opt/openssl/lib - libcurl: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado: PYCURL_SSL_LIBRARY=openssl + framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -393,7 +406,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - libcurl: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* @@ -416,7 +429,7 @@ changedir = agent_streaming: tests/agent_streaming agent_unittests: tests/agent_unittests application_celery: tests/application_celery - ; application_gearman: tests/application_gearman + application_gearman: tests/application_gearman component_djangorestframework: tests/component_djangorestframework component_flask_rest: tests/component_flask_rest component_graphqlserver: tests/component_graphqlserver @@ -489,6 +502,7 @@ source = newrelic source = newrelic/ .tox/**/site-packages/newrelic/ + /__w/**/site-packages/newrelic/ [coverage:html] directory = ${TOX_ENV_DIR-.}/htmlcov From a7dfe33e70a7bbdd3657f771451ad505b436b420 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 26 Jun 2023 12:08:27 -0700 Subject: [PATCH 061/525] Instrument Redis waitaof (#851) * Add uninstrumented command to redis * Update logic for datastore_aioredis instance info * [Mega-Linter] Apply linters fixes * Bump tests * Update defaults for aioredis port --------- Co-authored-by: TimPansino --- newrelic/hooks/datastore_aioredis.py | 19 +++++++++++-------- newrelic/hooks/datastore_redis.py | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/newrelic/hooks/datastore_aioredis.py b/newrelic/hooks/datastore_aioredis.py index 9bd5b17b0e..47c9879717 100644 --- a/newrelic/hooks/datastore_aioredis.py +++ b/newrelic/hooks/datastore_aioredis.py @@ -15,13 +15,13 @@ from newrelic.api.datastore_trace import DatastoreTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper, function_wrapper +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version_tuple from newrelic.hooks.datastore_redis import ( _redis_client_methods, _redis_multipart_commands, _redis_operation_re, ) -from newrelic.common.package_version_utils import get_package_version_tuple def _conn_attrs_to_dict(connection): @@ -39,14 +39,13 @@ def _conn_attrs_to_dict(connection): def _instance_info(kwargs): host = kwargs.get("host") or "localhost" - port_path_or_id = str(kwargs.get("port") or kwargs.get("path", 6379)) + port_path_or_id = str(kwargs.get("path") or kwargs.get("port", 6379)) db = str(kwargs.get("db") or 0) return (host, port_path_or_id, db) def _wrap_AioRedis_method_wrapper(module, instance_class_name, operation): - @function_wrapper async def _nr_wrapper_AioRedis_async_method_(wrapped, instance, args, kwargs): transaction = current_transaction() @@ -55,7 +54,7 @@ async def _nr_wrapper_AioRedis_async_method_(wrapped, instance, args, kwargs): with DatastoreTrace(product="Redis", target=None, operation=operation): return await wrapped(*args, **kwargs) - + def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): # Check for transaction and return early if found. # Method will return synchronously without executing, @@ -64,6 +63,7 @@ def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): if aioredis_version and aioredis_version < (2,): # AioRedis v1 uses a RedisBuffer instead of a real connection for queueing up pipeline commands from aioredis.commands.transaction import _RedisBuffer + if isinstance(instance._pool_or_conn, _RedisBuffer): # Method will return synchronously without executing, # it will be added to the command stack and run later. @@ -80,7 +80,6 @@ def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): # Method should be run when awaited, therefore we wrap in an async wrapper. return _nr_wrapper_AioRedis_async_method_(wrapped)(*args, **kwargs) - name = "%s.%s" % (instance_class_name, operation) wrap_function_wrapper(module, name, _nr_wrapper_AioRedis_method_) @@ -109,7 +108,9 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): # If it's not a multi part command, there's no need to trace it, so # we can return early. - if operation.split()[0] not in _redis_multipart_commands: # Set the datastore info on the DatastoreTrace containing this function call. + if ( + operation.split()[0] not in _redis_multipart_commands + ): # Set the datastore info on the DatastoreTrace containing this function call. trace = current_trace() # Find DatastoreTrace no matter how many other traces are inbetween @@ -161,7 +162,9 @@ def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): # If it's not a multi part command, there's no need to trace it, so # we can return early. - if operation.split()[0] not in _redis_multipart_commands: # Set the datastore info on the DatastoreTrace containing this function call. + if ( + operation.split()[0] not in _redis_multipart_commands + ): # Set the datastore info on the DatastoreTrace containing this function call. trace = current_trace() # Find DatastoreTrace no matter how many other traces are inbetween diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index b32c848b35..26ab2f5c79 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -394,6 +394,7 @@ "unsubscribe", "unwatch", "wait", + "waitaof", "watch", "xack", "xadd", From 33aa11199d69678b5c1f9246998898f9c3663c4e Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Mon, 26 Jun 2023 12:23:30 -0700 Subject: [PATCH 062/525] Ignore patched hooks files. (#849) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/codecov.yml b/codecov.yml index ec600226a2..61c135aba3 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,8 +1,10 @@ ignore: - "newrelic/packages/**/*" - "newrelic/packages/*" + - "newreilc/hooks/component_sentry.py" - "newrelic/hooks/adapter_meinheld.py" - "newrelic/hooks/adapter_flup.py" + - "newrelic/hooks/adapter_paste.py" - "newrelic/hooks/component_piston.py" - "newrelic/hooks/datastore_pyelasticsearch.py" - "newrelic/hooks/external_pywapi.py" @@ -13,6 +15,7 @@ ignore: - "newrelic/hooks/framework_web2py.py" - "newrelic/hooks/middleware_weberror.py" - "newrelic/hooks/framework_webpy.py" + - "newrelic/hooks/datastore_motor.py" - "newrelic/hooks/database_oursql.py" - "newrelic/hooks/database_psycopg2ct.py" - "newrelic/hooks/datastore_umemcache.py" From e707cc05032773ee7b1f039e9f4725a9b055d02c Mon Sep 17 00:00:00 2001 From: Hannah Stepanek Date: Mon, 26 Jun 2023 13:33:32 -0700 Subject: [PATCH 063/525] Fix local scoped package reporting (#837) * Include isort stdlibs for determining stdlib modules * Use isort & sys to eliminate std & builtin modules Previously, the logic would fail to identify third party modules installed within the local user socpe. This fixes that issue by skipping builtin and stdlib modules by name, instead of attempting to identify third party modules based on file paths. * Handle importlib_metadata.version being a callable * Add isort into third party notices * [Mega-Linter] Apply linters fixes * Remove Python 2.7 and pypy2 testing (#835) * Change setup-python to @v2 for py2.7 * Remove py27 and pypy testing * Fix syntax errors * Fix comma related syntax errors * Fix more issues in tox * Remove gearman test * Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs * Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting * Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python * Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires * Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino * Trigger tests --------- Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: hmstepanek Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: TimPansino Co-authored-by: Uma Annamalai --- MANIFEST.in | 1 + THIRD_PARTY_NOTICES.md | 19 +- newrelic/common/package_version_utils.py | 6 +- newrelic/core/environment.py | 51 ++- newrelic/packages/isort/LICENSE | 21 ++ newrelic/packages/isort/__init__.py | 0 newrelic/packages/isort/stdlibs/__init__.py | 2 + newrelic/packages/isort/stdlibs/all.py | 3 + newrelic/packages/isort/stdlibs/py2.py | 3 + newrelic/packages/isort/stdlibs/py27.py | 301 ++++++++++++++++++ newrelic/packages/isort/stdlibs/py3.py | 3 + newrelic/packages/isort/stdlibs/py310.py | 222 +++++++++++++ newrelic/packages/isort/stdlibs/py311.py | 222 +++++++++++++ newrelic/packages/isort/stdlibs/py36.py | 224 +++++++++++++ newrelic/packages/isort/stdlibs/py37.py | 225 +++++++++++++ newrelic/packages/isort/stdlibs/py38.py | 224 +++++++++++++ newrelic/packages/isort/stdlibs/py39.py | 224 +++++++++++++ setup.py | 2 + .../test_package_version_utils.py | 14 + 19 files changed, 1747 insertions(+), 20 deletions(-) create mode 100644 newrelic/packages/isort/LICENSE create mode 100644 newrelic/packages/isort/__init__.py create mode 100644 newrelic/packages/isort/stdlibs/__init__.py create mode 100644 newrelic/packages/isort/stdlibs/all.py create mode 100644 newrelic/packages/isort/stdlibs/py2.py create mode 100644 newrelic/packages/isort/stdlibs/py27.py create mode 100644 newrelic/packages/isort/stdlibs/py3.py create mode 100644 newrelic/packages/isort/stdlibs/py310.py create mode 100644 newrelic/packages/isort/stdlibs/py311.py create mode 100644 newrelic/packages/isort/stdlibs/py36.py create mode 100644 newrelic/packages/isort/stdlibs/py37.py create mode 100644 newrelic/packages/isort/stdlibs/py38.py create mode 100644 newrelic/packages/isort/stdlibs/py39.py diff --git a/MANIFEST.in b/MANIFEST.in index 0a75ce7520..bf746435ce 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,4 @@ include newrelic/common/cacert.pem include newrelic/packages/wrapt/LICENSE include newrelic/packages/wrapt/README include newrelic/packages/urllib3/LICENSE.txt +include newrelic/packages/isort/LICENSE diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 3662484f6b..a1dd7e07d9 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -14,7 +14,16 @@ Copyright (c) Django Software Foundation and individual contributors. Distributed under the following license(s): - * [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) +* [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) + + +## [isort](https://pypi.org/project/isort) + +Copyright (c) 2013 Timothy Edmund Crosley + +Distributed under the following license(s): + +* [The MIT License](http://opensource.org/licenses/MIT) ## [six](https://pypi.org/project/six) @@ -23,7 +32,7 @@ Copyright (c) 2010-2013 Benjamin Peterson Distributed under the following license(s): - * [The MIT License](http://opensource.org/licenses/MIT) +* [The MIT License](http://opensource.org/licenses/MIT) ## [time.monotonic](newrelic/common/_monotonic.c) @@ -32,7 +41,7 @@ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, Distributed under the following license(s): - * [Python Software Foundation](https://docs.python.org/3/license.html) +* [Python Software Foundation](https://docs.python.org/3/license.html) ## [urllib3](https://pypi.org/project/urllib3) @@ -41,7 +50,7 @@ Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) Distributed under the following license(s): - * [The MIT License](http://opensource.org/licenses/MIT) +* [The MIT License](http://opensource.org/licenses/MIT) ## [wrapt](https://pypi.org/project/wrapt) @@ -51,5 +60,5 @@ All rights reserved. Distributed under the following license(s): - * [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause) +* [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause) diff --git a/newrelic/common/package_version_utils.py b/newrelic/common/package_version_utils.py index 13b8168780..f3d334e2a6 100644 --- a/newrelic/common/package_version_utils.py +++ b/newrelic/common/package_version_utils.py @@ -73,6 +73,10 @@ def _get_package_version(name): for attr in VERSION_ATTRS: try: version = getattr(module, attr, None) + # In certain cases like importlib_metadata.version, version is a callable + # function. + if callable(version): + continue # Cast any version specified as a list into a tuple. version = tuple(version) if isinstance(version, list) else version if version not in NULL_VERSIONS: @@ -95,4 +99,4 @@ def _get_package_version(name): if version not in NULL_VERSIONS: return version except Exception: - pass \ No newline at end of file + pass diff --git a/newrelic/core/environment.py b/newrelic/core/environment.py index 1306816efd..66efe61126 100644 --- a/newrelic/core/environment.py +++ b/newrelic/core/environment.py @@ -17,10 +17,10 @@ """ +import logging import os import platform import sys -import sysconfig import newrelic from newrelic.common.package_version_utils import get_package_version @@ -29,12 +29,15 @@ physical_processor_count, total_physical_memory, ) +from newrelic.packages.isort import stdlibs as isort_stdlibs try: import newrelic.core._thread_utilization except ImportError: pass +_logger = logging.getLogger(__name__) + def environment_settings(): """Returns an array of arrays of environment settings""" @@ -195,8 +198,7 @@ def environment_settings(): env.extend(dispatcher) # Module information. - purelib = sysconfig.get_path("purelib") - platlib = sysconfig.get_path("platlib") + stdlib_builtin_module_names = _get_stdlib_builtin_module_names() plugins = [] @@ -208,29 +210,50 @@ def environment_settings(): # list for name, module in sys.modules.copy().items(): # Exclude lib.sub_paths as independent modules except for newrelic.hooks. - if "." in name and not name.startswith("newrelic.hooks."): + nr_hook = name.startswith("newrelic.hooks.") + if "." in name and not nr_hook or name.startswith("_"): continue + # If the module isn't actually loaded (such as failed relative imports # in Python 2.7), the module will be None and should not be reported. if not module: continue + # Exclude standard library/built-in modules. - # Third-party modules can be installed in either purelib or platlib directories. - # See https://docs.python.org/3/library/sysconfig.html#installation-paths. - if ( - not hasattr(module, "__file__") - or not module.__file__ - or not module.__file__.startswith(purelib) - or not module.__file__.startswith(platlib) - ): + if name in stdlib_builtin_module_names: continue try: version = get_package_version(name) - plugins.append("%s (%s)" % (name, version)) except Exception: - plugins.append(name) + version = None + + # If it has no version it's likely not a real package so don't report it unless + # it's a new relic hook. + if version or nr_hook: + plugins.append("%s (%s)" % (name, version)) env.append(("Plugin List", plugins)) return env + + +def _get_stdlib_builtin_module_names(): + builtins = set(sys.builtin_module_names) + # Since sys.stdlib_module_names is not available in versions of python below 3.10, + # use isort's hardcoded stdlibs instead. + python_version = sys.version_info[0:2] + if python_version < (3,): + stdlibs = isort_stdlibs.py27.stdlib + elif (3, 7) <= python_version < (3, 8): + stdlibs = isort_stdlibs.py37.stdlib + elif python_version < (3, 9): + stdlibs = isort_stdlibs.py38.stdlib + elif python_version < (3, 10): + stdlibs = isort_stdlibs.py39.stdlib + elif python_version >= (3, 10): + stdlibs = sys.stdlib_module_names + else: + _logger.warn("Unsupported Python version. Unable to determine stdlibs.") + return builtins + return builtins | stdlibs diff --git a/newrelic/packages/isort/LICENSE b/newrelic/packages/isort/LICENSE new file mode 100644 index 0000000000..b5083a50d8 --- /dev/null +++ b/newrelic/packages/isort/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Timothy Edmund Crosley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/newrelic/packages/isort/__init__.py b/newrelic/packages/isort/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/newrelic/packages/isort/stdlibs/__init__.py b/newrelic/packages/isort/stdlibs/__init__.py new file mode 100644 index 0000000000..3394a7eda8 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/__init__.py @@ -0,0 +1,2 @@ +from . import all as _all +from . import py2, py3, py27, py36, py37, py38, py39, py310, py311 diff --git a/newrelic/packages/isort/stdlibs/all.py b/newrelic/packages/isort/stdlibs/all.py new file mode 100644 index 0000000000..08a365e19e --- /dev/null +++ b/newrelic/packages/isort/stdlibs/all.py @@ -0,0 +1,3 @@ +from . import py2, py3 + +stdlib = py2.stdlib | py3.stdlib diff --git a/newrelic/packages/isort/stdlibs/py2.py b/newrelic/packages/isort/stdlibs/py2.py new file mode 100644 index 0000000000..74af019e49 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py2.py @@ -0,0 +1,3 @@ +from . import py27 + +stdlib = py27.stdlib diff --git a/newrelic/packages/isort/stdlibs/py27.py b/newrelic/packages/isort/stdlibs/py27.py new file mode 100644 index 0000000000..a9bc99d0c7 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py27.py @@ -0,0 +1,301 @@ +""" +File contains the standard library of Python 2.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "AL", + "BaseHTTPServer", + "Bastion", + "CGIHTTPServer", + "Carbon", + "ColorPicker", + "ConfigParser", + "Cookie", + "DEVICE", + "DocXMLRPCServer", + "EasyDialogs", + "FL", + "FrameWork", + "GL", + "HTMLParser", + "MacOS", + "MimeWriter", + "MiniAEFrame", + "Nav", + "PixMapWrapper", + "Queue", + "SUNAUDIODEV", + "ScrolledText", + "SimpleHTTPServer", + "SimpleXMLRPCServer", + "SocketServer", + "StringIO", + "Tix", + "Tkinter", + "UserDict", + "UserList", + "UserString", + "W", + "__builtin__", + "_ast", + "_winreg", + "abc", + "aepack", + "aetools", + "aetypes", + "aifc", + "al", + "anydbm", + "applesingle", + "argparse", + "array", + "ast", + "asynchat", + "asyncore", + "atexit", + "audioop", + "autoGIL", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "bsddb", + "buildtools", + "bz2", + "cPickle", + "cProfile", + "cStringIO", + "calendar", + "cd", + "cfmfile", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "commands", + "compileall", + "compiler", + "contextlib", + "cookielib", + "copy", + "copy_reg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbhash", + "dbm", + "decimal", + "difflib", + "dircache", + "dis", + "distutils", + "dl", + "doctest", + "dumbdbm", + "dummy_thread", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "errno", + "exceptions", + "fcntl", + "filecmp", + "fileinput", + "findertools", + "fl", + "flp", + "fm", + "fnmatch", + "formatter", + "fpectl", + "fpformat", + "fractions", + "ftplib", + "functools", + "future_builtins", + "gc", + "gdbm", + "gensuitemodule", + "getopt", + "getpass", + "gettext", + "gl", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "hotshot", + "htmlentitydefs", + "htmllib", + "httplib", + "ic", + "icopen", + "imageop", + "imaplib", + "imgfile", + "imghdr", + "imp", + "importlib", + "imputil", + "inspect", + "io", + "itertools", + "jpeg", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "macerrors", + "macostools", + "macpath", + "macresource", + "mailbox", + "mailcap", + "marshal", + "math", + "md5", + "mhlib", + "mimetools", + "mimetypes", + "mimify", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multifile", + "multiprocessing", + "mutex", + "netrc", + "new", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "popen2", + "poplib", + "posix", + "posixfile", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "quopri", + "random", + "re", + "readline", + "resource", + "rexec", + "rfc822", + "rlcompleter", + "robotparser", + "runpy", + "sched", + "select", + "sets", + "sgmllib", + "sha", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statvfs", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "sunaudiodev", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "thread", + "threading", + "time", + "timeit", + "token", + "tokenize", + "trace", + "traceback", + "ttk", + "tty", + "turtle", + "types", + "unicodedata", + "unittest", + "urllib", + "urllib2", + "urlparse", + "user", + "uu", + "uuid", + "videoreader", + "warnings", + "wave", + "weakref", + "webbrowser", + "whichdb", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpclib", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py3.py b/newrelic/packages/isort/stdlibs/py3.py new file mode 100644 index 0000000000..9882543853 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py3.py @@ -0,0 +1,3 @@ +from . import py36, py37, py38, py39, py310, py311 + +stdlib = py36.stdlib | py37.stdlib | py38.stdlib | py39.stdlib | py310.stdlib | py311.stdlib diff --git a/newrelic/packages/isort/stdlibs/py310.py b/newrelic/packages/isort/stdlibs/py310.py new file mode 100644 index 0000000000..f45cf50a38 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py310.py @@ -0,0 +1,222 @@ +""" +File contains the standard library of Python 3.10. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/newrelic/packages/isort/stdlibs/py311.py b/newrelic/packages/isort/stdlibs/py311.py new file mode 100644 index 0000000000..6fa42e9952 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py311.py @@ -0,0 +1,222 @@ +""" +File contains the standard library of Python 3.11. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/newrelic/packages/isort/stdlibs/py36.py b/newrelic/packages/isort/stdlibs/py36.py new file mode 100644 index 0000000000..59ebd24cb4 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py36.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.6. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fpectl", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py37.py b/newrelic/packages/isort/stdlibs/py37.py new file mode 100644 index 0000000000..e0ad1228a8 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py37.py @@ -0,0 +1,225 @@ +""" +File contains the standard library of Python 3.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py38.py b/newrelic/packages/isort/stdlibs/py38.py new file mode 100644 index 0000000000..3d89fd26b3 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py38.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.8. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py39.py b/newrelic/packages/isort/stdlibs/py39.py new file mode 100644 index 0000000000..4b7dd59543 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py39.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.9. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/setup.py b/setup.py index 044125a23e..2b1e5191e4 100644 --- a/setup.py +++ b/setup.py @@ -102,6 +102,8 @@ def build_extension(self, ext): "newrelic.hooks", "newrelic.network", "newrelic/packages", + "newrelic/packages/isort", + "newrelic/packages/isort/stdlibs", "newrelic/packages/urllib3", "newrelic/packages/urllib3/util", "newrelic/packages/urllib3/contrib", diff --git a/tests/agent_unittests/test_package_version_utils.py b/tests/agent_unittests/test_package_version_utils.py index d80714d778..435d74947f 100644 --- a/tests/agent_unittests/test_package_version_utils.py +++ b/tests/agent_unittests/test_package_version_utils.py @@ -58,6 +58,20 @@ def test_get_package_version(attr, value, expected_value): delattr(pytest, attr) +def test_skips_version_callables(): + # There is no file/module here, so we monkeypatch + # pytest instead for our purposes + setattr(pytest, "version", lambda x: "1.2.3.4") + setattr(pytest, "version_tuple", [3, 1, "0b2"]) + + version = get_package_version("pytest") + + assert version == "3.1.0b2" + + delattr(pytest, "version") + delattr(pytest, "version_tuple") + + @pytest.mark.parametrize( "attr,value,expected_value", ( From ab590a2074c3e3e148ca077bf1ec24d6a09b2d89 Mon Sep 17 00:00:00 2001 From: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Date: Mon, 26 Jun 2023 17:04:21 -0700 Subject: [PATCH 064/525] MSSQL Testing (#852) * For mysql tests into mssql * Add tox envs for mssql * Add mssql DB settings * Add correct MSSQL tests * Add mssql to GHA * Add MSSQL libs to CI image * Pin to dev CI image sha * Swap SQLServer container image * Fix healthcheck * Put MSSQL image back * Drop pypy37 tests * Unpin dev image sha --- .github/containers/Dockerfile | 3 + .github/workflows/tests.yml | 90 +++++++++++++++--- tests/datastore_pymssql/conftest.py | 36 +++++++ tests/datastore_pymssql/test_database.py | 115 +++++++++++++++++++++++ tests/testing_support/db_settings.py | 28 +++++- tox.ini | 3 + 6 files changed, 261 insertions(+), 14 deletions(-) create mode 100644 tests/datastore_pymssql/conftest.py create mode 100644 tests/datastore_pymssql/test_database.py diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 3b4b0a7f87..260c01d89f 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -23,12 +23,15 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ build-essential \ curl \ expat \ + freetds-common \ + freetds-dev \ gcc \ git \ libbz2-dev \ libcurl4-openssl-dev \ libffi-dev \ libgmp-dev \ + libkrb5-dev \ liblzma-dev \ libmpfr-dev \ libncurses-dev \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f8e5182434..f4cb8b8223 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -41,6 +41,7 @@ jobs: #- kafka - memcached - mongodb + - mssql - mysql - postgres - rabbitmq @@ -118,7 +119,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -163,7 +164,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -208,7 +209,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -257,6 +258,69 @@ jobs: path: ./**/.coverage.* retention-days: 1 + mssql: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-20.04 + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 + + services: + mssql: + image: mcr.microsoft.com/azure-sql-edge:latest + env: + MSSQL_USER: python_agent + MSSQL_PASSWORD: python_agent + MSSQL_SA_PASSWORD: "python_agent#1234" + ACCEPT_EULA: "Y" + ports: + - 8080:1433 + - 8081:1433 + # Set health checks to wait until mysql has started + options: >- + --health-cmd "/opt/mssql-tools/bin/sqlcmd -U SA -P $MSSQL_SA_PASSWORD -Q 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@v3 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@v3 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 + mysql: env: TOTAL_GROUPS: 2 @@ -268,7 +332,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -331,7 +395,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -389,7 +453,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -449,7 +513,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -507,7 +571,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -566,7 +630,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/${{ github.repository }}-ci:latest + # image: ghcr.io/newrelic/newrelic-python-agent-ci:latest # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -646,7 +710,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -704,7 +768,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -764,7 +828,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -825,7 +889,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/${{ github.repository }}-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 diff --git a/tests/datastore_pymssql/conftest.py b/tests/datastore_pymssql/conftest.py new file mode 100644 index 0000000000..a6584cdffe --- /dev/null +++ b/tests/datastore_pymssql/conftest.py @@ -0,0 +1,36 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from testing_support.fixtures import ( + collector_agent_registration_fixture, + collector_available_fixture, +) # noqa: F401; pylint: disable=W0611 + + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_pymssql)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pymssql/test_database.py b/tests/datastore_pymssql/test_database.py new file mode 100644 index 0000000000..bdbf75c15f --- /dev/null +++ b/tests/datastore_pymssql/test_database.py @@ -0,0 +1,115 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pymssql + +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_database_trace_inputs import validate_database_trace_inputs + +from testing_support.db_settings import mssql_settings + +from newrelic.api.background_task import background_task + +DB_SETTINGS = mssql_settings()[0] +TABLE_NAME = "datastore_pymssql_" + DB_SETTINGS["namespace"] +PROCEDURE_NAME = "hello_" + DB_SETTINGS["namespace"] + + +def execute_db_calls_with_cursor(cursor): + cursor.execute("""drop table if exists %s""" % TABLE_NAME) + + cursor.execute("""create table %s """ % TABLE_NAME + """(a integer, b real, c text)""") + + cursor.executemany( + """insert into %s """ % TABLE_NAME + """values (%s, %s, %s)""", + [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], + ) + + cursor.execute("""select * from %s""" % TABLE_NAME) + + for row in cursor: + pass + + cursor.execute("""update %s""" % TABLE_NAME + """ set a=%s, b=%s, """ """c=%s where a=%s""", (4, 4.0, "4.0", 1)) + + cursor.execute("""delete from %s where a=2""" % TABLE_NAME) + cursor.execute("""drop procedure if exists %s""" % PROCEDURE_NAME) + cursor.execute( + """CREATE PROCEDURE %s AS + BEGIN + SELECT 'Hello World!'; + END""" + % PROCEDURE_NAME + ) + + cursor.callproc(PROCEDURE_NAME) + + +_test_scoped_metrics = [ + ("Function/pymssql._pymssql:connect", 1), + ("Datastore/statement/MSSQL/%s/select" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/insert" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/update" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/delete" % TABLE_NAME, 1), + ("Datastore/operation/MSSQL/drop", 2), + ("Datastore/operation/MSSQL/create", 2), + ("Datastore/statement/MSSQL/%s/call" % PROCEDURE_NAME, 1), + ("Datastore/operation/MSSQL/commit", 2), + ("Datastore/operation/MSSQL/rollback", 1), +] + +_test_rollup_metrics = [ + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MSSQL/all", 13), + ("Datastore/MSSQL/allOther", 13), + ("Datastore/statement/MSSQL/%s/select" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/insert" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/update" % TABLE_NAME, 1), + ("Datastore/statement/MSSQL/%s/delete" % TABLE_NAME, 1), + ("Datastore/operation/MSSQL/select", 1), + ("Datastore/operation/MSSQL/insert", 1), + ("Datastore/operation/MSSQL/update", 1), + ("Datastore/operation/MSSQL/delete", 1), + ("Datastore/statement/MSSQL/%s/call" % PROCEDURE_NAME, 1), + ("Datastore/operation/MSSQL/call", 1), + ("Datastore/operation/MSSQL/drop", 2), + ("Datastore/operation/MSSQL/create", 2), + ("Datastore/operation/MSSQL/commit", 2), + ("Datastore/operation/MSSQL/rollback", 1), +] + + +@validate_transaction_metrics( + "test_database:test_execute_via_cursor_context_manager", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_context_manager(): + connection = pymssql.connect( + user=DB_SETTINGS["user"], password=DB_SETTINGS["password"], host=DB_SETTINGS["host"], port=DB_SETTINGS["port"] + ) + + with connection: + cursor = connection.cursor() + + with cursor: + execute_db_calls_with_cursor(cursor) + + connection.commit() + connection.rollback() + connection.commit() diff --git a/tests/testing_support/db_settings.py b/tests/testing_support/db_settings.py index bda3180622..ef9a3419c1 100644 --- a/tests/testing_support/db_settings.py +++ b/tests/testing_support/db_settings.py @@ -46,7 +46,7 @@ def postgresql_settings(): def mysql_settings(): - """Return a list of dict of settings for connecting to postgresql. + """Return a list of dict of settings for connecting to MySQL. Will return the correct settings, depending on which of the environments it is running in. It attempts to set variables in the following order, where @@ -72,6 +72,32 @@ def mysql_settings(): return settings +def mssql_settings(): + """Return a list of dict of settings for connecting to MS SQL. + + Will return the correct settings, depending on which of the environments it + is running in. It attempts to set variables in the following order, where + later environments override earlier ones. + + 1. Local + 2. Github Actions + """ + + host = "host.docker.internal" if "GITHUB_ACTIONS" in os.environ else "127.0.0.1" + instances = 1 + settings = [ + { + "user": "SA", + "password": "python_agent#1234", + "host": host, + "port": 8080 + instance_num, + "namespace": str(os.getpid()), + } + for instance_num in range(instances) + ] + return settings + + def redis_settings(): """Return a list of dict of settings for connecting to redis. diff --git a/tox.ini b/tox.ini index 0ac7331298..94722ce8bf 100644 --- a/tox.ini +++ b/tox.ini @@ -89,6 +89,7 @@ envlist = memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mssql-datastore_pymssql-{py37,py38,py39,py310,py311}, mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, @@ -246,6 +247,7 @@ deps = datastore_pymemcache: pymemcache datastore_pymongo-pymongo03: pymongo<4.0 datastore_pymongo-pymongo04: pymongo<5.0 + datastore_pymssql: pymssql datastore_pymysql: PyMySQL<0.11 datastore_pysolr: pysolr<4.0 datastore_redis-redislatest: redis @@ -448,6 +450,7 @@ changedir = datastore_pylibmc: tests/datastore_pylibmc datastore_pymemcache: tests/datastore_pymemcache datastore_pymongo: tests/datastore_pymongo + datastore_pymssql: tests/datastore_pymssql datastore_pymysql: tests/datastore_pymysql datastore_pysolr: tests/datastore_pysolr datastore_redis: tests/datastore_redis From db075239a656b8cc757609776acf05226783caed Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:02:51 -0700 Subject: [PATCH 065/525] Exclude command line functionality from test coverage (#855) --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codecov.yml b/codecov.yml index 61c135aba3..8ed9c30200 100644 --- a/codecov.yml +++ b/codecov.yml @@ -22,3 +22,5 @@ ignore: # Temporarily disable kafka - "newrelic/hooks/messagebroker_kafkapython.py" - "newrelic/hooks/messagebroker_confluentkafka.py" + - "newrelic/admin/*" + - "newrelic/console.py" From c2fd5e32aa9fa6c981ed66060718d4bb4401bfed Mon Sep 17 00:00:00 2001 From: Ahmed Helil Date: Tue, 27 Jun 2023 23:37:44 +0200 Subject: [PATCH 066/525] FIX: resilient environment settings (#825) if the application uses generalimport to manage optional depedencies, it's possible that generalimport.MissingOptionalDependency is raised. In this case, we should not report the module as it is not actually loaded and is not a runtime dependency of the application. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek --- newrelic/core/environment.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/newrelic/core/environment.py b/newrelic/core/environment.py index 66efe61126..9bca085a3a 100644 --- a/newrelic/core/environment.py +++ b/newrelic/core/environment.py @@ -216,7 +216,15 @@ def environment_settings(): # If the module isn't actually loaded (such as failed relative imports # in Python 2.7), the module will be None and should not be reported. - if not module: + try: + if not module: + continue + except Exception: + # if the application uses generalimport to manage optional depedencies, + # it's possible that generalimport.MissingOptionalDependency is raised. + # In this case, we should not report the module as it is not actually loaded and + # is not a runtime dependency of the application. + # continue # Exclude standard library/built-in modules. From 9883c2b0580ce3c5e1927a4ff83fc6f72f687a7a Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:09:09 -0700 Subject: [PATCH 067/525] Replace drop_transaction logic by using transaction context manager (#832) * Replace drop_transaction call * [Mega-Linter] Apply linters fixes * Empty commit to start tests * Change logic in BG Wrappers --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- newrelic/api/background_task.py | 38 ++++++----------------------- newrelic/api/message_transaction.py | 26 +++----------------- 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/newrelic/api/background_task.py b/newrelic/api/background_task.py index a4a9e8e6a6..4cdcd8a0d4 100644 --- a/newrelic/api/background_task.py +++ b/newrelic/api/background_task.py @@ -13,19 +13,16 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.transaction import Transaction, current_transaction -from newrelic.common.async_proxy import async_proxy, TransactionContext +from newrelic.common.async_proxy import TransactionContext, async_proxy from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object class BackgroundTask(Transaction): - def __init__(self, application, name, group=None, source=None): - # Initialise the common transaction base class. super(BackgroundTask, self).__init__(application, source=source) @@ -53,7 +50,6 @@ def __init__(self, application, name, group=None, source=None): def BackgroundTaskWrapper(wrapped, application=None, name=None, group=None): - def wrapper(wrapped, instance, args, kwargs): if callable(name): if instance is not None: @@ -107,39 +103,19 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that a transaction already exists, so we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def background_task(application=None, name=None, group=None): - return functools.partial(BackgroundTaskWrapper, - application=application, name=name, group=group) + return functools.partial(BackgroundTaskWrapper, application=application, name=name, group=group) -def wrap_background_task(module, object_path, application=None, - name=None, group=None): - wrap_object(module, object_path, BackgroundTaskWrapper, - (application, name, group)) +def wrap_background_task(module, object_path, application=None, name=None, group=None): + wrap_object(module, object_path, BackgroundTaskWrapper, (application, name, group)) diff --git a/newrelic/api/message_transaction.py b/newrelic/api/message_transaction.py index 291a3897e6..54a71f6eff 100644 --- a/newrelic/api/message_transaction.py +++ b/newrelic/api/message_transaction.py @@ -13,7 +13,6 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.background_task import BackgroundTask @@ -39,7 +38,6 @@ def __init__( transport_type="AMQP", source=None, ): - name, group = self.get_transaction_name(library, destination_type, destination_name) super(MessageTransaction, self).__init__(application, name, group=group, source=source) @@ -218,30 +216,12 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that transaction already exists and we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: # Catch all - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) From 998b03556f7c5d30a24f84335e3fa21f775d215d Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Thu, 29 Jun 2023 23:13:36 -0700 Subject: [PATCH 068/525] Upgrade to Pypy38 for TypedDict (#861) * Fix base branch * Revert tox dependencies * Replace all pypy37 with pypy38 * Remove action.yml file * Push Empty Commit * Fix skip_missing_interpreters behavior * Fix skip_missing_interpreters behavior * Pin dev CI image sha * Remove unsupported Tornado tests * Add latest tests to Tornado * Remove pypy38 (for now) --------- Co-authored-by: Tim Pansino --- .../actions/setup-python-matrix/action.yml | 50 -------- .github/containers/Dockerfile | 2 +- .github/workflows/tests.yml | 28 ++--- tox.ini | 114 +++++++++--------- 4 files changed, 73 insertions(+), 121 deletions(-) delete mode 100644 .github/actions/setup-python-matrix/action.yml diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml deleted file mode 100644 index a11e2197c2..0000000000 --- a/.github/actions/setup-python-matrix/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "setup-python-matrix" -description: "Sets up all versions of python required for matrix testing in this repo." -runs: - using: "composite" - steps: - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "pypy-2.7" - # architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "2.7" - # architecture: x64 - - - name: Install Dependencies - shell: bash - run: | - python3.10 -m pip install -U pip - python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 260c01d89f..8d3a187cc6 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -79,7 +79,7 @@ RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ pyenv update # Install Python -ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7" +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.8" COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh COPY ./requirements.txt /requirements.txt RUN /tmp/install-python.sh && \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4cb8b8223..52576c155b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -119,7 +119,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -164,7 +164,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -209,7 +209,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -269,7 +269,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -332,7 +332,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -395,7 +395,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -453,7 +453,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -513,7 +513,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -571,7 +571,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -630,7 +630,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + # image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -710,7 +710,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -768,7 +768,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -828,7 +828,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -889,7 +889,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 diff --git a/tox.ini b/tox.ini index 94722ce8bf..29b74feae7 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy27,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy38 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -28,7 +28,7 @@ ; 5. With or without New Relic C extensions (Optional). Used for testing agent features. ; Examples: with_extensions, without_extensions ; envlist = -; python-agent_features-pypy37-without_extensions, +; python-agent_features-pypy38-without_extensions, ; python-agent_streaming-py37-{with,without}_extensions, ; ; Full Format: @@ -42,6 +42,8 @@ [tox] requires = virtualenv<20.22.0 setupdir = {toxinidir} +; Fail tests when interpreters are missing. +skip_missing_interpreters = false envlist = python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, @@ -56,29 +58,29 @@ envlist = python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_features-{pypy27,pypy38}-without_extensions, python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy27,pypy37}-without_extensions, - python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-agent_unittests-{pypy27,pypy38}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, gearman-application_gearman-{py27,pypy27}, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, - python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py37,py38,py39,pypy38}-flaskrestxlatest, python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, python-component_tastypie-{py27,pypy27}-tastypie0143, - python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, - python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, + python-component_tastypie-{py37,py38,py39,pypy38}-tastypie{0143,latest}, + python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy38}, python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, - elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-elasticsearch07, + elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy38}-elasticsearch08, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-memcached01, mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, @@ -86,83 +88,83 @@ envlist = postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest memcached-datastore_pylibmc-{py27,py37}, - memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy38}-pymongo04, mssql-datastore_pymssql-{py37,py38,py39,py310,py311}, - mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, - redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, - redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, - redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy38}-redis03, + redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest}, + redis-datastore_aioredis-{py37,py38,py39,py310,pypy38}-aioredislatest, + redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy38}-redislatest, + redis-datastore_aredis-{py37,py38,py39,pypy38}-aredislatest, solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, - python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, python-external_feedparser-py27-feedparser{05,06}, python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, - python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, - python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, - python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-urllib3latest, + python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy38}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, python-framework_bottle-py27-bottle{0008,0009,0010}, - python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-{py27,py37,py38,py39,pypy38}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 - python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, + python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy38}-CherryPylatest, python-framework_django-{pypy27,py27}-Django0103, python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, - python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy38}-falcon0103, + python-framework_falcon-{py37,py38,py39,py310,pypy38}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, python-framework_flask-{pypy27,py27}-flask0012, - python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy38}-flask0101, ; temporarily disabling flaskmaster tests - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, + python-framework_flask-{py37,py38,py39,py310,py311,pypy38}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy38}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-graphql02, + python-framework_graphql-{py37,py38,py39,py310,py311,pypy38}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, - python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, - python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, - python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, - python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, - python-framework_starlette-{py310,pypy37}-starlette{0014,0015,0019}, + python-framework_pyramid-{pypy27,py27,pypy38,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{py37,py38,py39,py310,py311,pypy38}-Pyramidlatest, + python-framework_sanic-{py38,pypy38}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, + python-framework_sanic-{py37,py38,py39,py310,py311,pypy38}-saniclatest, + python-framework_starlette-{py310,pypy38}-starlette{0014,0015,0019}, python-framework_starlette-{py37,py38}-starlette{002001}, - python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, + python-framework_starlette-{py37,py38,py39,py310,py311,pypy38}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + python-logger_loguru-{py37,py38,py39,py310,py311,pypy38}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadolatest, python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, - rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy38}-pika0.13, + rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy38}-pikalatest, kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy38}-kafkapythonlatest, kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, python-template_genshi-{py27,py37,py311}-genshilatest python-template_mako-{py27,py37,py310,py311} @@ -170,7 +172,7 @@ envlist = [testenv] deps = # Base Dependencies - {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py37,py38,py39,py310,py311,pypy38}: pytest==7.2.2 {py27,pypy27}: pytest==4.6.11 iniconfig coverage @@ -202,7 +204,7 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_celery-{py37,pypy38}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 @@ -221,8 +223,8 @@ deps = component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 - component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 - component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ + component_tastypie-{py37,py38,py39,py310,py311,pypy38}-tastypie0143: django<3.0.1 + component_tastypie-{py37,py38,py39,py310,py311,pypy38}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie component_tastypie-tastypielatest: django<4.1 component_tastypie-tastypielatest: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ @@ -360,7 +362,7 @@ deps = framework_strawberry: starlette framework_strawberry-strawberrylatest: strawberry-graphql framework_tornado: pycurl - framework_tornado-tornado0600: tornado<6.1 + framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip logger_loguru-logurulatest: loguru logger_loguru-loguru06: loguru<0.7 @@ -391,9 +393,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - framework_tornado: PYCURL_SSL_LIBRARY=openssl - framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib - framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado-{py38,py39,py310,py311}: PYCURL_SSL_LIBRARY=openssl + framework_tornado-{py38,py39,py310,py311}: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado-{py38,py39,py310,py311}: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -408,7 +410,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado-{py38,py39,py310,py311}: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* From 66c2e19ff2dac4ae963a8b68b465669a9e6d1293 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Fri, 30 Jun 2023 00:03:07 -0700 Subject: [PATCH 069/525] Add profile_trace testing (#858) * Include isort stdlibs for determining stdlib modules * Use isort & sys to eliminate std & builtin modules Previously, the logic would fail to identify third party modules installed within the local user socpe. This fixes that issue by skipping builtin and stdlib modules by name, instead of attempting to identify third party modules based on file paths. * Handle importlib_metadata.version being a callable * Add isort into third party notices * [Mega-Linter] Apply linters fixes * Remove Python 2.7 and pypy2 testing (#835) * Change setup-python to @v2 for py2.7 * Remove py27 and pypy testing * Fix syntax errors * Fix comma related syntax errors * Fix more issues in tox * Remove gearman test * Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs * Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting * Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python * Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires * Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino * Trigger tests * Add testing for profile trace. * [Mega-Linter] Apply linters fixes * Ignore __call__ from coverage on profile_trace. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: Hannah Stepanek Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: hmstepanek Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: TimPansino Co-authored-by: umaannamalai --- newrelic/api/profile_trace.py | 50 +++++------- tests/agent_features/test_profile_trace.py | 88 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 tests/agent_features/test_profile_trace.py diff --git a/newrelic/api/profile_trace.py b/newrelic/api/profile_trace.py index 28113b1d81..93aa191a4a 100644 --- a/newrelic/api/profile_trace.py +++ b/newrelic/api/profile_trace.py @@ -13,31 +13,27 @@ # limitations under the License. import functools -import sys import os +import sys -from newrelic.packages import six - -from newrelic.api.time_trace import current_trace +from newrelic import __file__ as AGENT_PACKAGE_FILE from newrelic.api.function_trace import FunctionTrace -from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.api.time_trace import current_trace from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.packages import six -from newrelic import __file__ as AGENT_PACKAGE_FILE -AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + '/' +AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + "/" class ProfileTrace(object): - def __init__(self, depth): self.function_traces = [] self.maximum_depth = depth self.current_depth = 0 - def __call__(self, frame, event, arg): - - if event not in ['call', 'c_call', 'return', 'c_return', - 'exception', 'c_exception']: + def __call__(self, frame, event, arg): # pragma: no cover + if event not in ["call", "c_call", "return", "c_return", "exception", "c_exception"]: return parent = current_trace() @@ -49,8 +45,7 @@ def __call__(self, frame, event, arg): # coroutine systems based on greenlets so don't run # if we detect may be using greenlets. - if (hasattr(sys, '_current_frames') and - parent.thread_id not in sys._current_frames()): + if hasattr(sys, "_current_frames") and parent.thread_id not in sys._current_frames(): return co = frame.f_code @@ -84,7 +79,7 @@ def _callable(): except Exception: pass - if event in ['call', 'c_call']: + if event in ["call", "c_call"]: # Skip the outermost as we catch that with the root # function traces for the profile trace. @@ -100,19 +95,17 @@ def _callable(): self.function_traces.append(None) return - if event == 'call': + if event == "call": func = _callable() if func: name = callable_name(func) else: - name = '%s:%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:%s#%s" % (func_filename, func_name, func_line_no) else: func = arg name = callable_name(arg) if not name: - name = '%s:@%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:@%s#%s" % (func_filename, func_name, func_line_no) function_trace = FunctionTrace(name=name, parent=parent) function_trace.__enter__() @@ -127,7 +120,7 @@ def _callable(): self.function_traces.append(function_trace) self.current_depth += 1 - elif event in ['return', 'c_return', 'c_exception']: + elif event in ["return", "c_return", "c_exception"]: if not self.function_traces: return @@ -143,9 +136,7 @@ def _callable(): self.current_depth -= 1 -def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, - params=None, depth=3): - +def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, params=None, depth=3): def wrapper(wrapped, instance, args, kwargs): parent = current_trace() @@ -192,7 +183,7 @@ def wrapper(wrapped, instance, args, kwargs): _params = params with FunctionTrace(_name, _group, _label, _params, parent=parent, source=wrapped): - if not hasattr(sys, 'getprofile'): + if not hasattr(sys, "getprofile"): return wrapped(*args, **kwargs) profiler = sys.getprofile() @@ -212,11 +203,8 @@ def wrapper(wrapped, instance, args, kwargs): def profile_trace(name=None, group=None, label=None, params=None, depth=3): - return functools.partial(ProfileTraceWrapper, name=name, - group=group, label=label, params=params, depth=depth) + return functools.partial(ProfileTraceWrapper, name=name, group=group, label=label, params=params, depth=depth) -def wrap_profile_trace(module, object_path, name=None, - group=None, label=None, params=None, depth=3): - return wrap_object(module, object_path, ProfileTraceWrapper, - (name, group, label, params, depth)) +def wrap_profile_trace(module, object_path, name=None, group=None, label=None, params=None, depth=3): + return wrap_object(module, object_path, ProfileTraceWrapper, (name, group, label, params, depth)) diff --git a/tests/agent_features/test_profile_trace.py b/tests/agent_features/test_profile_trace.py new file mode 100644 index 0000000000..f696b74809 --- /dev/null +++ b/tests/agent_features/test_profile_trace.py @@ -0,0 +1,88 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.profile_trace import ProfileTraceWrapper, profile_trace + + +def test_profile_trace_wrapper(): + def _test(): + def nested_fn(): + pass + + nested_fn() + + wrapped_test = ProfileTraceWrapper(_test) + wrapped_test() + + +@validate_transaction_metrics("test_profile_trace:test_profile_trace_empty_args", background_task=True) +@background_task() +def test_profile_trace_empty_args(): + @profile_trace() + def _test(): + pass + + _test() + + +_test_profile_trace_defined_args_scoped_metrics = [("Custom/TestTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_defined_args", + scoped_metrics=_test_profile_trace_defined_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_defined_args(): + @profile_trace(name="TestTrace", group="Custom", label="Label", params={"key": "value"}, depth=7) + def _test(): + pass + + _test() + + +_test_profile_trace_callable_args_scoped_metrics = [("Function/TestProfileTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_callable_args", + scoped_metrics=_test_profile_trace_callable_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_callable_args(): + def name_callable(): + return "TestProfileTrace" + + def group_callable(): + return "Function" + + def label_callable(): + return "HSM" + + def params_callable(): + return {"account_id": "12345"} + + @profile_trace(name=name_callable, group=group_callable, label=label_callable, params=params_callable, depth=0) + def _test(): + pass + + _test() From e663c3602e1fa2c80b2f6e3e4494a9d6d72604e6 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 30 Jun 2023 09:19:53 -0700 Subject: [PATCH 070/525] Add Transaction API Tests (#857) * Test for suppress_apdex_metric * Add custom_metrics tests * Add distributed_trace_headers testing in existing tests * [Mega-Linter] Apply linters fixes * Remove redundant if-statement * Ignore deprecated transaction function from coverage * [Mega-Linter] Apply linters fixes * Push empty commit * Update newrelic/api/transaction.py --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: Uma Annamalai --- newrelic/api/transaction.py | 53 ++-- tests/agent_features/test_apdex_metrics.py | 31 ++- tests/agent_features/test_custom_metrics.py | 62 +++++ .../test_distributed_tracing.py | 11 +- tests/cross_agent/test_w3c_trace_context.py | 253 ++++++++++-------- 5 files changed, 257 insertions(+), 153 deletions(-) create mode 100644 tests/agent_features/test_custom_metrics.py diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index f04bcba849..f4d85a4b45 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -159,13 +159,11 @@ def path(self): class Transaction(object): - STATE_PENDING = 0 STATE_RUNNING = 1 STATE_STOPPED = 2 def __init__(self, application, enabled=None, source=None): - self._application = application self._source = source @@ -343,7 +341,6 @@ def __del__(self): self.__exit__(None, None, None) def __enter__(self): - assert self._state == self.STATE_PENDING # Bail out if the transaction is not enabled. @@ -403,7 +400,6 @@ def __enter__(self): return self def __exit__(self, exc, value, tb): - # Bail out if the transaction is not enabled. if not self.enabled: @@ -636,7 +632,6 @@ def __exit__(self, exc, value, tb): # new samples can cause an error. if not self.ignore_transaction: - self._application.record_transaction(node) @property @@ -929,9 +924,7 @@ def filter_request_parameters(self, params): @property def request_parameters(self): if (self.capture_params is None) or self.capture_params: - if self._request_params: - r_attrs = {} for k, v in self._request_params.items(): @@ -1095,7 +1088,6 @@ def _generate_distributed_trace_headers(self, data=None): try: data = data or self._create_distributed_trace_data() if data: - traceparent = W3CTraceParent(data).text() yield ("traceparent", traceparent) @@ -1192,11 +1184,10 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): except: return False - if "pr" in data: - try: - data["pr"] = float(data["pr"]) - except: - data["pr"] = None + try: + data["pr"] = float(data["pr"]) + except Exception: + data["pr"] = None self._accept_distributed_trace_data(data, transport_type) self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Success") @@ -1382,7 +1373,6 @@ def _generate_response_headers(self, read_length=None): # process web external calls. if self.client_cross_process_id is not None: - # Need to work out queueing time and duration up to this # point for inclusion in metrics and response header. If the # recording of the transaction had been prematurely stopped @@ -1426,11 +1416,17 @@ def _generate_response_headers(self, read_length=None): return nr_headers - def get_response_metadata(self): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def get_response_metadata(self): # pragma: no cover nr_headers = dict(self._generate_response_headers()) return convert_to_cat_metadata_value(nr_headers) - def process_request_metadata(self, cat_linking_value): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def process_request_metadata(self, cat_linking_value): # pragma: no cover try: payload = base64_decode(cat_linking_value) except: @@ -1447,7 +1443,6 @@ def process_request_metadata(self, cat_linking_value): return self._process_incoming_cat_headers(encoded_cross_process_id, encoded_txn_header) def set_transaction_name(self, name, group=None, priority=None): - # Always perform this operation even if the transaction # is not active at the time as will be called from # constructor. If path has been frozen do not allow @@ -1517,7 +1512,9 @@ def record_log_event(self, message, level=None, timestamp=None, priority=None): self._log_events.add(event, priority=priority) - def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): # pragma: no cover # Deprecation Warning warnings.warn( ("The record_exception function is deprecated. Please use the new api named notice_error instead."), @@ -1684,7 +1681,9 @@ def add_custom_attributes(self, items): return result - def add_custom_parameter(self, name, value): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameter(self, name, value): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), @@ -1692,7 +1691,9 @@ def add_custom_parameter(self, name, value): ) return self.add_custom_attribute(name, value) - def add_custom_parameters(self, items): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameters(self, items): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), @@ -1796,19 +1797,23 @@ def add_custom_attributes(items): return False -def add_custom_parameter(key, value): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameter(key, value): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), + ("The add_custom_parameter API has been deprecated. Please use the add_custom_attribute API."), DeprecationWarning, ) return add_custom_attribute(key, value) -def add_custom_parameters(items): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameters(items): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), + ("The add_custom_parameters API has been deprecated. Please use the add_custom_attributes API."), DeprecationWarning, ) return add_custom_attributes(items) diff --git a/tests/agent_features/test_apdex_metrics.py b/tests/agent_features/test_apdex_metrics.py index e32a96e312..c150fcf7e6 100644 --- a/tests/agent_features/test_apdex_metrics.py +++ b/tests/agent_features/test_apdex_metrics.py @@ -13,24 +13,41 @@ # limitations under the License. import webtest - -from testing_support.validators.validate_apdex_metrics import ( - validate_apdex_metrics) from testing_support.sample_applications import simple_app +from testing_support.validators.validate_apdex_metrics import validate_apdex_metrics +from newrelic.api.transaction import current_transaction, suppress_apdex_metric +from newrelic.api.wsgi_application import wsgi_application normal_application = webtest.TestApp(simple_app) - # NOTE: This test validates that the server-side apdex_t is set to 0.5 # If the server-side configuration changes, this test will start to fail. @validate_apdex_metrics( - name='', - group='Uri', + name="", + group="Uri", apdex_t_min=0.5, apdex_t_max=0.5, ) def test_apdex(): - normal_application.get('/') + normal_application.get("/") + + +# This has to be a Web Transaction. +# The apdex measurement only applies to Web Transactions +def test_apdex_suppression(): + @wsgi_application() + def simple_apdex_supression_app(environ, start_response): + suppress_apdex_metric() + + start_response(status="200 OK", response_headers=[]) + transaction = current_transaction() + + assert transaction.suppress_apdex + assert transaction.apdex == 0 + return [] + + apdex_suppression_app = webtest.TestApp(simple_apdex_supression_app) + apdex_suppression_app.get("/") diff --git a/tests/agent_features/test_custom_metrics.py b/tests/agent_features/test_custom_metrics.py new file mode 100644 index 0000000000..21a67149a2 --- /dev/null +++ b/tests/agent_features/test_custom_metrics.py @@ -0,0 +1,62 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_custom_metrics_outside_transaction import ( + validate_custom_metrics_outside_transaction, +) + +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import ( + current_transaction, + record_custom_metric, + record_custom_metrics, +) + + +# Testing record_custom_metric +@reset_core_stats_engine() +@background_task() +def test_custom_metric_inside_transaction(): + transaction = current_transaction() + record_custom_metric("CustomMetric/InsideTransaction/Count", 1) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetric/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetric/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metric_outside_transaction_with_app(): + app = application() + record_custom_metric("CustomMetric/OutsideTransaction/Count", 1, application=app) + + +# Testing record_custom_metricS +@reset_core_stats_engine() +@background_task() +def test_custom_metrics_inside_transaction(): + transaction = current_transaction() + record_custom_metrics([("CustomMetrics/InsideTransaction/Count", 1)]) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetrics/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetrics/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metrics_outside_transaction_with_app(): + app = application() + record_custom_metrics([("CustomMetrics/OutsideTransaction/Count", 1)], application=app) diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 4db6d2dab9..3ded79af76 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -32,6 +32,9 @@ from newrelic.api.background_task import BackgroundTask, background_task from newrelic.api.time_trace import current_trace from newrelic.api.transaction import ( + accept_distributed_trace_headers, + accept_distributed_trace_payload, + create_distributed_trace_payload, current_span_id, current_trace_id, current_transaction, @@ -185,10 +188,10 @@ def _test(): payload["d"]["pa"] = "5e5733a911cfbc73" if accept_payload: - result = txn.accept_distributed_trace_payload(payload) + result = accept_distributed_trace_payload(payload) assert result else: - txn._create_distributed_trace_payload() + create_distributed_trace_payload() try: raise ValueError("cookies") @@ -319,7 +322,6 @@ def _test(): ) @override_application_settings(_override_settings) def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, newrelic, metrics): - headers = [] if traceparent: headers.append(("traceparent", TRACEPARENT)) @@ -333,8 +335,7 @@ def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, ne ) @background_task(name="test_distributed_tracing_backwards_compatibility") def _test(): - transaction = current_transaction() - transaction.accept_distributed_trace_headers(headers) + accept_distributed_trace_headers(headers) _test() diff --git a/tests/cross_agent/test_w3c_trace_context.py b/tests/cross_agent/test_w3c_trace_context.py index 05f157f7b7..893274ce44 100644 --- a/tests/cross_agent/test_w3c_trace_context.py +++ b/tests/cross_agent/test_w3c_trace_context.py @@ -14,88 +14,105 @@ import json import os + import pytest import webtest -from newrelic.packages import six - -from newrelic.api.transaction import current_transaction +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + current_transaction, + insert_distributed_trace_headers, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.fixtures import (override_application_settings, - validate_attributes) from newrelic.common.encoding_utils import W3CTraceState -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from newrelic.common.object_wrapper import transient_function_wrapper +from newrelic.packages import six CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ('test_name', 'trusted_account_key', 'account_id', - 'web_transaction', 'raises_exception', 'force_sampled_true', - 'span_events_enabled', 'transport_type', 'inbound_headers', - 'outbound_payloads', 'intrinsics', 'expected_metrics') - -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = ( + "test_name", + "trusted_account_key", + "account_id", + "web_transaction", + "raises_exception", + "force_sampled_true", + "span_events_enabled", + "transport_type", + "inbound_headers", + "outbound_payloads", + "intrinsics", + "expected_metrics", +) + +_parameters = ",".join(_parameters_list) XFAIL_TESTS = [ - 'spans_disabled_root', - 'missing_traceparent', - 'missing_traceparent_and_tracestate', - 'w3c_and_newrelc_headers_present_error_parsing_traceparent' + "spans_disabled_root", + "missing_traceparent", + "missing_traceparent_and_tracestate", + "w3c_and_newrelc_headers_present_error_parsing_traceparent", ] + def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'trace_context.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "trace_context.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result ATTR_MAP = { - 'traceparent.version': 0, - 'traceparent.trace_id': 1, - 'traceparent.parent_id': 2, - 'traceparent.trace_flags': 3, - 'tracestate.version': 0, - 'tracestate.parent_type': 1, - 'tracestate.parent_account_id': 2, - 'tracestate.parent_application_id': 3, - 'tracestate.span_id': 4, - 'tracestate.transaction_id': 5, - 'tracestate.sampled': 6, - 'tracestate.priority': 7, - 'tracestate.timestamp': 8, - 'tracestate.tenant_id': None, + "traceparent.version": 0, + "traceparent.trace_id": 1, + "traceparent.parent_id": 2, + "traceparent.trace_flags": 3, + "tracestate.version": 0, + "tracestate.parent_type": 1, + "tracestate.parent_account_id": 2, + "tracestate.parent_application_id": 3, + "tracestate.span_id": 4, + "tracestate.transaction_id": 5, + "tracestate.sampled": 6, + "tracestate.priority": 7, + "tracestate.timestamp": 8, + "tracestate.tenant_id": None, } def validate_outbound_payload(actual, expected, trusted_account_key): - traceparent = '' - tracestate = '' + traceparent = "" + tracestate = "" for key, value in actual: - if key == 'traceparent': - traceparent = value.split('-') - elif key == 'tracestate': + if key == "traceparent": + traceparent = value.split("-") + elif key == "tracestate": vendors = W3CTraceState.decode(value) - nr_entry = vendors.pop(trusted_account_key + '@nr', '') - tracestate = nr_entry.split('-') - exact_values = expected.get('exact', {}) - expected_attrs = expected.get('expected', []) - unexpected_attrs = expected.get('unexpected', []) - expected_vendors = expected.get('vendors', []) + nr_entry = vendors.pop(trusted_account_key + "@nr", "") + tracestate = nr_entry.split("-") + exact_values = expected.get("exact", {}) + expected_attrs = expected.get("expected", []) + unexpected_attrs = expected.get("unexpected", []) + expected_vendors = expected.get("vendors", []) for key, value in exact_values.items(): - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: if isinstance(value, bool): @@ -106,13 +123,13 @@ def validate_outbound_payload(actual, expected, trusted_account_key): assert header[attr] == str(value) for key in expected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert header[attr], key for key in unexpected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert not header[attr], key @@ -125,127 +142,129 @@ def validate_outbound_payload(actual, expected, trusted_account_key): def target_wsgi_application(environ, start_response): transaction = current_transaction() - if not environ['.web_transaction']: + if not environ[".web_transaction"]: transaction.background_task = True - if environ['.raises_exception']: + if environ[".raises_exception"]: try: raise ValueError("oops") except: transaction.notice_error() - if '.inbound_headers' in environ: - transaction.accept_distributed_trace_headers( - environ['.inbound_headers'], - transport_type=environ['.transport_type'], + if ".inbound_headers" in environ: + accept_distributed_trace_headers( + environ[".inbound_headers"], + transport_type=environ[".transport_type"], ) payloads = [] - for _ in range(environ['.outbound_calls']): + for _ in range(environ[".outbound_calls"]): payloads.append([]) - transaction.insert_distributed_trace_headers(payloads[-1]) + insert_distributed_trace_headers(payloads[-1]) - start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps(payloads).encode('utf-8')] + start_response("200 OK", [("Content-Type", "application/json")]) + return [json.dumps(payloads).encode("utf-8")] test_application = webtest.TestApp(target_wsgi_application) def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @pytest.mark.parametrize(_parameters, load_tests()) -def test_trace_context(test_name, trusted_account_key, account_id, - web_transaction, raises_exception, force_sampled_true, - span_events_enabled, transport_type, inbound_headers, - outbound_payloads, intrinsics, expected_metrics): - +def test_trace_context( + test_name, + trusted_account_key, + account_id, + web_transaction, + raises_exception, + force_sampled_true, + span_events_enabled, + transport_type, + inbound_headers, + outbound_payloads, + intrinsics, + expected_metrics, +): if test_name in XFAIL_TESTS: pytest.xfail("Waiting on cross agent tests update.") # Prepare assertions if not intrinsics: intrinsics = {} - common = intrinsics.get('common', {}) - common_required = common.get('expected', []) - common_forgone = common.get('unexpected', []) - common_exact = common.get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common = intrinsics.get("common", {}) + common_required = common.get("expected", []) + common_forgone = common.get("unexpected", []) + common_exact = common.get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } extra_environ = { - '.web_transaction': web_transaction, - '.raises_exception': raises_exception, - '.transport_type': transport_type, - '.outbound_calls': outbound_payloads and len(outbound_payloads) or 0, + ".web_transaction": web_transaction, + ".raises_exception": raises_exception, + ".transport_type": transport_type, + ".outbound_calls": outbound_payloads and len(outbound_payloads) or 0, } inbound_headers = inbound_headers and inbound_headers[0] or None - if transport_type != 'HTTP': - extra_environ['.inbound_headers'] = inbound_headers + if transport_type != "HTTP": + extra_environ[".inbound_headers"] = inbound_headers inbound_headers = None elif six.PY2 and inbound_headers: - inbound_headers = { - k.encode('utf-8'): v.encode('utf-8') - for k, v in inbound_headers.items()} - - @validate_transaction_metrics(test_name, - group="Uri", - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + inbound_headers = {k.encode("utf-8"): v.encode("utf-8") for k, v in inbound_headers.items()} + + @validate_transaction_metrics( + test_name, group="Uri", rollup_metrics=expected_metrics, background_task=not web_transaction + ) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_application_settings(override_settings) @override_compute_sampled(force_sampled_true) def _test(): return test_application.get( - '/' + test_name, + "/" + test_name, headers=inbound_headers, extra_environ=extra_environ, ) - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) response = _test() - assert response.status == '200 OK' + assert response.status == "200 OK" payloads = response.json if outbound_payloads: assert len(payloads) == len(outbound_payloads) From 3bdb013a8c093481c26f4cb8af4c09d507f301b4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Fri, 30 Jun 2023 09:43:45 -0700 Subject: [PATCH 071/525] Add tests for jinja2. (#842) * Add tests for jinja2. * [Mega-Linter] Apply linters fixes * Update tox.ini Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> --- tests/template_jinja2/conftest.py | 30 ++++++++++++++++++++ tests/template_jinja2/test_jinja2.py | 41 ++++++++++++++++++++++++++++ tox.ini | 3 ++ 3 files changed, 74 insertions(+) create mode 100644 tests/template_jinja2/conftest.py create mode 100644 tests/template_jinja2/test_jinja2.py diff --git a/tests/template_jinja2/conftest.py b/tests/template_jinja2/conftest.py new file mode 100644 index 0000000000..a6922078d4 --- /dev/null +++ b/tests/template_jinja2/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (template_jinja2)", default_settings=_default_settings +) diff --git a/tests/template_jinja2/test_jinja2.py b/tests/template_jinja2/test_jinja2.py new file mode 100644 index 0000000000..c64dac9234 --- /dev/null +++ b/tests/template_jinja2/test_jinja2.py @@ -0,0 +1,41 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Template +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + + +@validate_transaction_metrics( + "test_render", + background_task=True, + scoped_metrics=( + ("Template/Render/