From db075239a656b8cc757609776acf05226783caed Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:02:51 -0700 Subject: [PATCH 01/32] Exclude command line functionality from test coverage (#855) --- codecov.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codecov.yml b/codecov.yml index 61c135aba..8ed9c3020 100644 --- a/codecov.yml +++ b/codecov.yml @@ -22,3 +22,5 @@ ignore: # Temporarily disable kafka - "newrelic/hooks/messagebroker_kafkapython.py" - "newrelic/hooks/messagebroker_confluentkafka.py" + - "newrelic/admin/*" + - "newrelic/console.py" From c2fd5e32aa9fa6c981ed66060718d4bb4401bfed Mon Sep 17 00:00:00 2001 From: Ahmed Helil Date: Tue, 27 Jun 2023 23:37:44 +0200 Subject: [PATCH 02/32] FIX: resilient environment settings (#825) if the application uses generalimport to manage optional depedencies, it's possible that generalimport.MissingOptionalDependency is raised. In this case, we should not report the module as it is not actually loaded and is not a runtime dependency of the application. Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Hannah Stepanek --- newrelic/core/environment.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/newrelic/core/environment.py b/newrelic/core/environment.py index 66efe6112..9bca085a3 100644 --- a/newrelic/core/environment.py +++ b/newrelic/core/environment.py @@ -216,7 +216,15 @@ def environment_settings(): # If the module isn't actually loaded (such as failed relative imports # in Python 2.7), the module will be None and should not be reported. - if not module: + try: + if not module: + continue + except Exception: + # if the application uses generalimport to manage optional depedencies, + # it's possible that generalimport.MissingOptionalDependency is raised. + # In this case, we should not report the module as it is not actually loaded and + # is not a runtime dependency of the application. + # continue # Exclude standard library/built-in modules. From 9883c2b0580ce3c5e1927a4ff83fc6f72f687a7a Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Wed, 28 Jun 2023 16:09:09 -0700 Subject: [PATCH 03/32] Replace drop_transaction logic by using transaction context manager (#832) * Replace drop_transaction call * [Mega-Linter] Apply linters fixes * Empty commit to start tests * Change logic in BG Wrappers --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- newrelic/api/background_task.py | 38 ++++++----------------------- newrelic/api/message_transaction.py | 26 +++----------------- 2 files changed, 10 insertions(+), 54 deletions(-) diff --git a/newrelic/api/background_task.py b/newrelic/api/background_task.py index a4a9e8e6a..4cdcd8a0d 100644 --- a/newrelic/api/background_task.py +++ b/newrelic/api/background_task.py @@ -13,19 +13,16 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.transaction import Transaction, current_transaction -from newrelic.common.async_proxy import async_proxy, TransactionContext +from newrelic.common.async_proxy import TransactionContext, async_proxy from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object class BackgroundTask(Transaction): - def __init__(self, application, name, group=None, source=None): - # Initialise the common transaction base class. super(BackgroundTask, self).__init__(application, source=source) @@ -53,7 +50,6 @@ def __init__(self, application, name, group=None, source=None): def BackgroundTaskWrapper(wrapped, application=None, name=None, group=None): - def wrapper(wrapped, instance, args, kwargs): if callable(name): if instance is not None: @@ -107,39 +103,19 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that a transaction already exists, so we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def background_task(application=None, name=None, group=None): - return functools.partial(BackgroundTaskWrapper, - application=application, name=name, group=group) + return functools.partial(BackgroundTaskWrapper, application=application, name=name, group=group) -def wrap_background_task(module, object_path, application=None, - name=None, group=None): - wrap_object(module, object_path, BackgroundTaskWrapper, - (application, name, group)) +def wrap_background_task(module, object_path, application=None, name=None, group=None): + wrap_object(module, object_path, BackgroundTaskWrapper, (application, name, group)) diff --git a/newrelic/api/message_transaction.py b/newrelic/api/message_transaction.py index 291a3897e..54a71f6ef 100644 --- a/newrelic/api/message_transaction.py +++ b/newrelic/api/message_transaction.py @@ -13,7 +13,6 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.background_task import BackgroundTask @@ -39,7 +38,6 @@ def __init__( transport_type="AMQP", source=None, ): - name, group = self.get_transaction_name(library, destination_type, destination_name) super(MessageTransaction, self).__init__(application, name, group=group, source=source) @@ -218,30 +216,12 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that transaction already exists and we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: # Catch all - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) From 998b03556f7c5d30a24f84335e3fa21f775d215d Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Thu, 29 Jun 2023 23:13:36 -0700 Subject: [PATCH 04/32] Upgrade to Pypy38 for TypedDict (#861) * Fix base branch * Revert tox dependencies * Replace all pypy37 with pypy38 * Remove action.yml file * Push Empty Commit * Fix skip_missing_interpreters behavior * Fix skip_missing_interpreters behavior * Pin dev CI image sha * Remove unsupported Tornado tests * Add latest tests to Tornado * Remove pypy38 (for now) --------- Co-authored-by: Tim Pansino --- .../actions/setup-python-matrix/action.yml | 50 -------- .github/containers/Dockerfile | 2 +- .github/workflows/tests.yml | 28 ++--- tox.ini | 114 +++++++++--------- 4 files changed, 73 insertions(+), 121 deletions(-) delete mode 100644 .github/actions/setup-python-matrix/action.yml diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml deleted file mode 100644 index a11e2197c..000000000 --- a/.github/actions/setup-python-matrix/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "setup-python-matrix" -description: "Sets up all versions of python required for matrix testing in this repo." -runs: - using: "composite" - steps: - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "pypy-2.7" - # architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - # - uses: actions/setup-python@v4 - # with: - # python-version: "2.7" - # architecture: x64 - - - name: Install Dependencies - shell: bash - run: | - python3.10 -m pip install -U pip - python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile index 260c01d89..8d3a187cc 100644 --- a/.github/containers/Dockerfile +++ b/.github/containers/Dockerfile @@ -79,7 +79,7 @@ RUN echo 'eval "$(pyenv init -)"' >>$HOME/.bashrc && \ pyenv update # Install Python -ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.7" +ARG PYTHON_VERSIONS="3.10 3.9 3.8 3.7 3.11 2.7 pypy2.7-7.3.11 pypy3.8" COPY --chown=1000:1000 --chmod=+x ./install-python.sh /tmp/install-python.sh COPY ./requirements.txt /requirements.txt RUN /tmp/install-python.sh && \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4cb8b822..52576c155 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -119,7 +119,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -164,7 +164,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -209,7 +209,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -269,7 +269,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -332,7 +332,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -395,7 +395,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -453,7 +453,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -513,7 +513,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -571,7 +571,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -630,7 +630,7 @@ jobs: # runs-on: ubuntu-20.04 # container: - # image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + # image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 # options: >- # --add-host=host.docker.internal:host-gateway # timeout-minutes: 30 @@ -710,7 +710,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -768,7 +768,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -828,7 +828,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 @@ -889,7 +889,7 @@ jobs: runs-on: ubuntu-20.04 container: - image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + image: ghcr.io/newrelic/newrelic-python-agent-ci:sha-b0ffe8bdbb28ba0579377076ad680054da8fbc28 options: >- --add-host=host.docker.internal:host-gateway timeout-minutes: 30 diff --git a/tox.ini b/tox.ini index 94722ce8b..29b74feae 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ ; framework_aiohttp-aiohttp01: aiohttp<2 ; framework_aiohttp-aiohttp0202: aiohttp<2.3 ; 3. Python version required. Uses the standard tox definitions. (https://tox.readthedocs.io/en/latest/config.html#tox-environments) -; Examples: py27,py37,py38,py39,pypy27,pypy37 +; Examples: py27,py37,py38,py39,pypy27,pypy38 ; 4. Library and version (Optional). Used when testing multiple versions of the library, and may be omitted when only testing a single version. ; Versions should be specified with 2 digits per version number, so <3 becomes 02 and <3.5 becomes 0304. latest and master are also acceptable versions. ; Examples: uvicorn03, CherryPy0302, uvicornlatest @@ -28,7 +28,7 @@ ; 5. With or without New Relic C extensions (Optional). Used for testing agent features. ; Examples: with_extensions, without_extensions ; envlist = -; python-agent_features-pypy37-without_extensions, +; python-agent_features-pypy38-without_extensions, ; python-agent_streaming-py37-{with,without}_extensions, ; ; Full Format: @@ -42,6 +42,8 @@ [tox] requires = virtualenv<20.22.0 setupdir = {toxinidir} +; Fail tests when interpreters are missing. +skip_missing_interpreters = false envlist = python-adapter_cheroot-{py27,py37,py38,py39,py310,py311}, python-adapter_daphne-{py37,py38,py39,py310,py311}-daphnelatest, @@ -56,29 +58,29 @@ envlist = python-adapter_waitress-{py37,py38,py39,py310}-waitress02, python-adapter_waitress-{py37,py38,py39,py310,py311}-waitresslatest, python-agent_features-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_features-{pypy27,pypy37}-without_extensions, + python-agent_features-{pypy27,pypy38}-without_extensions, python-agent_streaming-py27-grpc0125-{with,without}_extensions, python-agent_streaming-{py37,py38,py39,py310,py311}-protobuf04-{with,without}_extensions, python-agent_streaming-py39-protobuf{03,0319}-{with,without}_extensions, python-agent_unittests-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, - python-agent_unittests-{pypy27,pypy37}-without_extensions, - python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-agent_unittests-{pypy27,pypy38}-without_extensions, + python-application_celery-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, gearman-application_gearman-{py27,pypy27}, python-component_djangorestframework-py27-djangorestframework0300, python-component_djangorestframework-{py37,py38,py39,py310,py311}-djangorestframeworklatest, - python-component_flask_rest-{py37,py38,py39,pypy37}-flaskrestxlatest, + python-component_flask_rest-{py37,py38,py39,pypy38}-flaskrestxlatest, python-component_flask_rest-{py27,pypy27}-flaskrestx051, python-component_graphqlserver-{py37,py38,py39,py310,py311}, python-component_tastypie-{py27,pypy27}-tastypie0143, - python-component_tastypie-{py37,py38,py39,pypy37}-tastypie{0143,latest}, - python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy37}, + python-component_tastypie-{py37,py38,py39,pypy38}-tastypie{0143,latest}, + python-coroutines_asyncio-{py37,py38,py39,py310,py311,pypy38}, python-cross_agent-{py27,py37,py38,py39,py310,py311}-{with,without}_extensions, python-cross_agent-pypy27-without_extensions, postgres-datastore_asyncpg-{py37,py38,py39,py310,py311}, memcached-datastore_bmemcached-{pypy27,py27,py37,py38,py39,py310,py311}-memcached030, - elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-elasticsearch07, - elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy37}-elasticsearch08, - memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-memcached01, + elasticsearchserver07-datastore_elasticsearch-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-elasticsearch07, + elasticsearchserver08-datastore_elasticsearch-{py37,py38,py39,py310,py311,pypy38}-elasticsearch08, + memcached-datastore_memcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-memcached01, mysql-datastore_mysql-mysql080023-py27, mysql-datastore_mysql-mysqllatest-{py37,py38,py39,py310,py311}, postgres-datastore_postgresql-{py37,py38,py39}, @@ -86,83 +88,83 @@ envlist = postgres-datastore_psycopg2cffi-{py27,pypy27,py37,py38,py39,py310,py311}-psycopg2cffilatest, postgres-datastore_pyodbc-{py27,py37,py311}-pyodbclatest memcached-datastore_pylibmc-{py27,py37}, - memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + memcached-datastore_pymemcache-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, mongodb-datastore_pymongo-{py27,py37,py38,py39,py310,py311,pypy27}-pymongo{03}, - mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy37}-pymongo04, + mongodb-datastore_pymongo-{py37,py38,py39,py310,py311,pypy27,pypy38}-pymongo04, mssql-datastore_pymssql-{py37,py38,py39,py310,py311}, - mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - redis-datastore_redis-{py27,py37,py38,pypy27,pypy37}-redis03, - redis-datastore_redis-{py37,py38,py39,py310,py311,pypy37}-redis{0400,latest}, - redis-datastore_aioredis-{py37,py38,py39,py310,pypy37}-aioredislatest, - redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy37}-redislatest, - redis-datastore_aredis-{py37,py38,py39,pypy37}-aredislatest, + mysql-datastore_pymysql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + solr-datastore_pysolr-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + redis-datastore_redis-{py27,py37,py38,pypy27,pypy38}-redis03, + redis-datastore_redis-{py37,py38,py39,py310,py311,pypy38}-redis{0400,latest}, + redis-datastore_aioredis-{py37,py38,py39,py310,pypy38}-aioredislatest, + redis-datastore_aioredis-{py37,py38,py39,py310,py311,pypy38}-redislatest, + redis-datastore_aredis-{py37,py38,py39,pypy38}-aredislatest, solr-datastore_solrpy-{py27,pypy27}-solrpy{00,01}, - python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-datastore_sqlite-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_boto3-{py27,py37,py38,py39,py310,py311}-boto01, python-external_botocore-{py37,py38,py39,py310,py311}-botocorelatest, python-external_botocore-{py311}-botocore128, python-external_botocore-py310-botocore0125, python-external_feedparser-py27-feedparser{05,06}, python-external_http-{py27,py37,py38,py39,py310,py311,pypy27}, - python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_httplib-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + python-external_httplib2-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_httpx-{py37,py38,py39,py310,py311}, - python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, + python-external_requests-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, python-external_urllib3-{py27,py37,pypy27}-urllib3{0109}, - python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-urllib3latest, - python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy37}-aiohttp03, + python-external_urllib3-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-urllib3latest, + python-framework_aiohttp-{py37,py38,py39,py310,py311,pypy38}-aiohttp03, python-framework_ariadne-{py37,py38,py39,py310,py311}-ariadnelatest, python-framework_ariadne-py37-ariadne{0011,0012,0013}, python-framework_bottle-py27-bottle{0008,0009,0010}, - python-framework_bottle-{py27,py37,py38,py39,pypy37}-bottle{0011,0012}, + python-framework_bottle-{py27,py37,py38,py39,pypy38}-bottle{0011,0012}, python-framework_bottle-{py310,py311}-bottle0012, python-framework_bottle-pypy27-bottle{0008,0009,0010,0011,0012}, ; CherryPy still uses inspect.getargspec, deprecated in favor of inspect.getfullargspec. Not supported in 3.11 - python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy37}-CherryPylatest, + python-framework_cherrypy-{py37,py38,py39,py310,py311,pypy38}-CherryPylatest, python-framework_django-{pypy27,py27}-Django0103, python-framework_django-{pypy27,py27,py37}-Django0108, python-framework_django-{py39}-Django{0200,0201,0202,0300,0301,latest}, python-framework_django-{py37,py38,py39,py310,py311}-Django0302, - python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy37}-falcon0103, - python-framework_falcon-{py37,py38,py39,py310,pypy37}-falcon{0200,master}, + python-framework_falcon-{py27,py37,py38,py39,pypy27,pypy38}-falcon0103, + python-framework_falcon-{py37,py38,py39,py310,pypy38}-falcon{0200,master}, # Falcon master branch failing on 3.11 currently. python-framework_falcon-py311-falcon0200, python-framework_fastapi-{py37,py38,py39,py310,py311}, python-framework_flask-{pypy27,py27}-flask0012, - python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy37}-flask0101, + python-framework_flask-{pypy27,py27,py37,py38,py39,py310,py311,pypy38}-flask0101, ; temporarily disabling flaskmaster tests - python-framework_flask-{py37,py38,py39,py310,py311,pypy37}-flask{latest}, + python-framework_flask-{py37,py38,py39,py310,py311,pypy38}-flask{latest}, python-framework_graphene-{py37,py38,py39,py310,py311}-graphenelatest, - python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy37}-graphene{0200,0201}, + python-framework_graphene-{py27,py37,py38,py39,pypy27,pypy38}-graphene{0200,0201}, python-framework_graphene-{py310,py311}-graphene0201, - python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}-graphql02, - python-framework_graphql-{py37,py38,py39,py310,py311,pypy37}-graphql03, + python-framework_graphql-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}-graphql02, + python-framework_graphql-{py37,py38,py39,py310,py311,pypy38}-graphql03, ; temporarily disabling graphqlmaster tests python-framework_graphql-py37-graphql{0202,0203,0300,0301,0302}, grpc-framework_grpc-py27-grpc0125, grpc-framework_grpc-{py37,py38,py39,py310,py311}-grpclatest, python-framework_pyramid-{pypy27,py27,py38}-Pyramid0104, - python-framework_pyramid-{pypy27,py27,pypy37,py37,py38,py39,py310,py311}-Pyramid0110-cornice, - python-framework_pyramid-{py37,py38,py39,py310,py311,pypy37}-Pyramidlatest, - python-framework_sanic-{py38,pypy37}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, - python-framework_sanic-{py37,py38,py39,py310,py311,pypy37}-saniclatest, - python-framework_starlette-{py310,pypy37}-starlette{0014,0015,0019}, + python-framework_pyramid-{pypy27,py27,pypy38,py37,py38,py39,py310,py311}-Pyramid0110-cornice, + python-framework_pyramid-{py37,py38,py39,py310,py311,pypy38}-Pyramidlatest, + python-framework_sanic-{py38,pypy38}-sanic{190301,1906,1912,200904,210300,2109,2112,2203,2290}, + python-framework_sanic-{py37,py38,py39,py310,py311,pypy38}-saniclatest, + python-framework_starlette-{py310,pypy38}-starlette{0014,0015,0019}, python-framework_starlette-{py37,py38}-starlette{002001}, - python-framework_starlette-{py37,py38,py39,py310,py311,pypy37}-starlettelatest, + python-framework_starlette-{py37,py38,py39,py310,py311,pypy38}-starlettelatest, python-framework_strawberry-{py37,py38,py39,py310,py311}-strawberrylatest, - python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy37}, - python-logger_loguru-{py37,py38,py39,py310,py311,pypy37}-logurulatest, + python-logger_logging-{py27,py37,py38,py39,py310,py311,pypy27,pypy38}, + python-logger_loguru-{py37,py38,py39,py310,py311,pypy38}-logurulatest, python-logger_loguru-py39-loguru{06,05,04,03}, - python-framework_tornado-{py37,py38,py39,py310,py311,pypy37}-tornado0600, + python-framework_tornado-{py38,py39,py310,py311}-tornadolatest, python-framework_tornado-{py38,py39,py310,py311}-tornadomaster, - rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy37}-pika0.13, - rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy37}-pikalatest, + rabbitmq-messagebroker_pika-{py27,py37,py38,py39,pypy27,pypy38}-pika0.13, + rabbitmq-messagebroker_pika-{py37,py38,py39,py310,py311,pypy38}-pikalatest, kafka-messagebroker_confluentkafka-{py27,py37,py38,py39,py310,py311}-confluentkafkalatest, kafka-messagebroker_confluentkafka-{py27,py39}-confluentkafka{0107,0106}, ; confluent-kafka had a bug in 1.8.2's setup.py file which was incompatible with 2.7. kafka-messagebroker_confluentkafka-{py39}-confluentkafka{0108}, - kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy37}-kafkapythonlatest, + kafka-messagebroker_kafkapython-{pypy27,py27,py37,py38,pypy38}-kafkapythonlatest, kafka-messagebroker_kafkapython-{py27,py38}-kafkapython{020001,020000,0104}, python-template_genshi-{py27,py37,py311}-genshilatest python-template_mako-{py27,py37,py310,py311} @@ -170,7 +172,7 @@ envlist = [testenv] deps = # Base Dependencies - {py37,py38,py39,py310,py311,pypy37}: pytest==7.2.2 + {py37,py38,py39,py310,py311,pypy38}: pytest==7.2.2 {py27,pypy27}: pytest==4.6.11 iniconfig coverage @@ -202,7 +204,7 @@ deps = adapter_waitress-waitresslatest: waitress agent_features: beautifulsoup4 application_celery: celery<6.0 - application_celery-{py37,pypy37}: importlib-metadata<5.0 + application_celery-{py37,pypy38}: importlib-metadata<5.0 application_gearman: gearman<3.0.0 component_djangorestframework-djangorestframework0300: Django<1.9 component_djangorestframework-djangorestframework0300: djangorestframework<3.1 @@ -221,8 +223,8 @@ deps = component_graphqlserver: jinja2<3.1 component_tastypie-tastypie0143: django-tastypie<0.14.4 component_tastypie-{py27,pypy27}-tastypie0143: django<1.12 - component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: django<3.0.1 - component_tastypie-{py37,py38,py39,py310,py311,pypy37}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ + component_tastypie-{py37,py38,py39,py310,py311,pypy38}-tastypie0143: django<3.0.1 + component_tastypie-{py37,py38,py39,py310,py311,pypy38}-tastypie0143: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ component_tastypie-tastypielatest: django-tastypie component_tastypie-tastypielatest: django<4.1 component_tastypie-tastypielatest: asgiref<3.7.1 # asgiref==3.7.1 only suppport Python 3.10+ @@ -360,7 +362,7 @@ deps = framework_strawberry: starlette framework_strawberry-strawberrylatest: strawberry-graphql framework_tornado: pycurl - framework_tornado-tornado0600: tornado<6.1 + framework_tornado-tornadolatest: tornado framework_tornado-tornadomaster: https://github.com/tornadoweb/tornado/archive/master.zip logger_loguru-logurulatest: loguru logger_loguru-loguru06: loguru<0.7 @@ -391,9 +393,9 @@ setenv = without_extensions: NEW_RELIC_EXTENSIONS = false agent_features: NEW_RELIC_APDEX_T = 1000 framework_grpc: PYTHONPATH={toxinidir}/tests/:{toxinidir}/tests/framework_grpc/sample_application - framework_tornado: PYCURL_SSL_LIBRARY=openssl - framework_tornado: LDFLAGS=-L/usr/local/opt/openssl/lib - framework_tornado: CPPFLAGS=-I/usr/local/opt/openssl/include + framework_tornado-{py38,py39,py310,py311}: PYCURL_SSL_LIBRARY=openssl + framework_tornado-{py38,py39,py310,py311}: LDFLAGS=-L/usr/local/opt/openssl/lib + framework_tornado-{py38,py39,py310,py311}: CPPFLAGS=-I/usr/local/opt/openssl/include passenv = NEW_RELIC_DEVELOPER_MODE @@ -408,7 +410,7 @@ commands = framework_grpc: --grpc_python_out={toxinidir}/tests/framework_grpc/sample_application \ framework_grpc: /{toxinidir}/tests/framework_grpc/sample_application/sample_application.proto - framework_tornado: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl + framework_tornado-{py38,py39,py310,py311}: pip install --ignore-installed --config-settings="--build-option=--with-openssl" pycurl coverage run -m pytest -v [] allowlist_externals={toxinidir}/.github/scripts/* From 66c2e19ff2dac4ae963a8b68b465669a9e6d1293 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Fri, 30 Jun 2023 00:03:07 -0700 Subject: [PATCH 05/32] Add profile_trace testing (#858) * Include isort stdlibs for determining stdlib modules * Use isort & sys to eliminate std & builtin modules Previously, the logic would fail to identify third party modules installed within the local user socpe. This fixes that issue by skipping builtin and stdlib modules by name, instead of attempting to identify third party modules based on file paths. * Handle importlib_metadata.version being a callable * Add isort into third party notices * [Mega-Linter] Apply linters fixes * Remove Python 2.7 and pypy2 testing (#835) * Change setup-python to @v2 for py2.7 * Remove py27 and pypy testing * Fix syntax errors * Fix comma related syntax errors * Fix more issues in tox * Remove gearman test * Containerized CI Pipeline (#836) * Revert "Remove Python 2.7 and pypy2 testing (#835)" This reverts commit abb6405d2bfd629ed83f48e8a17b4a28e3a3c352. * Containerize CI process * Publish new docker container for CI images * Rename github actions job * Copyright tag scripts * Drop debug line * Swap to new CI image * Move pip install to just main python * Remove libcurl special case from tox * Install special case packages into main image * Remove unused packages * Remove all other triggers besides manual * Add make run command * Cleanup small bugs * Fix CI Image Tagging (#838) * Correct templated CI image name * Pin pypy2.7 in image * Fix up scripting * Temporarily Restore Old CI Pipeline (#841) * Restore old pipelines * Remove python 2 from setup-python * Rework CI Pipeline (#839) Change pypy to pypy27 in tox. Fix checkout logic Pin tox requires * Fix Tests on New CI (#843) * Remove non-root user * Test new CI image * Change pypy to pypy27 in tox. * Fix checkout logic * Fetch git tags properly * Pin tox requires * Adjust default db settings for github actions * Rename elasticsearch services * Reset to new pipelines * [Mega-Linter] Apply linters fixes * Fix timezone * Fix docker networking * Pin dev image to new sha * Standardize gearman DB settings * Fix elasticsearch settings bug * Fix gearman bug * Add missing odbc headers * Add more debug messages * Swap out dev ci image * Fix required virtualenv version * Swap out dev ci image * Swap out dev ci image * Remove aioredis v1 for EOL * Add coverage paths for docker container * Unpin ci container --------- Co-authored-by: TimPansino * Trigger tests * Add testing for profile trace. * [Mega-Linter] Apply linters fixes * Ignore __call__ from coverage on profile_trace. * [Mega-Linter] Apply linters fixes --------- Co-authored-by: Hannah Stepanek Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: hmstepanek Co-authored-by: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: TimPansino Co-authored-by: umaannamalai --- newrelic/api/profile_trace.py | 50 +++++------- tests/agent_features/test_profile_trace.py | 88 ++++++++++++++++++++++ 2 files changed, 107 insertions(+), 31 deletions(-) create mode 100644 tests/agent_features/test_profile_trace.py diff --git a/newrelic/api/profile_trace.py b/newrelic/api/profile_trace.py index 28113b1d8..93aa191a4 100644 --- a/newrelic/api/profile_trace.py +++ b/newrelic/api/profile_trace.py @@ -13,31 +13,27 @@ # limitations under the License. import functools -import sys import os +import sys -from newrelic.packages import six - -from newrelic.api.time_trace import current_trace +from newrelic import __file__ as AGENT_PACKAGE_FILE from newrelic.api.function_trace import FunctionTrace -from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.api.time_trace import current_trace from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.packages import six -from newrelic import __file__ as AGENT_PACKAGE_FILE -AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + '/' +AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + "/" class ProfileTrace(object): - def __init__(self, depth): self.function_traces = [] self.maximum_depth = depth self.current_depth = 0 - def __call__(self, frame, event, arg): - - if event not in ['call', 'c_call', 'return', 'c_return', - 'exception', 'c_exception']: + def __call__(self, frame, event, arg): # pragma: no cover + if event not in ["call", "c_call", "return", "c_return", "exception", "c_exception"]: return parent = current_trace() @@ -49,8 +45,7 @@ def __call__(self, frame, event, arg): # coroutine systems based on greenlets so don't run # if we detect may be using greenlets. - if (hasattr(sys, '_current_frames') and - parent.thread_id not in sys._current_frames()): + if hasattr(sys, "_current_frames") and parent.thread_id not in sys._current_frames(): return co = frame.f_code @@ -84,7 +79,7 @@ def _callable(): except Exception: pass - if event in ['call', 'c_call']: + if event in ["call", "c_call"]: # Skip the outermost as we catch that with the root # function traces for the profile trace. @@ -100,19 +95,17 @@ def _callable(): self.function_traces.append(None) return - if event == 'call': + if event == "call": func = _callable() if func: name = callable_name(func) else: - name = '%s:%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:%s#%s" % (func_filename, func_name, func_line_no) else: func = arg name = callable_name(arg) if not name: - name = '%s:@%s#%s' % (func_filename, func_name, - func_line_no) + name = "%s:@%s#%s" % (func_filename, func_name, func_line_no) function_trace = FunctionTrace(name=name, parent=parent) function_trace.__enter__() @@ -127,7 +120,7 @@ def _callable(): self.function_traces.append(function_trace) self.current_depth += 1 - elif event in ['return', 'c_return', 'c_exception']: + elif event in ["return", "c_return", "c_exception"]: if not self.function_traces: return @@ -143,9 +136,7 @@ def _callable(): self.current_depth -= 1 -def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, - params=None, depth=3): - +def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, params=None, depth=3): def wrapper(wrapped, instance, args, kwargs): parent = current_trace() @@ -192,7 +183,7 @@ def wrapper(wrapped, instance, args, kwargs): _params = params with FunctionTrace(_name, _group, _label, _params, parent=parent, source=wrapped): - if not hasattr(sys, 'getprofile'): + if not hasattr(sys, "getprofile"): return wrapped(*args, **kwargs) profiler = sys.getprofile() @@ -212,11 +203,8 @@ def wrapper(wrapped, instance, args, kwargs): def profile_trace(name=None, group=None, label=None, params=None, depth=3): - return functools.partial(ProfileTraceWrapper, name=name, - group=group, label=label, params=params, depth=depth) + return functools.partial(ProfileTraceWrapper, name=name, group=group, label=label, params=params, depth=depth) -def wrap_profile_trace(module, object_path, name=None, - group=None, label=None, params=None, depth=3): - return wrap_object(module, object_path, ProfileTraceWrapper, - (name, group, label, params, depth)) +def wrap_profile_trace(module, object_path, name=None, group=None, label=None, params=None, depth=3): + return wrap_object(module, object_path, ProfileTraceWrapper, (name, group, label, params, depth)) diff --git a/tests/agent_features/test_profile_trace.py b/tests/agent_features/test_profile_trace.py new file mode 100644 index 000000000..f696b7480 --- /dev/null +++ b/tests/agent_features/test_profile_trace.py @@ -0,0 +1,88 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.profile_trace import ProfileTraceWrapper, profile_trace + + +def test_profile_trace_wrapper(): + def _test(): + def nested_fn(): + pass + + nested_fn() + + wrapped_test = ProfileTraceWrapper(_test) + wrapped_test() + + +@validate_transaction_metrics("test_profile_trace:test_profile_trace_empty_args", background_task=True) +@background_task() +def test_profile_trace_empty_args(): + @profile_trace() + def _test(): + pass + + _test() + + +_test_profile_trace_defined_args_scoped_metrics = [("Custom/TestTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_defined_args", + scoped_metrics=_test_profile_trace_defined_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_defined_args(): + @profile_trace(name="TestTrace", group="Custom", label="Label", params={"key": "value"}, depth=7) + def _test(): + pass + + _test() + + +_test_profile_trace_callable_args_scoped_metrics = [("Function/TestProfileTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_callable_args", + scoped_metrics=_test_profile_trace_callable_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_callable_args(): + def name_callable(): + return "TestProfileTrace" + + def group_callable(): + return "Function" + + def label_callable(): + return "HSM" + + def params_callable(): + return {"account_id": "12345"} + + @profile_trace(name=name_callable, group=group_callable, label=label_callable, params=params_callable, depth=0) + def _test(): + pass + + _test() From e663c3602e1fa2c80b2f6e3e4494a9d6d72604e6 Mon Sep 17 00:00:00 2001 From: Lalleh Rafeei <84813886+lrafeei@users.noreply.github.com> Date: Fri, 30 Jun 2023 09:19:53 -0700 Subject: [PATCH 06/32] Add Transaction API Tests (#857) * Test for suppress_apdex_metric * Add custom_metrics tests * Add distributed_trace_headers testing in existing tests * [Mega-Linter] Apply linters fixes * Remove redundant if-statement * Ignore deprecated transaction function from coverage * [Mega-Linter] Apply linters fixes * Push empty commit * Update newrelic/api/transaction.py --------- Co-authored-by: lrafeei Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> Co-authored-by: Uma Annamalai --- newrelic/api/transaction.py | 53 ++-- tests/agent_features/test_apdex_metrics.py | 31 ++- tests/agent_features/test_custom_metrics.py | 62 +++++ .../test_distributed_tracing.py | 11 +- tests/cross_agent/test_w3c_trace_context.py | 253 ++++++++++-------- 5 files changed, 257 insertions(+), 153 deletions(-) create mode 100644 tests/agent_features/test_custom_metrics.py diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index f04bcba84..f4d85a4b4 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -159,13 +159,11 @@ def path(self): class Transaction(object): - STATE_PENDING = 0 STATE_RUNNING = 1 STATE_STOPPED = 2 def __init__(self, application, enabled=None, source=None): - self._application = application self._source = source @@ -343,7 +341,6 @@ def __del__(self): self.__exit__(None, None, None) def __enter__(self): - assert self._state == self.STATE_PENDING # Bail out if the transaction is not enabled. @@ -403,7 +400,6 @@ def __enter__(self): return self def __exit__(self, exc, value, tb): - # Bail out if the transaction is not enabled. if not self.enabled: @@ -636,7 +632,6 @@ def __exit__(self, exc, value, tb): # new samples can cause an error. if not self.ignore_transaction: - self._application.record_transaction(node) @property @@ -929,9 +924,7 @@ def filter_request_parameters(self, params): @property def request_parameters(self): if (self.capture_params is None) or self.capture_params: - if self._request_params: - r_attrs = {} for k, v in self._request_params.items(): @@ -1095,7 +1088,6 @@ def _generate_distributed_trace_headers(self, data=None): try: data = data or self._create_distributed_trace_data() if data: - traceparent = W3CTraceParent(data).text() yield ("traceparent", traceparent) @@ -1192,11 +1184,10 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): except: return False - if "pr" in data: - try: - data["pr"] = float(data["pr"]) - except: - data["pr"] = None + try: + data["pr"] = float(data["pr"]) + except Exception: + data["pr"] = None self._accept_distributed_trace_data(data, transport_type) self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Success") @@ -1382,7 +1373,6 @@ def _generate_response_headers(self, read_length=None): # process web external calls. if self.client_cross_process_id is not None: - # Need to work out queueing time and duration up to this # point for inclusion in metrics and response header. If the # recording of the transaction had been prematurely stopped @@ -1426,11 +1416,17 @@ def _generate_response_headers(self, read_length=None): return nr_headers - def get_response_metadata(self): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def get_response_metadata(self): # pragma: no cover nr_headers = dict(self._generate_response_headers()) return convert_to_cat_metadata_value(nr_headers) - def process_request_metadata(self, cat_linking_value): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def process_request_metadata(self, cat_linking_value): # pragma: no cover try: payload = base64_decode(cat_linking_value) except: @@ -1447,7 +1443,6 @@ def process_request_metadata(self, cat_linking_value): return self._process_incoming_cat_headers(encoded_cross_process_id, encoded_txn_header) def set_transaction_name(self, name, group=None, priority=None): - # Always perform this operation even if the transaction # is not active at the time as will be called from # constructor. If path has been frozen do not allow @@ -1517,7 +1512,9 @@ def record_log_event(self, message, level=None, timestamp=None, priority=None): self._log_events.add(event, priority=priority) - def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): # pragma: no cover # Deprecation Warning warnings.warn( ("The record_exception function is deprecated. Please use the new api named notice_error instead."), @@ -1684,7 +1681,9 @@ def add_custom_attributes(self, items): return result - def add_custom_parameter(self, name, value): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameter(self, name, value): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), @@ -1692,7 +1691,9 @@ def add_custom_parameter(self, name, value): ) return self.add_custom_attribute(name, value) - def add_custom_parameters(self, items): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameters(self, items): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), @@ -1796,19 +1797,23 @@ def add_custom_attributes(items): return False -def add_custom_parameter(key, value): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameter(key, value): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), + ("The add_custom_parameter API has been deprecated. Please use the add_custom_attribute API."), DeprecationWarning, ) return add_custom_attribute(key, value) -def add_custom_parameters(items): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameters(items): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), + ("The add_custom_parameters API has been deprecated. Please use the add_custom_attributes API."), DeprecationWarning, ) return add_custom_attributes(items) diff --git a/tests/agent_features/test_apdex_metrics.py b/tests/agent_features/test_apdex_metrics.py index e32a96e31..c150fcf7e 100644 --- a/tests/agent_features/test_apdex_metrics.py +++ b/tests/agent_features/test_apdex_metrics.py @@ -13,24 +13,41 @@ # limitations under the License. import webtest - -from testing_support.validators.validate_apdex_metrics import ( - validate_apdex_metrics) from testing_support.sample_applications import simple_app +from testing_support.validators.validate_apdex_metrics import validate_apdex_metrics +from newrelic.api.transaction import current_transaction, suppress_apdex_metric +from newrelic.api.wsgi_application import wsgi_application normal_application = webtest.TestApp(simple_app) - # NOTE: This test validates that the server-side apdex_t is set to 0.5 # If the server-side configuration changes, this test will start to fail. @validate_apdex_metrics( - name='', - group='Uri', + name="", + group="Uri", apdex_t_min=0.5, apdex_t_max=0.5, ) def test_apdex(): - normal_application.get('/') + normal_application.get("/") + + +# This has to be a Web Transaction. +# The apdex measurement only applies to Web Transactions +def test_apdex_suppression(): + @wsgi_application() + def simple_apdex_supression_app(environ, start_response): + suppress_apdex_metric() + + start_response(status="200 OK", response_headers=[]) + transaction = current_transaction() + + assert transaction.suppress_apdex + assert transaction.apdex == 0 + return [] + + apdex_suppression_app = webtest.TestApp(simple_apdex_supression_app) + apdex_suppression_app.get("/") diff --git a/tests/agent_features/test_custom_metrics.py b/tests/agent_features/test_custom_metrics.py new file mode 100644 index 000000000..21a67149a --- /dev/null +++ b/tests/agent_features/test_custom_metrics.py @@ -0,0 +1,62 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_custom_metrics_outside_transaction import ( + validate_custom_metrics_outside_transaction, +) + +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import ( + current_transaction, + record_custom_metric, + record_custom_metrics, +) + + +# Testing record_custom_metric +@reset_core_stats_engine() +@background_task() +def test_custom_metric_inside_transaction(): + transaction = current_transaction() + record_custom_metric("CustomMetric/InsideTransaction/Count", 1) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetric/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetric/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metric_outside_transaction_with_app(): + app = application() + record_custom_metric("CustomMetric/OutsideTransaction/Count", 1, application=app) + + +# Testing record_custom_metricS +@reset_core_stats_engine() +@background_task() +def test_custom_metrics_inside_transaction(): + transaction = current_transaction() + record_custom_metrics([("CustomMetrics/InsideTransaction/Count", 1)]) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetrics/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetrics/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metrics_outside_transaction_with_app(): + app = application() + record_custom_metrics([("CustomMetrics/OutsideTransaction/Count", 1)], application=app) diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 4db6d2dab..3ded79af7 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -32,6 +32,9 @@ from newrelic.api.background_task import BackgroundTask, background_task from newrelic.api.time_trace import current_trace from newrelic.api.transaction import ( + accept_distributed_trace_headers, + accept_distributed_trace_payload, + create_distributed_trace_payload, current_span_id, current_trace_id, current_transaction, @@ -185,10 +188,10 @@ def _test(): payload["d"]["pa"] = "5e5733a911cfbc73" if accept_payload: - result = txn.accept_distributed_trace_payload(payload) + result = accept_distributed_trace_payload(payload) assert result else: - txn._create_distributed_trace_payload() + create_distributed_trace_payload() try: raise ValueError("cookies") @@ -319,7 +322,6 @@ def _test(): ) @override_application_settings(_override_settings) def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, newrelic, metrics): - headers = [] if traceparent: headers.append(("traceparent", TRACEPARENT)) @@ -333,8 +335,7 @@ def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, ne ) @background_task(name="test_distributed_tracing_backwards_compatibility") def _test(): - transaction = current_transaction() - transaction.accept_distributed_trace_headers(headers) + accept_distributed_trace_headers(headers) _test() diff --git a/tests/cross_agent/test_w3c_trace_context.py b/tests/cross_agent/test_w3c_trace_context.py index 05f157f7b..893274ce4 100644 --- a/tests/cross_agent/test_w3c_trace_context.py +++ b/tests/cross_agent/test_w3c_trace_context.py @@ -14,88 +14,105 @@ import json import os + import pytest import webtest -from newrelic.packages import six - -from newrelic.api.transaction import current_transaction +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + current_transaction, + insert_distributed_trace_headers, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.fixtures import (override_application_settings, - validate_attributes) from newrelic.common.encoding_utils import W3CTraceState -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from newrelic.common.object_wrapper import transient_function_wrapper +from newrelic.packages import six CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ('test_name', 'trusted_account_key', 'account_id', - 'web_transaction', 'raises_exception', 'force_sampled_true', - 'span_events_enabled', 'transport_type', 'inbound_headers', - 'outbound_payloads', 'intrinsics', 'expected_metrics') - -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = ( + "test_name", + "trusted_account_key", + "account_id", + "web_transaction", + "raises_exception", + "force_sampled_true", + "span_events_enabled", + "transport_type", + "inbound_headers", + "outbound_payloads", + "intrinsics", + "expected_metrics", +) + +_parameters = ",".join(_parameters_list) XFAIL_TESTS = [ - 'spans_disabled_root', - 'missing_traceparent', - 'missing_traceparent_and_tracestate', - 'w3c_and_newrelc_headers_present_error_parsing_traceparent' + "spans_disabled_root", + "missing_traceparent", + "missing_traceparent_and_tracestate", + "w3c_and_newrelc_headers_present_error_parsing_traceparent", ] + def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'trace_context.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "trace_context.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result ATTR_MAP = { - 'traceparent.version': 0, - 'traceparent.trace_id': 1, - 'traceparent.parent_id': 2, - 'traceparent.trace_flags': 3, - 'tracestate.version': 0, - 'tracestate.parent_type': 1, - 'tracestate.parent_account_id': 2, - 'tracestate.parent_application_id': 3, - 'tracestate.span_id': 4, - 'tracestate.transaction_id': 5, - 'tracestate.sampled': 6, - 'tracestate.priority': 7, - 'tracestate.timestamp': 8, - 'tracestate.tenant_id': None, + "traceparent.version": 0, + "traceparent.trace_id": 1, + "traceparent.parent_id": 2, + "traceparent.trace_flags": 3, + "tracestate.version": 0, + "tracestate.parent_type": 1, + "tracestate.parent_account_id": 2, + "tracestate.parent_application_id": 3, + "tracestate.span_id": 4, + "tracestate.transaction_id": 5, + "tracestate.sampled": 6, + "tracestate.priority": 7, + "tracestate.timestamp": 8, + "tracestate.tenant_id": None, } def validate_outbound_payload(actual, expected, trusted_account_key): - traceparent = '' - tracestate = '' + traceparent = "" + tracestate = "" for key, value in actual: - if key == 'traceparent': - traceparent = value.split('-') - elif key == 'tracestate': + if key == "traceparent": + traceparent = value.split("-") + elif key == "tracestate": vendors = W3CTraceState.decode(value) - nr_entry = vendors.pop(trusted_account_key + '@nr', '') - tracestate = nr_entry.split('-') - exact_values = expected.get('exact', {}) - expected_attrs = expected.get('expected', []) - unexpected_attrs = expected.get('unexpected', []) - expected_vendors = expected.get('vendors', []) + nr_entry = vendors.pop(trusted_account_key + "@nr", "") + tracestate = nr_entry.split("-") + exact_values = expected.get("exact", {}) + expected_attrs = expected.get("expected", []) + unexpected_attrs = expected.get("unexpected", []) + expected_vendors = expected.get("vendors", []) for key, value in exact_values.items(): - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: if isinstance(value, bool): @@ -106,13 +123,13 @@ def validate_outbound_payload(actual, expected, trusted_account_key): assert header[attr] == str(value) for key in expected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert header[attr], key for key in unexpected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert not header[attr], key @@ -125,127 +142,129 @@ def validate_outbound_payload(actual, expected, trusted_account_key): def target_wsgi_application(environ, start_response): transaction = current_transaction() - if not environ['.web_transaction']: + if not environ[".web_transaction"]: transaction.background_task = True - if environ['.raises_exception']: + if environ[".raises_exception"]: try: raise ValueError("oops") except: transaction.notice_error() - if '.inbound_headers' in environ: - transaction.accept_distributed_trace_headers( - environ['.inbound_headers'], - transport_type=environ['.transport_type'], + if ".inbound_headers" in environ: + accept_distributed_trace_headers( + environ[".inbound_headers"], + transport_type=environ[".transport_type"], ) payloads = [] - for _ in range(environ['.outbound_calls']): + for _ in range(environ[".outbound_calls"]): payloads.append([]) - transaction.insert_distributed_trace_headers(payloads[-1]) + insert_distributed_trace_headers(payloads[-1]) - start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps(payloads).encode('utf-8')] + start_response("200 OK", [("Content-Type", "application/json")]) + return [json.dumps(payloads).encode("utf-8")] test_application = webtest.TestApp(target_wsgi_application) def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @pytest.mark.parametrize(_parameters, load_tests()) -def test_trace_context(test_name, trusted_account_key, account_id, - web_transaction, raises_exception, force_sampled_true, - span_events_enabled, transport_type, inbound_headers, - outbound_payloads, intrinsics, expected_metrics): - +def test_trace_context( + test_name, + trusted_account_key, + account_id, + web_transaction, + raises_exception, + force_sampled_true, + span_events_enabled, + transport_type, + inbound_headers, + outbound_payloads, + intrinsics, + expected_metrics, +): if test_name in XFAIL_TESTS: pytest.xfail("Waiting on cross agent tests update.") # Prepare assertions if not intrinsics: intrinsics = {} - common = intrinsics.get('common', {}) - common_required = common.get('expected', []) - common_forgone = common.get('unexpected', []) - common_exact = common.get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common = intrinsics.get("common", {}) + common_required = common.get("expected", []) + common_forgone = common.get("unexpected", []) + common_exact = common.get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } extra_environ = { - '.web_transaction': web_transaction, - '.raises_exception': raises_exception, - '.transport_type': transport_type, - '.outbound_calls': outbound_payloads and len(outbound_payloads) or 0, + ".web_transaction": web_transaction, + ".raises_exception": raises_exception, + ".transport_type": transport_type, + ".outbound_calls": outbound_payloads and len(outbound_payloads) or 0, } inbound_headers = inbound_headers and inbound_headers[0] or None - if transport_type != 'HTTP': - extra_environ['.inbound_headers'] = inbound_headers + if transport_type != "HTTP": + extra_environ[".inbound_headers"] = inbound_headers inbound_headers = None elif six.PY2 and inbound_headers: - inbound_headers = { - k.encode('utf-8'): v.encode('utf-8') - for k, v in inbound_headers.items()} - - @validate_transaction_metrics(test_name, - group="Uri", - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + inbound_headers = {k.encode("utf-8"): v.encode("utf-8") for k, v in inbound_headers.items()} + + @validate_transaction_metrics( + test_name, group="Uri", rollup_metrics=expected_metrics, background_task=not web_transaction + ) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_application_settings(override_settings) @override_compute_sampled(force_sampled_true) def _test(): return test_application.get( - '/' + test_name, + "/" + test_name, headers=inbound_headers, extra_environ=extra_environ, ) - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) response = _test() - assert response.status == '200 OK' + assert response.status == "200 OK" payloads = response.json if outbound_payloads: assert len(payloads) == len(outbound_payloads) From 3bdb013a8c093481c26f4cb8af4c09d507f301b4 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Fri, 30 Jun 2023 09:43:45 -0700 Subject: [PATCH 07/32] Add tests for jinja2. (#842) * Add tests for jinja2. * [Mega-Linter] Apply linters fixes * Update tox.ini Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> --------- Co-authored-by: umaannamalai Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: Timothy Pansino <11214426+TimPansino@users.noreply.github.com> --- tests/template_jinja2/conftest.py | 30 ++++++++++++++++++++ tests/template_jinja2/test_jinja2.py | 41 ++++++++++++++++++++++++++++ tox.ini | 3 ++ 3 files changed, 74 insertions(+) create mode 100644 tests/template_jinja2/conftest.py create mode 100644 tests/template_jinja2/test_jinja2.py diff --git a/tests/template_jinja2/conftest.py b/tests/template_jinja2/conftest.py new file mode 100644 index 000000000..a6922078d --- /dev/null +++ b/tests/template_jinja2/conftest.py @@ -0,0 +1,30 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (template_jinja2)", default_settings=_default_settings +) diff --git a/tests/template_jinja2/test_jinja2.py b/tests/template_jinja2/test_jinja2.py new file mode 100644 index 000000000..c64dac923 --- /dev/null +++ b/tests/template_jinja2/test_jinja2.py @@ -0,0 +1,41 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Template +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + + +@validate_transaction_metrics( + "test_render", + background_task=True, + scoped_metrics=( + ("Template/Render/