From 5aa4b11c501170be8fbec803244d0ae5516f411a Mon Sep 17 00:00:00 2001 From: Giovanni M Guidini <99758426+giovanni-guidini@users.noreply.github.com> Date: Fri, 10 May 2024 14:37:52 +0200 Subject: [PATCH] upgrade to 3.12 (#483) * Copy changes from ba7e2b78bf3603f53ced9e8efeef844be1230689 Chaging the base image (already updated to python 3.12) * upgrade packages until the project builds and tests run * fix tests and make them pass * improve pytest-cov performance * update pytest and its plugins --- Makefile | 6 +- api/internal/tests/views/test_repo_view.py | 4 +- codecov/settings_prod.py | 6 - codecov_auth/tests/test_admin.py | 6 +- core/tests/test_admin.py | 2 +- dev.sh | 2 +- docker-compose.yml | 3 + docker/Dockerfile-proxy | 14 +- docker/Dockerfile.requirements | 3 +- .../test_delete_component_measurements.py | 2 +- .../tests/mutation/test_delete_flag.py | 2 +- requirements.in | 7 +- requirements.txt | 124 +++++++++--------- upload/tests/test_upload_download.py | 4 +- 14 files changed, 96 insertions(+), 89 deletions(-) diff --git a/Makefile b/Makefile index acd439881c..3e1f2a0ffc 100644 --- a/Makefile +++ b/Makefile @@ -31,13 +31,13 @@ check-for-migration-conflicts: python manage.py check_for_migration_conflicts test: - python -m pytest --cov=./ --junitxml=junit.xml + COVERAGE_CORE=sysmon python -m pytest --cov=./ --junitxml=junit.xml test.unit: - python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml + COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "not integration" --cov-report=xml:unit.coverage.xml --junitxml=unit.junit.xml test.integration: - python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml + COVERAGE_CORE=sysmon python -m pytest --cov=./ -m "integration" --cov-report=xml:integration.coverage.xml --junitxml=integration.junit.xml lint: make lint.install diff --git a/api/internal/tests/views/test_repo_view.py b/api/internal/tests/views/test_repo_view.py index 1bf9fb6c93..b34aebf9ab 100644 --- a/api/internal/tests/views/test_repo_view.py +++ b/api/internal/tests/views/test_repo_view.py @@ -1040,7 +1040,7 @@ def test_retrieve_returns_latest_commit_data(self, mocked_get_permissions): response.data["latest_commit"]["report"]["totals"] == expected_commit_payload["report"]["totals"] ) - self.assertEquals( + self.assertEqual( response.data["latest_commit"]["report"]["files"], [ { @@ -1181,7 +1181,7 @@ def side_effect(path, *args, **kwargs): response.data["latest_commit"]["report"]["totals"] == expected_commit_payload["report"]["totals"] ) - self.assertEquals( + self.assertEqual( response.data["latest_commit"]["report"]["files"], [ { diff --git a/codecov/settings_prod.py b/codecov/settings_prod.py index 79303d243e..8050006770 100644 --- a/codecov/settings_prod.py +++ b/codecov/settings_prod.py @@ -1,11 +1,5 @@ import os -import sentry_sdk -from sentry_sdk.integrations.celery import CeleryIntegration -from sentry_sdk.integrations.django import DjangoIntegration -from sentry_sdk.integrations.httpx import HttpxIntegration -from sentry_sdk.integrations.redis import RedisIntegration - from .settings_base import * DEBUG = False diff --git a/codecov_auth/tests/test_admin.py b/codecov_auth/tests/test_admin.py index f63618b51f..2afb74596d 100644 --- a/codecov_auth/tests/test_admin.py +++ b/codecov_auth/tests/test_admin.py @@ -125,7 +125,7 @@ def test_prev_and_new_values_in_log_entry(self, mocked_super_log_change): message = [] message.append({"changed": {"fields": ["staff"]}}) self.owner_admin.log_change(MagicMock, owner, message) - assert mocked_super_log_change.called_once() + mocked_super_log_change.assert_called_once() assert message == [ {"changed": {"fields": ["staff"]}}, {"staff": "prev value: True, new value: False"}, @@ -229,7 +229,7 @@ def test_org_token_refresh_request_calls_service_to_refresh_token( "_continue": ["Save and continue editing"], } response = self.client.post(request_url, data=fake_data) - assert mock_refresh.called_with(str(org_token.id)) + mock_refresh.assert_called_with(str(org_token.id)) @patch( "codecov_auth.services.org_level_token_service.OrgLevelTokenService.refresh_token" @@ -265,7 +265,7 @@ def test_org_token_request_doesnt_call_service_to_refresh_token(self, mock_refre "_continue": ["Save and continue editing"], } response = self.client.post(request_url, data=fake_data) - assert mock_refresh.not_called() + mock_refresh.assert_not_called() def test_start_trial_ui_display(self): owner = OwnerFactory() diff --git a/core/tests/test_admin.py b/core/tests/test_admin.py index 6b9018a9fa..6ecb682527 100644 --- a/core/tests/test_admin.py +++ b/core/tests/test_admin.py @@ -47,7 +47,7 @@ def test_prev_and_new_values_in_log_entry(self, mocked_super_log_change): message = [] message.append({"changed": {"fields": ["using_integration"]}}) self.repo_admin.log_change(MagicMock, repo, message) - assert mocked_super_log_change.called_once() + mocked_super_log_change.assert_called_once() assert message == [ {"changed": {"fields": ["using_integration"]}}, {"using_integration": "prev value: True, new value: False"}, diff --git a/dev.sh b/dev.sh index af3dd22d44..be4b78ffe0 100755 --- a/dev.sh +++ b/dev.sh @@ -15,7 +15,7 @@ _start_gunicorn() { if [[ "$STATSD_HOST" ]]; then suffix="--statsd-host ${STATSD_HOST}:${STATSD_PORT}" fi - if [[ "$RUN_ENV" == "ENTERPRISE" ]] || [[ "$RUN_ENV" == "enterprise" ]] || [[ "$RUN_ENV" == "DEV" ]]; then + if [ "$RUN_ENV" == "ENTERPRISE" ] || [ "$RUN_ENV" == "DEV" ]; then python manage.py migrate python manage.py migrate --database "timeseries" timeseries python manage.py pgpartition --yes --skip-delete diff --git a/docker-compose.yml b/docker-compose.yml index 335f91a47a..7d170ccbcb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,9 @@ services: environment: - RUN_ENV=DEV - SETUP__TIMESERIES__ENABLED=${TIMESERIES_ENABLED-true} + # Improves pytest-cov performance in python 3.12 + # https://github.com/nedbat/coveragepy/issues/1665#issuecomment-1937075835 + - COVERAGE_CORE=sysmon env_file: - .testenv postgres: diff --git a/docker/Dockerfile-proxy b/docker/Dockerfile-proxy index 07e2514266..17c51ad5ec 100644 --- a/docker/Dockerfile-proxy +++ b/docker/Dockerfile-proxy @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.3 -FROM python:3.9.18-slim-bookworm +FROM python:3.12-slim-bookworm ENV FRP_VERSION=v0.51.3 @@ -8,12 +8,12 @@ RUN apt-get update RUN apt-get install -y curl RUN addgroup -S frp \ -&& adduser -D -S -h /var/frp -s /sbin/nologin -G frp frp \ -&& curl -fSL https://github.com/fatedier/frp/releases/download/${FRP_VERSION}/frp_${FRP_VERSION:1}_linux_amd64.tar.gz -o frp.tar.gz \ -&& tar -zxv -f frp.tar.gz \ -&& rm -rf frp.tar.gz \ -&& mv frp_*_linux_amd64 /frp \ -&& chown -R frp:frp /frp + && adduser -D -S -h /var/frp -s /sbin/nologin -G frp frp \ + && curl -fSL https://github.com/fatedier/frp/releases/download/${FRP_VERSION}/frp_${FRP_VERSION:1}_linux_amd64.tar.gz -o frp.tar.gz \ + && tar -zxv -f frp.tar.gz \ + && rm -rf frp.tar.gz \ + && mv frp_*_linux_amd64 /frp \ + && chown -R frp:frp /frp COPY --chown=frp:frp docker/frpc-entrypoint.sh /frp/entrypoint.sh RUN chmod 755 /frp/entrypoint.sh diff --git a/docker/Dockerfile.requirements b/docker/Dockerfile.requirements index 9506560473..f3997575b2 100644 --- a/docker/Dockerfile.requirements +++ b/docker/Dockerfile.requirements @@ -1,9 +1,10 @@ # syntax=docker/dockerfile:1.4 -ARG PYTHON_IMAGE=python:3.9.18-slim-bookworm +ARG PYTHON_IMAGE=python:3.12-slim-bookworm # BUILD STAGE - Download dependencies from GitHub that require SSH access FROM $PYTHON_IMAGE as build + # Pinning a specific nightly version so that builds don't suddenly break if a # "this feature is now stabilized" warning is promoted to an error or something. # We would like to keep up with nightly if we can. diff --git a/graphql_api/tests/mutation/test_delete_component_measurements.py b/graphql_api/tests/mutation/test_delete_component_measurements.py index 0ba08ac5af..0e090a0bb3 100644 --- a/graphql_api/tests/mutation/test_delete_component_measurements.py +++ b/graphql_api/tests/mutation/test_delete_component_measurements.py @@ -42,7 +42,7 @@ def test_delete_component_measurements(self, execute_mock): assert data == {"deleteComponentMeasurements": None} - assert execute_mock.called_once_with( + execute_mock.assert_called_once_with( owner_username="test-owner", repo_name="test-repo", component_id="test-component", diff --git a/graphql_api/tests/mutation/test_delete_flag.py b/graphql_api/tests/mutation/test_delete_flag.py index 21ef7bed94..a2522ed5e5 100644 --- a/graphql_api/tests/mutation/test_delete_flag.py +++ b/graphql_api/tests/mutation/test_delete_flag.py @@ -40,7 +40,7 @@ def test_delete_flag(self, execute_mock): assert data == {"deleteFlag": None} - assert execute_mock.called_once_with( + execute_mock.assert_called_once_with( owner_username="test-owner", repo_name="test-repo", flag_name="test-flag", diff --git a/requirements.in b/requirements.in index fc607f5da9..f0a23e495f 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,7 @@ aiodataloader ariadne ariadne_django -celery[redis] +celery>=5.3.6 cerberus ddtrace Django @@ -26,8 +26,8 @@ gunicorn>=22.0.0 https://github.com/photocrowd/django-cursor-pagination/archive/f560902696b0c8509e4d95c10ba0d62700181d84.tar.gz idna>=3.7 minio -opentelemetry-instrumentation-django>=0.41b0 -opentelemetry-sdk>=1.20.0 +opentelemetry-instrumentation-django>=0.45b0 +opentelemetry-sdk>=1.24.0 opentracing pre-commit psycopg2 @@ -45,6 +45,7 @@ redis regex requests sentry-sdk>=1.40.0 +sentry-sdk[celery] setproctitle simplejson stripe diff --git a/requirements.txt b/requirements.txt index 5fc2b42f3e..93a63eef62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.12 # by the following command: # # pip-compile requirements.in # -aiodataloader==0.2.0 +aiodataloader==0.4.0 # via -r requirements.in -amqp==5.1.0 +amqp==5.2.0 # via kombu analytics-python==1.3.0b1 # via shared @@ -27,10 +27,13 @@ asgiref==3.6.0 async-timeout==4.0.2 # via redis attrs==20.3.0 - # via jsonschema + # via + # cattrs + # ddtrace + # jsonschema backoff==1.6.0 # via analytics-python -billiard==3.6.4.0 +billiard==4.2.0 # via celery boto3==1.20.25 # via shared @@ -38,12 +41,18 @@ botocore==1.23.25 # via # boto3 # s3transfer +bytecode==0.15.1 + # via ddtrace cachetools==4.1.1 # via # google-auth # shared -celery[redis]==5.2.7 - # via -r requirements.in +cattrs==23.1.2 + # via ddtrace +celery==5.3.6 + # via + # -r requirements.in + # sentry-sdk cerberus==1.3.2 # via # -r requirements.in @@ -65,7 +74,7 @@ cfgv==3.2.0 # via pre-commit charset-normalizer==2.0.9 # via requests -click==8.0.4 +click==8.1.7 # via # celery # click-didyoumean @@ -83,13 +92,15 @@ codecovopentelem @ https://github.com/codecov/opentelem-python/archive/refs/tags # via -r requirements.in colour==0.1.5 # via shared -coverage==6.0.1 +coverage[toml]==7.5.1 # via # codecovopentelem # pytest-cov cryptography==42.0.5 # via shared -ddtrace==0.46.0 +ddsketch==3.0.1 + # via ddtrace +ddtrace==2.7.6 # via -r requirements.in deprecated==1.2.12 # via opentelemetry-api @@ -149,8 +160,8 @@ ecdsa==0.18.0 # via tlslite-ng elastic-apm==6.13.2 # via -r requirements.in -exceptiongroup==1.2.1 - # via pytest +envier==0.5.1 + # via ddtrace factory-boy==3.2.0 # via -r requirements.in faker==4.1.3 @@ -163,7 +174,6 @@ freezegun==1.1.0 # via -r requirements.in google-api-core[grpc]==2.11.1 # via - # google-api-core # google-cloud-core # google-cloud-pubsub # google-cloud-storage @@ -194,7 +204,7 @@ graphql-core==3.2.3 # via ariadne grpc-google-iam-v1==0.12.6 # via google-cloud-pubsub -grpcio==1.58.0 +grpcio==1.62.1 # via # google-api-core # google-cloud-pubsub @@ -232,15 +242,13 @@ inflection==0.5.1 # via drf-spectacular iniconfig==1.1.1 # via pytest -intervaltree==3.1.0 - # via ddtrace jmespath==0.10.0 # via # boto3 # botocore jsonschema==4.14.0 # via drf-spectacular -kombu==5.2.4 +kombu==5.3.6 # via celery minio==7.1.13 # via @@ -258,30 +266,31 @@ oauth2==1.9.0.post1 # via shared oauthlib==3.1.0 # via shared -opentelemetry-api==1.20.0 +opentelemetry-api==1.24.0 # via + # ddtrace # opentelemetry-instrumentation # opentelemetry-instrumentation-django # opentelemetry-instrumentation-wsgi # opentelemetry-sdk -opentelemetry-instrumentation==0.41b0 +opentelemetry-instrumentation==0.45b0 # via # opentelemetry-instrumentation-django # opentelemetry-instrumentation-wsgi -opentelemetry-instrumentation-django==0.41b0 +opentelemetry-instrumentation-django==0.45b0 # via -r requirements.in -opentelemetry-instrumentation-wsgi==0.41b0 +opentelemetry-instrumentation-wsgi==0.45b0 # via opentelemetry-instrumentation-django -opentelemetry-sdk==1.20.0 +opentelemetry-sdk==1.24.0 # via # -r requirements.in # codecovopentelem -opentelemetry-semantic-conventions==0.41b0 +opentelemetry-semantic-conventions==0.45b0 # via # opentelemetry-instrumentation-django # opentelemetry-instrumentation-wsgi # opentelemetry-sdk -opentelemetry-util-http==0.41b0 +opentelemetry-util-http==0.45b0 # via # opentelemetry-instrumentation-django # opentelemetry-instrumentation-wsgi @@ -291,7 +300,7 @@ packaging==20.9 # via # gunicorn # pytest -pluggy==0.13.1 +pluggy==1.5.0 # via pytest pre-commit==2.11.1 # via -r requirements.in @@ -333,39 +342,39 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -pytest==7.4.2 +pytest==8.1.1 # via # -r requirements.in # pytest-asyncio # pytest-cov # pytest-django # pytest-mock -pytest-asyncio==0.18.2 +pytest-asyncio==0.23.6 # via -r requirements.in -pytest-cov==2.12.1 +pytest-cov==5.0.0 # via -r requirements.in -pytest-django==4.7.0 +pytest-django==4.8.0 # via -r requirements.in -pytest-mock==1.12.1 +pytest-mock==3.14.0 # via -r requirements.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements.in # analytics-python # botocore + # celery # django-postgres-extra # faker # freezegun -python-json-logger==0.1.11 +python-json-logger==2.0.7 # via -r requirements.in -python-redis-lock==3.7.0 +python-redis-lock==4.0.0 # via # -r requirements.in # shared pytz==2022.1 # via # -r requirements.in - # celery # djangorestframework # shared pyyaml==6.0.1 @@ -376,7 +385,6 @@ pyyaml==6.0.1 redis==4.4.4 # via # -r requirements.in - # celery # fakeredis # python-redis-lock # shared @@ -392,14 +400,12 @@ requests==2.31.0 # shared # stripe rfc3986[idna2008]==1.4.0 - # via - # httpx - # rfc3986 + # via httpx rsa==4.7.2 # via google-auth s3transfer==0.5.0 # via boto3 -sentry-sdk==1.40.4 +sentry-sdk[celery]==1.44.1 # via # -r requirements.in # shared @@ -409,16 +415,16 @@ shared @ https://github.com/codecov/shared/archive/f6c2c3852530192ab0c6b9fd0c0a8 # via -r requirements.in simplejson==3.17.2 # via -r requirements.in -six==1.15.0 +six==1.16.0 # via # analytics-python # click-repl + # ddsketch + # ddtrace # django-dynamic-fixture # ecdsa # google-auth # python-dateutil - # tenacity - # vcrpy # virtualenv sniffio==1.2.0 # via @@ -426,39 +432,37 @@ sniffio==1.2.0 # httpcore # httpx sortedcontainers==2.4.0 - # via - # fakeredis - # intervaltree + # via fakeredis sqlalchemy==1.4.50 # via shared sqlparse==0.5.0 - # via django + # via + # ddtrace + # django starlette==0.36.2 # via ariadne statsd==3.3.0 # via shared -stripe==2.55.2 +stripe==8.9.0 # via -r requirements.in -tenacity==6.2.0 - # via ddtrace text-unidecode==1.3 # via faker tlslite-ng==0.8.0b1 # via shared toml==0.10.2 - # via - # pre-commit - # pytest-cov -tomli==2.0.1 - # via pytest + # via pre-commit typing==3.7.4.3 # via shared typing-extensions==4.6.2 # via + # aiodataloader # ariadne + # ddtrace # opentelemetry-sdk # shared - # starlette + # stripe +tzdata==2024.1 + # via celery uritemplate==4.1.1 # via drf-spectacular urllib3==1.26.18 @@ -471,9 +475,9 @@ urllib3==1.26.18 # requests # sentry-sdk # shared -vcrpy==2.0.1 +vcrpy==6.0.1 # via -r requirements.in -vine==5.0.0 +vine==5.1.0 # via # amqp # celery @@ -484,13 +488,15 @@ wcwidth==0.2.5 # via prompt-toolkit whitenoise==5.2.0 # via -r requirements.in -wrapt==1.14.1 +wrapt==1.16.0 # via # deprecated # elastic-apm # opentelemetry-instrumentation # vcrpy -yarl==1.5.1 +xmltodict==0.13.0 + # via ddtrace +yarl==1.9.4 # via vcrpy zipp==3.17.0 # via importlib-metadata diff --git a/upload/tests/test_upload_download.py b/upload/tests/test_upload_download.py index 486aac85b8..ee3e3cba73 100644 --- a/upload/tests/test_upload_download.py +++ b/upload/tests/test_upload_download.py @@ -109,7 +109,9 @@ def test_valid_repo_archive_path(self, create_presigned_get, get_archive_hash): assert response.status_code == 302 headers = response.headers assert headers["location"] == "presigned-url" - assert create_presigned_get.called_once_with("archive", "v4/raw/hasssshhh", 30) + create_presigned_get.assert_called_once_with( + "archive", "v4/raw/hasssshhh", expires=30 + ) @patch("services.archive.ArchiveService.read_file") def test_invalid_repo_archive_path(self, mock_read_file):