From fd61a207cd3d0186dda2ec9eb6fef6ea0da26b41 Mon Sep 17 00:00:00 2001 From: Graeme Watt Date: Wed, 11 Dec 2024 15:46:37 +0000 Subject: [PATCH] ci: migrate to Sauce Connect 5 (closes #754) * Upgrade from Sauce Connect 4.9.1 before end-of-life on 5th May 2025. * Add SQLALCHEMY_WARN_20 environment variable to CI and install docs. * Also switch from OpenSearch 2.11.0 to 2.15.0 in CI and install docs. --- .github/workflows/ci.yml | 11 +++++++++-- Dockerfile | 2 +- INSTALL.rst | 38 ++++++++++++++++++++------------------ docker-compose.yml | 8 ++++++-- docker/db/Dockerfile | 2 +- tests/e2e/conftest.py | 7 ++++--- 6 files changed, 41 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c73e3248d..938268524 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,7 +54,7 @@ jobs: matrix: postgres-version: [ 14 ] - os-version: [ '2.11.0' ] + os-version: [ '2.15.0' ] python-version: [ '3.9' ] # Service containers to run with `runner-job` @@ -139,11 +139,15 @@ jobs: python -m pip install --force-reinstall -r requirements.txt python -m pip install -e .[tests] - name: Initialise hepdata + env: + SQLALCHEMY_WARN_20: 1 run: | hepdata db init hepdata db create hepdata utils reindex -rc True - name: Setup npm and build assets + env: + SQLALCHEMY_WARN_20: 1 run: | hepdata collect -v hepdata webpack buildall @@ -151,6 +155,7 @@ jobs: - name: Run tests env: COVERAGE_FILE: '.coverage_func' + SQLALCHEMY_WARN_20: 1 run: | py.test -vv tests/*_test.py - name: Setup Sauce Connect @@ -161,7 +166,8 @@ jobs: accessKey: ${{ secrets.SAUCE_ACCESS_KEY }} region: eu-central tunnelName: ${{ github.run_id }} - scVersion: 4.9.1 + proxy-localhost: direct + scVersion: 5.2.2 verbose: true - name: Run end-to-end tests if: startsWith(matrix.python-version, '3.9') @@ -169,6 +175,7 @@ jobs: SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }} SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }} COVERAGE_FILE: '.coverage_e2e' + SQLALCHEMY_WARN_20: 1 run: | if [[ -n ${{ secrets.SAUCE_USERNAME }} && -n ${{ secrets.SAUCE_ACCESS_KEY}} ]]; then py.test -vv tests/e2e; fi - name: Run coveralls diff --git a/Dockerfile b/Dockerfile index 3a1f3b938..f5a675caf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,7 @@ RUN hepdata collect -v && \ RUN bash -c "echo $APP_ENVIRONMENT" -RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sc-4.9.1-${SAUCE_OS:-linux}.tar.gz && \ +RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sauce-connect/5.2.2/sauce-connect-5.2.2_${SAUCE_OS:-linux.x86_64}.tar.gz && \ tar -xvf sc-4.9.1-${SAUCE_OS:-linux}.tar.gz) || echo 'Not installing SC on prod or worker build'" WORKDIR /code diff --git a/INSTALL.rst b/INSTALL.rst index 597e341ed..7603e1fdc 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -36,31 +36,31 @@ for example, using ``yum`` or ``apt-get`` for Linux or ``brew`` for macOS: * `PostgreSQL `_ (version 14) database server * `Redis `_ for caching - * `OpenSearch `_ (version 2.11.0) for indexing and information retrieval. See below for further instructions. + * `OpenSearch `_ (version 2.15.0) for indexing and information retrieval. See below for further instructions. * `Node.js `_ (version 18) JavaScript run-time environment and its package manager `npm `_. -OpenSearch v2.11.0 +OpenSearch v2.15.0 ------------------ -We are currently using OpenSearch v2.11.0. Here, you can find the `download instructions. `_ +We are currently using OpenSearch v2.15.0. Here, you can find the `download instructions. `_ There are some examples below: **MacOS** -Install the latest version (currently, v2.11.0) with ``brew install opensearch``. -Alternatively, to install a specific version like v2.11.0 via Homebrew (if the latest version is newer), run: +Install the latest version (currently, v2.18.0) with ``brew install opensearch``. +Alternatively, to install a specific version like v2.15.0 via Homebrew (if the latest version is newer), run: .. code-block:: console $ brew tap-new opensearch/tap - $ brew extract --version=2.11.0 opensearch opensearch/tap - $ brew install opensearch/tap/opensearch@2.11.0 - $ brew services restart opensearch/tap/opensearch@2.11.0 + $ brew extract --version=2.15.0 opensearch opensearch/tap + $ brew install opensearch/tap/opensearch@2.15.0 + $ brew services restart opensearch/tap/opensearch@2.15.0 **Linux** -You can see the tarball instructions on the OpenSearch installation `webpage. `_ +You can see the tarball instructions on the OpenSearch installation `webpage. `_ To execute, run this command within the extracted folder. @@ -74,8 +74,8 @@ Alternatively, run OpenSearch after `installing Docker `_. 4. Create the variables ``SAUCE_USERNAME`` and ``SAUCE_ACCESS_KEY`` in your local environment (and add them to your - bash or zsh profile). + bash or zsh profile). Also set ``SAUCE_REGION=eu-central``, ``SAUCE_TUNNEL_NAME=${SAUCE_USERNAME}_tunnel_name`` and ``SAUCE_PROXY_LOCALHOST=direct``. + 5. Start a tunnel with the command ``sc run`` and wait for the message "Sauce Connect is up, you may start your tests". 2. Run Selenium locally using ChromeDriver. (Some tests are currently failing with this method.) 1. Install `ChromeDriver `_ @@ -344,9 +346,9 @@ Copy the file ``config_local.docker_compose.py`` to ``config_local.py``. In order to run the tests via Sauce Labs, ensure you have the variables ``$SAUCE_USERNAME`` and ``$SAUCE_ACCESS_KEY`` set in your environment (see :ref:`running-the-tests`) **before** starting the containers. -If using an M1 MacBook, also add ``export SAUCE_OS=linux-arm64`` to your bash or zsh profile. This is necessary to +If using an M1 MacBook, also add ``export SAUCE_OS=linux.aarch64`` to your bash or zsh profile. This is necessary to download the correct `Sauce Connect Proxy -`_ +`_ client. Start the containers: @@ -372,7 +374,7 @@ To run the tests: .. code-block:: console - $ docker-compose exec web bash -c "/usr/local/var/sc-4.9.1-${SAUCE_OS:-linux}/bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central & ./run-tests.sh" + $ docker-compose exec web bash -c "/usr/local/var/sauce-connect-5.2.2_${SAUCE_OS:-linux.x86_64}/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central -i ${SAUCE_USERNAME}_tunnel_name --proxy-localhost direct & ./run-tests.sh" .. _docker-compose-tips: diff --git a/docker-compose.yml b/docker-compose.yml index 3026c8858..ad3fb44d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,7 @@ services: context: . args: - "APP_ENVIRONMENT=local-web" - - "SAUCE_OS=${SAUCE_OS:-linux}" + - "SAUCE_OS=${SAUCE_OS:-linux.x86_64}" command: hepdata run -h 0.0.0.0 -p 5000 --debugger --reload ports: - "5000:5000" @@ -22,8 +22,12 @@ services: - "APP_SQLALCHEMY_DATABASE_URI=postgresql://hepdata:hepdata@db/hepdata" - "SAUCE_USERNAME=${SAUCE_USERNAME}" - "SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY}" + - "SAUCE_REGION=eu-central" + - "SAUCE_TUNNEL_NAME=${SAUCE_USERNAME}_tunnel_name" + - "SAUCE_PROXY_LOCALHOST=direct" - "FLASK_ENV=development" - "FLASK_DEBUG=1" + - "SQLALCHEMY_WARN_20=1" read_only: false volumes: - ".:/code" @@ -59,7 +63,7 @@ services: - "5432:5432" read_only: false os: - image: opensearchproject/opensearch:2.11.0 + image: opensearchproject/opensearch:2.15.0 read_only: false environment: - node.name=os01 diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile index 0f2dcc6e4..a90ccdf73 100644 --- a/docker/db/Dockerfile +++ b/docker/db/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:12 +FROM postgres:14 # Custom initialization scripts COPY ./create_test_db.sh /docker-entrypoint-initdb.d/20-create_test_db.sh diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index 2321ca5ed..a5c6b9484 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -199,8 +199,9 @@ def finalizer(): if not RUN_SELENIUM_LOCALLY: remote_url = "https://ondemand.eu-central-1.saucelabs.com:443/wd/hub" options = webdriver.ChromeOptions() - options.browser_version = '114' - options.platform_name = 'Windows 10' + options.browser_version = '131' + options.platform_name = 'Windows 11' + local_tunnel_name = os.environ.get('SAUCE_USERNAME', '') + '_tunnel_name' sauce_options = { 'extendedDebugging': True, 'screenResolution': '1280x1024', @@ -208,7 +209,7 @@ def finalizer(): 'build': os.environ.get('GITHUB_RUN_ID', datetime.utcnow().strftime("%Y-%m-%d %H:00ish")), 'username': os.environ.get('SAUCE_USERNAME', ''), 'accessKey': os.environ.get('SAUCE_ACCESS_KEY', ''), - 'tunnelName': os.environ.get('GITHUB_RUN_ID', ''), + 'tunnelName': os.environ.get('GITHUB_RUN_ID', local_tunnel_name), } for key in ['username', 'accessKey']: