ci: migrate to Sauce Connect 5 (closes #754)

* Upgrade from Sauce Connect 4.9.1 before end-of-life on 5th May 2025.
* Add SQLALCHEMY_WARN_20 environment variable to CI and install docs.
* Also switch from OpenSearch 2.11.0 to 2.15.0 in CI and install docs.

.github/workflows/ci.yml

@@ -54,7 +54,7 @@ jobs:
 
       matrix:
         postgres-version: [ 14 ]
-        os-version: [ '2.11.0' ]
+        os-version: [ '2.15.0' ]
         python-version: [ '3.9' ]
 
     # Service containers to run with `runner-job`
@@ -139,18 +139,23 @@ jobs:
         python -m pip install --force-reinstall -r requirements.txt
         python -m pip install -e .[tests]
     - name: Initialise hepdata
+      env:
+        SQLALCHEMY_WARN_20: 1
       run: |
         hepdata db init
         hepdata db create
         hepdata utils reindex -rc True
     - name: Setup npm and build assets
+      env:
+        SQLALCHEMY_WARN_20: 1
       run: |
         hepdata collect -v
         hepdata webpack buildall
         cp hepdata/config_local.gh.py hepdata/config_local.py
     - name: Run tests
       env:
         COVERAGE_FILE: '.coverage_func'
+        SQLALCHEMY_WARN_20: 1
       run: |
         py.test -vv tests/*_test.py
     - name: Setup Sauce Connect
@@ -161,14 +166,16 @@ jobs:
         accessKey: ${{ secrets.SAUCE_ACCESS_KEY }}
         region: eu-central
         tunnelName: ${{ github.run_id }}
-        scVersion: 4.9.1
+        proxy-localhost: direct
+        scVersion: 5.2.2
         verbose: true
     - name: Run end-to-end tests
       if: startsWith(matrix.python-version, '3.9')
       env:
         SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
         SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
         COVERAGE_FILE: '.coverage_e2e'
+        SQLALCHEMY_WARN_20: 1
       run: |
         if [[ -n ${{ secrets.SAUCE_USERNAME }} && -n ${{ secrets.SAUCE_ACCESS_KEY}} ]]; then py.test -vv tests/e2e; fi
     - name: Run coveralls

Dockerfile

@@ -46,7 +46,7 @@ RUN hepdata collect -v  && \
 
 RUN bash -c "echo $APP_ENVIRONMENT"
 
-RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sc-4.9.1-${SAUCE_OS:-linux}.tar.gz && \
+RUN bash -c "set -x; [[ ${APP_ENVIRONMENT:-prod} = local-web ]] && (cd /usr/local/var && wget https://saucelabs.com/downloads/sauce-connect/5.2.2/sauce-connect-5.2.2_${SAUCE_OS:-linux.x86_64}.tar.gz && \
   tar -xvf sc-4.9.1-${SAUCE_OS:-linux}.tar.gz) || echo 'Not installing SC on prod or worker build'"
 
 WORKDIR /code

INSTALL.rst

@@ -36,31 +36,31 @@ for example, using ``yum`` or ``apt-get`` for Linux or ``brew`` for macOS:
 
  * `PostgreSQL <http://www.postgresql.org/>`_ (version 14) database server
  * `Redis <http://redis.io/>`_ for caching
- * `OpenSearch <https://opensearch.org/>`_ (version 2.11.0) for indexing and information retrieval. See below for further instructions.
+ * `OpenSearch <https://opensearch.org/>`_ (version 2.15.0) for indexing and information retrieval. See below for further instructions.
  * `Node.js <https://nodejs.org>`_ (version 18) JavaScript run-time environment and its package manager `npm <https://www.npmjs.com/>`_.
 
-OpenSearch v2.11.0
+OpenSearch v2.15.0
 ------------------
 
-We are currently using OpenSearch v2.11.0. Here, you can find the `download instructions. <https://opensearch.org/versions/opensearch-2-11-0.html>`_
+We are currently using OpenSearch v2.15.0. Here, you can find the `download instructions. <https://opensearch.org/versions/opensearch-2-15-0.html>`_
 
 There are some examples below:
 
 **MacOS**
 
-Install the latest version (currently, v2.11.0) with ``brew install opensearch``.
-Alternatively, to install a specific version like v2.11.0 via Homebrew (if the latest version is newer), run:
+Install the latest version (currently, v2.18.0) with ``brew install opensearch``.
+Alternatively, to install a specific version like v2.15.0 via Homebrew (if the latest version is newer), run:
 
 .. code-block:: console
 
     $ brew tap-new opensearch/tap
-    $ brew extract --version=2.11.0 opensearch opensearch/tap
-    $ brew install opensearch/tap/opensearch@2.11.0
-    $ brew services restart opensearch/tap/opensearch@2.11.0
+    $ brew extract --version=2.15.0 opensearch opensearch/tap
+    $ brew install opensearch/tap/opensearch@2.15.0
+    $ brew services restart opensearch/tap/opensearch@2.15.0
 
 **Linux**
 
-You can see the tarball instructions on the OpenSearch installation `webpage. <https://opensearch.org/docs/2.11/install-and-configure/install-opensearch/tar/>`_
+You can see the tarball instructions on the OpenSearch installation `webpage. <https://opensearch.org/docs/2.15/install-and-configure/install-opensearch/tar/>`_
 
 To execute, run this command within the extracted folder.
 
@@ -74,8 +74,8 @@ Alternatively, run OpenSearch after `installing Docker <https://docs.docker.com/
 
 .. code-block:: console
 
-    $ docker pull opensearchproject/opensearch:2.11.0
-    $ docker run -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" opensearchproject/opensearch:2.11.0
+    $ docker pull opensearchproject/opensearch:2.15.0
+    $ docker run -d -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "plugins.security.disabled=true" opensearchproject/opensearch:2.15.0
 
 .. _installation:
 
@@ -114,13 +114,15 @@ reinstall PyYAML to ensure it's built with LibYAML bindings, e.g. on an M1 MacBo
 
    (venv)$ LDFLAGS="-L$(brew --prefix)/lib" CFLAGS="-I$(brew --prefix)/include" pip install --global-option="--with-libyaml" --force pyyaml==5.4.1
 
-The next line sets environment variables to switch Flask to run in development mode.
+The next lines set environment variables to switch Flask to run in development mode,
+and turns on ``RemovedIn20Warning`` deprecation warnings for SQLAlchemy 1.4.
 You may want to set these automatically in your bash or zsh profile.
 
 .. code-block:: console
 
    (venv)$ export FLASK_ENV=development
    (venv)$ export FLASK_DEBUG=1
+   (venv)$ export SQLALCHEMY_WARN_20=1
 
 Use of config_local.py
 ----------------------
@@ -269,10 +271,10 @@ To run the tests locally you have several options:
 1. Run a Sauce Connect tunnel (recommended).  This is used by GitHub Actions CI.
     1. Create a Sauce Labs account, or ask for the HEPData account details.
     2. Log into Sauce Labs, and go to the "Tunnel Proxies" page.
-    3. Follow the instructions there to install Sauce Connect and start a tunnel.
-       Do not name the tunnel with the ``--tunnel-name`` argument.
+    3. Follow the instructions there to `install Sauce Connect <https://docs.saucelabs.com/secure-connections/sauce-connect-5/installation/>`_.
     4. Create the variables ``SAUCE_USERNAME`` and ``SAUCE_ACCESS_KEY`` in your local environment (and add them to your
-       bash or zsh profile).
+       bash or zsh profile).  Also set ``SAUCE_REGION=eu-central``, ``SAUCE_TUNNEL_NAME=${SAUCE_USERNAME}_tunnel_name`` and ``SAUCE_PROXY_LOCALHOST=direct``.
+    5. Start a tunnel with the command ``sc run`` and wait for the message "Sauce Connect is up, you may start your tests".
 
 2. Run Selenium locally using ChromeDriver.  (Some tests are currently failing with this method.)
     1. Install `ChromeDriver <https://chromedriver.chromium.org>`_
@@ -344,9 +346,9 @@ Copy the file ``config_local.docker_compose.py`` to ``config_local.py``.
 In order to run the tests via Sauce Labs, ensure you have the variables ``$SAUCE_USERNAME`` and ``$SAUCE_ACCESS_KEY``
 set in your environment (see :ref:`running-the-tests`) **before** starting the containers.
 
-If using an M1 MacBook, also add ``export SAUCE_OS=linux-arm64`` to your bash or zsh profile. This is necessary to
+If using an M1 MacBook, also add ``export SAUCE_OS=linux.aarch64`` to your bash or zsh profile. This is necessary to
 download the correct `Sauce Connect Proxy
-<https://docs.saucelabs.com/secure-connections/sauce-connect/installation/#downloading-sauce-connect-proxy>`_
+<https://docs.saucelabs.com/secure-connections/sauce-connect-5/installation/>`_
 client.
 
 Start the containers:
@@ -372,7 +374,7 @@ To run the tests:
 
 .. code-block:: console
 
-   $ docker-compose exec web bash -c "/usr/local/var/sc-4.9.1-${SAUCE_OS:-linux}/bin/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central & ./run-tests.sh"
+   $ docker-compose exec web bash -c "/usr/local/var/sauce-connect-5.2.2_${SAUCE_OS:-linux.x86_64}/sc -u $SAUCE_USERNAME -k $SAUCE_ACCESS_KEY --region eu-central -i ${SAUCE_USERNAME}_tunnel_name --proxy-localhost direct & ./run-tests.sh"
 
 .. _docker-compose-tips:
 

docker-compose.yml

@@ -5,7 +5,7 @@ services:
       context: .
       args:
         - "APP_ENVIRONMENT=local-web"
-        - "SAUCE_OS=${SAUCE_OS:-linux}"
+        - "SAUCE_OS=${SAUCE_OS:-linux.x86_64}"
     command: hepdata run -h 0.0.0.0 -p 5000 --debugger --reload
     ports:
     - "5000:5000"
@@ -22,8 +22,12 @@ services:
     - "APP_SQLALCHEMY_DATABASE_URI=postgresql://hepdata:hepdata@db/hepdata"
     - "SAUCE_USERNAME=${SAUCE_USERNAME}"
     - "SAUCE_ACCESS_KEY=${SAUCE_ACCESS_KEY}"
+    - "SAUCE_REGION=eu-central"
+    - "SAUCE_TUNNEL_NAME=${SAUCE_USERNAME}_tunnel_name"
+    - "SAUCE_PROXY_LOCALHOST=direct"
     - "FLASK_ENV=development"
     - "FLASK_DEBUG=1"
+    - "SQLALCHEMY_WARN_20=1"
     read_only: false
     volumes:
     - ".:/code"
@@ -59,7 +63,7 @@ services:
     - "5432:5432"
     read_only: false
   os:
-    image: opensearchproject/opensearch:2.11.0
+    image: opensearchproject/opensearch:2.15.0
     read_only: false
     environment:
       - node.name=os01

docker/db/Dockerfile

@@ -1,4 +1,4 @@
-FROM postgres:12
+FROM postgres:14
 
 # Custom initialization scripts
 COPY ./create_test_db.sh     /docker-entrypoint-initdb.d/20-create_test_db.sh

tests/e2e/conftest.py

@@ -199,16 +199,17 @@ def finalizer():
     if not RUN_SELENIUM_LOCALLY:
         remote_url = "https://ondemand.eu-central-1.saucelabs.com:443/wd/hub"
         options = webdriver.ChromeOptions()
-        options.browser_version = '114'
-        options.platform_name = 'Windows 10'
+        options.browser_version = '131'
+        options.platform_name = 'Windows 11'
+        local_tunnel_name = os.environ.get('SAUCE_USERNAME', '') + '_tunnel_name'
         sauce_options = {
             'extendedDebugging': True,
             'screenResolution': '1280x1024',
             'name': request.node.name,
             'build': os.environ.get('GITHUB_RUN_ID', datetime.utcnow().strftime("%Y-%m-%d %H:00ish")),
             'username': os.environ.get('SAUCE_USERNAME', ''),
             'accessKey': os.environ.get('SAUCE_ACCESS_KEY', ''),
-            'tunnelName': os.environ.get('GITHUB_RUN_ID', ''),
+            'tunnelName': os.environ.get('GITHUB_RUN_ID', local_tunnel_name),
         }
 
         for key in ['username', 'accessKey']: