Update Dockerfile

.github/workflows/ci-build.yml

@@ -30,8 +30,48 @@ jobs:
           python-version: ${{ matrix.python-version }}
           architecture: x64
 
+      - name: Checkout DataGateway API
+        uses: actions/checkout@v2
+
+      # Install Nox, Poetry and API's dependencies
+      - name: Install Nox
+        run: pip install nox==2020.8.22
+      - name: Install Poetry
+        run: pip install poetry==1.1.9
+
+      # This command is a workaround for getting Poetry working with Python 3.10. An
+      # fix is made in Poetry 1.2.0a2 but there is currently no official release for
+      # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the
+      # experimental installer is a workaround for Poetry 1.1.x
+      # See https://github.com/python-poetry/poetry/issues/4210 for more details
+      - name: Disable Poetry's experimental new installer
+        if: ${{ matrix.python-version == '3.10' }}
+        run: poetry config experimental.new-installer false
+
+      # Prep for using the API for tests
+      - name: Create log file
+        run: touch logs.log
+      - name: Configure log file location
+        run: echo "`yq \
+          '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example
+      - name: Configure datagateway extension
+        run: echo "`yq \
+          '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
+      - name: Create config.yaml
+        run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
+      - name: Create search_api_mapping.json
+        run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
+
+      # Run Unit tests
+      - name: Run Nox unit tests session
+        run: nox -p ${{ matrix.python-version }} -s unit_tests -- --cov=datagateway_api --cov-report=xml
+      - name: Upload unit test code coverage report
+        if: matrix.python-version == '3.6'
+        uses: codecov/codecov-action@v3
+
       # ICAT Ansible clone and install dependencies
       - name: Checkout icat-ansible
+        if: success()
         uses: actions/checkout@v2
         with:
           repository: icatproject-contrib/icat-ansible
@@ -85,46 +125,35 @@ jobs:
       - name: Create log file
         run: touch logs.log
       - name: Configure log file location
-        run: echo "`jq -r --arg REPO_DIR "$GITHUB_WORKSPACE/logs.log" \
-          '.log_location=$REPO_DIR' datagateway_api/config.json.example`" > datagateway_api/config.json.example
-      - name: Create config.json
-        run: cp datagateway_api/config.json.example datagateway_api/config.json
+        run: echo "`yq \
+          '.log_location="${GITHUB_WORKSPACE}/logs.log"' datagateway_api/config.yaml.example | envsubst`" > datagateway_api/config.yaml.example
+      - name: Configure datagateway extension
+        run: echo "`yq \
+          '.datagateway_api.extension="/datagateway_api"' datagateway_api/config.yaml.example`" > datagateway_api/config.yaml.example
+      - name: Create config.yaml
+        run: cp datagateway_api/config.yaml.example datagateway_api/config.yaml
       - name: Create search_api_mapping.json
         run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
 
-      # Install Nox, Poetry and API's dependencies
-      - name: Install Nox
-        run: pip install nox==2020.8.22
-      - name: Install Poetry
-        run: pip install poetry==1.1.9
-
-      # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
       - name: Uninstall setuptools
         run: poetry run pip uninstall -y setuptools
       - name: Install older setuptools
         run: poetry run pip install 'setuptools<58.0.0'
 
-      # This command is a workaround for getting Poetry working with Python 3.10. An
-      # fix is made in Poetry 1.2.0a2 but there is currently no official release for
-      # Poetry 1.2 and am apprehensive to moving to a pre-release. Disabling the
-      # experimental installer is a workaround for Poetry 1.1.x
-      # See https://github.com/python-poetry/poetry/issues/4210 for more details
-      - name: Disable Poetry's experimental new installer
-        run: poetry config experimental.new-installer false
-
       - name: Install dependencies
         run: poetry install
 
       - name: Add dummy data to icatdb
         run: |
           poetry run python -m util.icat_db_generator
 
-      # Run Nox tests session, saves and uploads a coverage report to codecov
-      - name: Run Nox tests session
-        run: nox -p ${{ matrix.python-version }} -s tests -- --cov=datagateway_api --cov-report=xml
-      - name: Upload code coverage report
+      # Run Nox integration tests session, saves and uploads a coverage report to codecov
+      - name: Run Nox Integration tests session
+        if: success()
+        run: nox -p ${{ matrix.python-version }} -s integration_tests -- --cov=datagateway_api --cov-report=xml
+      - name: Upload integration test code coverage report
         if: matrix.python-version == '3.6'
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
 
   linting:
     runs-on: ubuntu-20.04
@@ -246,10 +275,10 @@ jobs:
       - name: Checkout DataGateway API
         uses: actions/checkout@v2
 
-      - name: Create config.json
-        run: cp datagateway_api/config.json.example datagateway_api/config.json
+      - name: Create config.yaml
+        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
       - name: Create search_api_mapping.json
-        run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
+        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
       - name: Install Poetry
         run: pip install poetry==1.1.9
 
@@ -264,7 +293,9 @@ jobs:
 
       - name: Add dummy data to icatdb
         run: poetry run python -m util.icat_db_generator
-      - name: Get SQL dump of dummy data
+      - name: Drop modTime and createTime
+        run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
+      - name: Get SQL dump of dummy data to be prepared
         run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_1.sql
 
       # Drop and re-create icatdb to remove generated data
@@ -279,6 +310,8 @@ jobs:
       - name: Add (new) dummy data to icatdb
         run: |
           cd /home/runner/work/datagateway-api/datagateway-api; poetry run python -m util.icat_db_generator
+      - name: Drop modTime and createTime
+        run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
       - name: Get SQL dump of new dummy data
         run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_2.sql
 
@@ -300,10 +333,10 @@ jobs:
         with:
           ref: ${{ github.event.repository.default_branch }}
 
-      - name: Create config.json
-        run: cp datagateway_api/config.json.example datagateway_api/config.json
+      - name: Create config.yaml
+        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/config.yaml.example datagateway_api/config.yaml
       - name: Create search_api_mapping.json
-        run: cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
+        run: cd /home/runner/work/datagateway-api/datagateway-api; cp datagateway_api/search_api_mapping.json.example datagateway_api/search_api_mapping.json
 
       # Installing an older version of setuptools for reasons explained at: https://github.com/icatproject/python-icat/issues/99
       - name: Uninstall setuptools
@@ -316,6 +349,8 @@ jobs:
 
       - name: Add dummy data to icatdb
         run: poetry run python -m util.icat_db_generator
+      - name: Drop modTime and createTime
+        run: mysql -picatdbuserpw -uicatdbuser icatdb < /home/runner/work/datagateway-api/datagateway-api/util/columns_to_drop.sql
       - name: Get SQL dump of dummy data from main's generator script
         run: mysqldump -picatdbuserpw -uicatdbuser --skip-comments icatdb > ~/generator_script_dump_main.sql
 

.gitignore

@@ -3,6 +3,7 @@ venv/
 *.pyc
 logs.log*
 config.json*
+config.yaml
 search_api_mapping.json*
 .vscode/
 .nox/

CHANGELOG.md

@@ -2,6 +2,59 @@
 
 <!--next-version-placeholder-->
 
+## v6.3.0 (2023-02-17)
+### Feature
+* Refactor icat generator to use python-icat ([`ae6ae0b`](https://github.com/ral-facilities/datagateway-api/commit/ae6ae0b2a2ce375fb0d47851b3091c952735f5c6))
+
+## v6.2.0 (2022-11-23)
+### Feature
+* Add new entity endpoints for icat5 ([`dd18e18`](https://github.com/ral-facilities/datagateway-api/commit/dd18e18bc19f87c5aa7c99c144f262b68e4fe0c6))
+
+### Documentation
+* Updated postman collection with new entities ([`4dd4895`](https://github.com/ral-facilities/datagateway-api/commit/4dd489574245f4f6020c490d5cddeb3fc3c65c91))
+
+## v6.1.1 (2022-11-23)
+### Fix
+* Do not attempt to create related entities if not included #387 ([`819e7a1`](https://github.com/ral-facilities/datagateway-api/commit/819e7a1582d8d108fe5f18b210c08d67cd0a3845))
+
+## v6.1.0 (2022-11-23)
+### Feature
+* Add new entity endpoints for icat5 ([`dc14f9f`](https://github.com/ral-facilities/datagateway-api/commit/dc14f9f8aeb5a9849bc69cf647bf068b04a63d81))
+
+### Documentation
+* Updated postman collection with new entities ([`c4b850f`](https://github.com/ral-facilities/datagateway-api/commit/c4b850f913dbded35e6b1a88c8f8d8e1b36cbdec))
+
+## v6.0.0 (2022-11-22)
+### Feature
+* Enable support for yaml configuration files for DatagatewayAPI ([`46723de`](https://github.com/ral-facilities/datagateway-api/commit/46723de2bf5336244b3dd37808f565e554b5cbdb))
+* Add support for yaml configuration ([`02c3e41`](https://github.com/ral-facilities/datagateway-api/commit/02c3e41c92c721d793b27b44c7b27cfb3af6ffbb))
+
+### Breaking
+* Remove support for json configuration files  ([`46723de`](https://github.com/ral-facilities/datagateway-api/commit/46723de2bf5336244b3dd37808f565e554b5cbdb))
+
+## v5.3.0 (2022-11-16)
+### Feature
+* Add warning for no api ([`3c91635`](https://github.com/ral-facilities/datagateway-api/commit/3c91635bc0c3465d456a4a85c8d6c8793df408c2))
+
+## v5.2.0 (2022-11-03)
+### Feature
+* Add warning that tests only work with ICAT 5 ([`d8825fd`](https://github.com/ral-facilities/datagateway-api/commit/d8825fd71740e2b2d9b387dca175110cee1f9915))
+* Changes to tests so they pass with icat 5 ([`73f3c77`](https://github.com/ral-facilities/datagateway-api/commit/73f3c777f8d3841cf09537140922a9bbdd8dad7c))
+
+### Documentation
+* Update the poetry installation documentation ([`68d4267`](https://github.com/ral-facilities/datagateway-api/commit/68d426722862613358b0b136188d2b61accdc56f))
+
+## v5.1.1 (2022-09-22)
+### Fix
+* Update to latest dependencies ([`11a0eeb`](https://github.com/ral-facilities/datagateway-api/commit/11a0eeb3cbc4b5db2a0fa8dcd825dbeceb6ac111))
+
+## v5.1.0 (2022-05-25)
+### Feature
+* Support skip and limit string parameters on search-api ([`db94b37`](https://github.com/ral-facilities/datagateway-api/commit/db94b375438873969db2d05621d9860a304ca8bd))
+* Support skip and limit string parameters on search-api ([`eb74970`](https://github.com/ral-facilities/datagateway-api/commit/eb74970c1a74c6e0bed62460dab5ba5881ba1df7))
+* Support skip/limit string parameters on search-api and added testing ([`8c3dde1`](https://github.com/ral-facilities/datagateway-api/commit/8c3dde11deb7614dfd22e6433c24727eeff2f800))
+* Support skip/limit string parameters on search-api ([`d601ba7`](https://github.com/ral-facilities/datagateway-api/commit/d601ba7e9b48186e624db4891c503484f8895b2a))
+
 ## v5.0.1 (2022-05-16)
 ### Fix
 * Fix internal server error when running DG API on its own #359 ([`b0d3e06`](https://github.com/ral-facilities/datagateway-api/commit/b0d3e064dc842143de7f7f0a31b947122fae9f88))

Dockerfile

@@ -1,19 +1,48 @@
 # Dockerfile to build and serve datagateway-api
 
-FROM python:3.6-slim-bullseye
+# Build stage
+FROM python:3.11-alpine3.17 as builder
 
-WORKDIR /datagateway-api
+WORKDIR /datagateway-api-build
 
-COPY poetry.lock pyproject.toml ./
+COPY . .
 
-RUN python -m pip install --upgrade pip \
-  && pip install 'poetry==1.1.13' \
-  && poetry run pip uninstall -y setuptools \
-  && poetry run pip install 'setuptools<58.0.0' \
-  && poetry run pip install 'gunicorn==20.1.0' \
-  && poetry install --no-dev
+RUN --mount=type=cache,target=/root/.cache \
+    python3 -m pip install 'poetry~=1.3.2' \
+ && poetry build
 
-COPY datagateway_api ./datagateway_api
+
+# Install & run stage
+FROM python:3.11-alpine3.17
+
+WORKDIR /datagateway-api-run
+
+COPY --from=builder /datagateway-api-build/dist/datagateway_api-*.whl /tmp
+
+RUN --mount=type=cache,target=/root/.cache \
+    python3 -m pip install \
+        'gunicorn~=20.1.0' \
+        /tmp/datagateway_api-*.whl \
+ && DATAGATEWAY_API_LOCATION="$(python3 -m pip show datagateway_api | awk '/^Location:/ { print $2 }')" \
+# Create search_api_mapping.json from its .example file
+ && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json.example" "$DATAGATEWAY_API_LOCATION/datagateway_api/search_api_mapping.json" \
+# Create config.yaml from its .example file. It will need to be editted by the entrypoint script so create it in our non-root user's home directory and create a symlink
+ && cp "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml.example" "/datagateway-api-run/config.yaml" \
+ && ln -s "/datagateway-api-run/config.yaml" "$DATAGATEWAY_API_LOCATION/datagateway_api/config.yaml" \
+# Create a non-root user to run as
+ && addgroup -S datagateway-api \
+ && adduser -S -D -G datagateway-api -H -h /datagateway-api-run datagateway-api \
+ && chown -R datagateway-api:datagateway-api /datagateway-api-run
+
+USER datagateway-api
+
+ENV ICAT_URL="http://localhost"
+ENV ICAT_CHECK_CERT="true"
+ENV LOG_LOCATION="/dev/stdout"
+
+COPY docker/docker-entrypoint.sh /usr/local/bin/
+ENTRYPOINT ["docker-entrypoint.sh"]
 
 # Serve the application using gunicorn - production ready WSGI server
-ENTRYPOINT ["poetry", "run", "gunicorn", "-c", "gunicorn.conf.py", "datagateway_api.wsgi:application"]
+CMD ["gunicorn", "-b", "0.0.0.0:8000", "datagateway_api.wsgi"]
+EXPOSE 8000