Merge pull request #34 from openmethane/dockerize

Dockerize

.dockerignore

@@ -0,0 +1,15 @@
+# Ignore any local virtual environments
+.venv
+venv
+.cache
+
+# Ignore the generated runs
+data
+outputs
+intermediates
+inputs
+
+
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+**/*.py[cod]

.github/workflows/build_docker.yaml

@@ -0,0 +1,184 @@
+name: build_docker
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
+  pull_request:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/openmethane/openmethane-prior
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        platform:
+          #          - linux/arm64
+          - linux/amd64
+
+    permissions:
+      contents: read
+      packages: write
+
+    outputs:
+      digest: ${{ steps.build.outputs.digest }}
+
+    # Builds and pushes the image
+    # Tags the image with the PR that it is linked to
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: "${{ env.IMAGE }}"
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+      - name: Build and push image
+        uses: docker/build-push-action@v5
+        id: build
+        with:
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          push: true
+          pull: false
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  test-unit:
+    # Simple test suite to verify that the docker container works as expected
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: read
+      packages: read
+    container:
+      image: ghcr.io/openmethane/openmethane-prior@${{ needs.build.outputs.digest }}
+      credentials:
+        username: ${{ github.actor }}
+        password: ${{ secrets.github_token }}
+    steps:
+      - name: Run a quick test suite
+        run: |
+          cd /opt/project
+          cp .env.example .env
+          python -m pytest -r a -v tests/test_domain_json.py
+        env:
+          CDSAPI_KEY: ${{ secrets.CDSAPI_ADS_KEY }}
+          CDSAPI_URL: https://ads.atmosphere.copernicus.eu/api/v2
+      - name: Upload artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-artifacts
+          path: /opt/project/data
+
+  # Tag the latest image if running on the main branch
+  # TODO: Handle tagged builds
+  tag-latest-image:
+    runs-on: ubuntu-latest
+    needs: [ test-unit ]
+    if: github.ref == 'refs/heads/main'
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Login to Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: "${{ env.IMAGE }}"
+          tags: |
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main'}}
+      - name: Push latest image
+        uses: docker/build-push-action@v5
+        id: build
+        with:
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: ${{ steps.meta.outputs.tags }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          push: true
+          load: true
+
+  # Push the image to ECR as well
+  push-ecr:
+    runs-on: ubuntu-latest
+    needs: [test-unit, build ]
+    permissions:
+      contents: read
+      packages: read
+    env:
+      GHCR_IMAGE_ID: ghcr.io/openmethane/openmethane-prior@${{ needs.build.outputs.digest }}
+      ECR_IMAGE: 654654509571.dkr.ecr.ap-southeast-2.amazonaws.com/github/openmethane/openmethane-prior
+    steps:
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Login to Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        # TODO: Use the OIDC token instead of the access key
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ap-southeast-2
+      - name: Login to Amazon ECR
+        uses: aws-actions/amazon-ecr-login@v2
+      - name: Pull built docker image
+        run: |
+          docker pull ${{ env.GHCR_IMAGE_ID }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: "${{ env.ECR_IMAGE }}"
+          tags: |
+            type=schedule
+            type=ref,event=branch
+            type=ref,event=pr
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main'}}
+      - name: Tag and push docker image to Amazon ECR
+        run: |
+          echo "${{ steps.meta.outputs.tags }}"
+          
+          for tag in "${{ steps.meta.outputs.tags }}"; do
+            docker tag "${{ env.GHCR_IMAGE_ID }}" "$tag"
+            docker push "$tag"
+          done

Dockerfile

@@ -0,0 +1,57 @@
+# Secret management
+FROM segment/chamber:2 AS chamber
+
+# Build the reqired dependecies
+FROM python:3.11 AS builder
+
+# Creates a standalone environment in /opt/venv
+RUN pip install poetry==1.8.2
+
+WORKDIR /opt/venv
+
+COPY pyproject.toml poetry.lock ./
+RUN touch README.md
+
+# This installs the python dependencies into /opt/venv
+RUN python -m venv /opt/venv && \
+    poetry export --with dev,tests -f requirements.txt --output requirements.txt && \
+    /opt/venv/bin/pip install -r requirements.txt
+
+# Container for running the project
+# This isn't a hyper optimised container but it's a good starting point
+FROM python:3.11
+
+LABEL org.opencontainers.image.authors="jared.lewis@climate-resource.com"
+
+# Configure Python
+ENV PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random
+
+# This is deliberately outside of the work directory
+# so that the local directory can be mounted as a volume of testing
+ENV VIRTUAL_ENV=/opt/venv \
+    PATH="/opt/venv/bin:$PATH" \
+    LD_LIBRARY_PATH="/opt/venv/lib:$LD_LIBRARY_PATH"
+
+WORKDIR /opt/project
+
+# Install additional apt dependencies
+#RUN apt-get update && \
+#    apt-get install -y csh bc file make wget && \
+#    rm -rf /var/lib/apt/lists/*
+
+# Secret management
+COPY --from=chamber /chamber /bin/chamber
+
+# Copy across the virtual environment
+COPY --from=builder /opt/venv /opt/venv
+
+# Copy in the rest of the project
+# For testing it might be easier to mount $(PWD):/opt/project so that local changes are reflected in the container
+COPY . /opt/project
+
+# Install the local package in editable mode
+RUN pip install -e .
+
+CMD ["/bin/bash"]

Makefile

@@ -2,6 +2,10 @@
 update-licenseheaders:  ## add or update license headers in all python files
 	licenseheaders -y 2023 --owner "The Superpower Institute Ltd" --projname "OpenMethane" --tmpl .copyright.tmpl --ext .py -x "venv/*"
 
+.PHONY: build
+build:  ## Build the docker container locally
+	docker build --platform=linux/amd64 -t openmethane-prior .
+
 .PHONY: virtual-environment
 virtual-environment:  ## update virtual environment, create a new one if it doesn't already exist
 	poetry lock --no-update