Improve caching key #496
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and Publish Mlflow Docker Image | |
| on: | |
| push: | |
| branches: [main] | |
| # Publish semver tags as releases. | |
| tags: | |
| - "*.*.*" | |
| - "*.*.*.*" | |
| paths: | |
| - ".github/workflows/docker-publish.yml" | |
| - ".env" | |
| - "poetry.lock" | |
| - "poetry.toml" | |
| - "pyproject.toml" | |
| - "Dockerfile" | |
| - "mlflowstack/**" | |
| - "docker-compose.*.yaml" | |
| - "tests/**" | |
| - "test-containers/**" | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| jobs: | |
| integrationtest: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| database-combo: | |
| - { | |
| postgres: "15", | |
| mysql: "8.0", | |
| minio: "RELEASE.2024-11-07T00-52-20Z", | |
| gcs: "1.50.2", | |
| azurite: "3.33.0", | |
| azure-storage-blob: "12.26.0", | |
| } | |
| - { | |
| postgres: "16", | |
| mysql: "8.4", | |
| minio: "RELEASE.2024-11-07T00-52-20Z", | |
| gcs: "1.50.2", | |
| azurite: "3.33.0", | |
| azure-storage-blob: "12.26.0", | |
| } | |
| - { | |
| postgres: "17", | |
| mysql: "9.1", | |
| minio: "RELEASE.2024-11-07T00-52-20Z", | |
| gcs: "1.50.2", | |
| azurite: "3.33.0", | |
| azure-storage-blob: "12.26.0", | |
| } | |
| name: Test PostgreSQL ${{ matrix.database-combo.postgres }} & MySQL ${{ matrix.database-combo.mysql }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up python | |
| id: setup-python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.12 | |
| #---------------------------------------------- | |
| # ----- install & configure poetry ----- | |
| #---------------------------------------------- | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| #---------------------------------------------- | |
| # load cached venv if cache exists | |
| #---------------------------------------------- | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} | |
| #---------------------------------------------- | |
| # install dependencies if cache does not exist | |
| #---------------------------------------------- | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| #---------------------------------------------- | |
| # install your root project, if required | |
| #---------------------------------------------- | |
| - name: Install library | |
| run: poetry install --no-interaction | |
| #---------------------------------------------- | |
| # Cache or Restore docker images | |
| #---------------------------------------------- | |
| - name: Cache Docker images | |
| id: cache-docker-images | |
| uses: actions/cache@v4 | |
| with: | |
| path: /tmp/.docker # This path stores the Docker images and cache | |
| key: ${{ runner.os }}-docker-images-${{ matrix.database-combo.postgres }}-${{ matrix.database-combo.mysql }}-${{ matrix.database-combo.minio }}-${{ matrix.database-combo.gcs }}-${{ matrix.database-combo.azurite }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-images- | |
| #---------------------------------------------- | |
| # Pull does not exist docker images | |
| #---------------------------------------------- | |
| - name: Pull Dependency Docker images | |
| run: | | |
| # Pull PostgreSQL image if not cached | |
| if ! docker image inspect postgres:${{ matrix.database-combo.postgres }} > /dev/null 2>&1; then | |
| echo "PostgreSQL ${{ matrix.database-combo.postgres }} image not found locally. Pulling image..." | |
| docker pull postgres:${{ matrix.database-combo.postgres }} | |
| else | |
| echo "PostgreSQL image already exists. Using cache." | |
| fi | |
| # Pull MySQL image if not cached | |
| if ! docker image inspect mysql:${{ matrix.database-combo.mysql }} > /dev/null 2>&1; then | |
| echo "MySQL ${{ matrix.database-combo.mysql }} image not found locally. Pulling image..." | |
| docker pull mysql:${{ matrix.database-combo.mysql }} | |
| else | |
| echo "MySQL image already exists. Using cache." | |
| fi | |
| # Pull Minio image if not cached | |
| if ! docker image inspect minio/minio:${{ matrix.database-combo.minio }} > /dev/null 2>&1; then | |
| echo "Minio ${{ matrix.database-combo.minio }} image not found locally. Pulling image..." | |
| docker pull minio/minio:${{ matrix.database-combo.minio }} | |
| else | |
| echo "Minio image already exists. Using cache." | |
| fi | |
| # Pull Fake GCS Server image if not cached | |
| if ! docker image inspect fsouza/fake-gcs-server:${{ matrix.database-combo.gcs }} > /dev/null 2>&1; then | |
| echo "Fake GCS Server ${{ matrix.database-combo.gcs }} image not found locally. Pulling image..." | |
| docker pull fsouza/fake-gcs-server:${{ matrix.database-combo.gcs }} | |
| else | |
| echo "Fake GCS Server image already exists. Using cache." | |
| fi | |
| # Pull Azurite image if not cached | |
| if ! docker image inspect mcr.microsoft.com/azure-storage/azurite:${{ matrix.database-combo.azurite }} > /dev/null 2>&1; then | |
| echo "Azurite ${{ matrix.database-combo.azurite }} image not found locally. Pulling image..." | |
| docker pull mcr.microsoft.com/azure-storage/azurite:${{ matrix.database-combo.azurite }} | |
| docker-compose -f docker-compose.azure-mysql-test.yaml build | |
| docker-compose -f docker-compose.azure-postgres-test.yaml build | |
| else | |
| echo "Azurite image already exists. Using cache." | |
| fi | |
| #---------------------------------------------- | |
| # Overwrite .env file to test different version of databases | |
| #---------------------------------------------- | |
| - name: Set up .env file | |
| run: | | |
| echo "POSTGRES_VERSION=${{ matrix.database-combo.postgres }}" > .env | |
| echo "MYSQL_VERSION=${{ matrix.database-combo.mysql }}" >> .env | |
| echo "MINIO_VERSION=${{ matrix.database-combo.minio }}" >> .env | |
| echo "FAKE_GCS_SERVER_VERSION=${{ matrix.database-combo.gcs }}" >> .env | |
| echo "AZURITE_VERSION=${{ matrix.database-combo.azurite }}" >> .env | |
| echo "AZURE_STORAGE_BLOB_VERSION=${{ matrix.database-combo.azure-storage-blob }}" >> .env | |
| #---------------------------------------------- | |
| # run test suite | |
| #---------------------------------------------- | |
| - name: Run tests | |
| env: | |
| POSTGRES_VERSION: ${{ matrix.database-combo.postgres }} | |
| MYSQL_VERSION: ${{ matrix.database-combo.mysql }} | |
| run: | | |
| source .venv/bin/activate | |
| poetry run pytest | |
| buildtestpush: | |
| runs-on: ubuntu-latest | |
| needs: integrationtest | |
| env: | |
| IMAGE_NAME: ${{ github.repository }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| # This is used to complete the identity challenge | |
| # with sigstore/fulcio when running outside of PRs. | |
| id-token: write | |
| # To upload sarif files | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Install ORAS | |
| id: install_oras | |
| uses: oras-project/setup-oras@main | |
| - name: Install Cosign | |
| id: install_cosign | |
| uses: sigstore/cosign-installer@v3.7.0 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3.2.0 | |
| with: | |
| platforms: all | |
| # Workaround: https://github.com/docker/build-push-action/issues/461 | |
| - name: Setup Docker buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # Login against a Docker Hub registry except on PR | |
| # https://github.com/docker/login-action | |
| - name: Login to Docker Hub | |
| id: docker_hub_login | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: docker.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.DOCKER_HUB_TOKEN }} | |
| # Login against a Docker registry except on PR | |
| # https://github.com/docker/login-action | |
| - name: Login to GitHub Container Registry | |
| id: ghcr_login | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to GitHub Container Registry (ORAS) | |
| id: oras_ghcr_login | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | oras login ghcr.io --username ${{ github.actor }} --password-stdin | |
| - name: Publish Artifact Hub Manifest | |
| id: publish_ah_manifest | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| run: | | |
| oras push \ | |
| ghcr.io/${{ env.IMAGE_NAME }}:artifacthub.io \ | |
| --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \ | |
| artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml | |
| # Extract metadata (tags, labels) for Docker | |
| # https://github.com/docker/metadata-action | |
| - name: Extract Docker metadata | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| id: meta | |
| uses: docker/metadata-action@v5 | |
| with: | |
| images: | | |
| docker.io/${{ env.IMAGE_NAME }} | |
| ghcr.io/${{ env.IMAGE_NAME }} | |
| flavor: | | |
| latest=true | |
| tags: | | |
| type=ref,event=tag | |
| labels: | | |
| io.artifacthub.package.readme-url=https://raw.githubusercontent.com/burakince/mlflow/main/README.md | |
| io.artifacthub.package.maintainers=[{"name":"burakince","email":"burak.ince@linux.org.tr"}] | |
| io.artifacthub.package.logo-url=https://raw.githubusercontent.com/mlflow/mlflow/master/assets/logo.svg | |
| io.artifacthub.package.keywords=machine-learning,ai,ml,model-management,mlflow,mlflow-tracking-server,mlflow-docker,mlflow-tracking,mlflow-kube | |
| io.artifacthub.package.license=MIT | |
| io.artifacthub.package.alternative-locations=docker.io/${{ env.IMAGE_NAME }} | |
| # Build Docker image with Buildx and don't push it | |
| - name: Build Docker image | |
| id: build-docker-image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/arm64/v8,linux/amd64 | |
| push: false | |
| tags: docker.io/${{ env.IMAGE_NAME }}:latest | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: Run Snyk to check Docker image for vulnerabilities | |
| id: docker-image-scan | |
| continue-on-error: true | |
| uses: snyk/actions/docker@0.4.0 | |
| env: | |
| SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} | |
| with: | |
| image: docker.io/${{ env.IMAGE_NAME }}:latest | |
| args: --file=Dockerfile --severity-threshold=medium --sarif-file-output=snyk.sarif | |
| - name: Upload result to GitHub Code Scanning | |
| if: hashFiles('snyk.sarif') != '' | |
| uses: github/codeql-action/upload-sarif@v3 | |
| with: | |
| sarif_file: snyk.sarif | |
| # Break the pipeline with failure status if security scan failed | |
| # - name: Check docker image scan status | |
| # if: ${{ steps.docker-image-scan.outcome == 'failure' }} | |
| # run: exit 1 | |
| # Build again and push Docker image with Buildx (don't push on PR) | |
| # https://github.com/docker/build-push-action | |
| - name: Build and push Docker image | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| id: build-and-push | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| platforms: linux/arm64/v8,linux/amd64 | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| # Sign the resulting Docker image digest except on PRs. | |
| # This will only write to the public Rekor transparency log when the Docker | |
| # repository is public to avoid leaking data. If you would like to publish | |
| # transparency data even for private images, pass --force to cosign below. | |
| # https://github.com/sigstore/cosign | |
| - name: Sign the published Docker images | |
| if: github.event_name != 'pull_request' && contains(github.ref, 'refs/tags/') | |
| run: | | |
| echo "${{ steps.meta.outputs.tags }}" | xargs -I {} cosign sign -y {}@${{ steps.build-and-push.outputs.digest }} |