Refactor GRID build, publish and test pipelines to consume driver fro…

…m config (#48) * Refactor GRID test pipeline and justfile to consume driver from config * add main.yaml, remove unnecessary $ and driver_version (for cuda) * update justfile
Azure · Aug 7, 2024 · 6205588 · 6205588
1 parent 666bb81
commit 6205588
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 25 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -50,17 +50,24 @@ jobs:
         run: |
           rm -r /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-  grid535:
+  grid:
       runs-on: ubuntu-latest
       strategy:
         matrix:
-          driver_version: ["535.161.08"]
           driver_kind: ["grid"]
-          driver_url: ["https://download.microsoft.com/download/8/d/a/8da4fb8e-3a9b-4e6a-bc9a-72ff64d7a13c/NVIDIA-Linux-x86_64-535.161.08-grid-azure.run"]
       steps:
         - uses: actions/checkout@v2
           with:
             fetch-depth: 0
+        - name: Load GRID config
+          id: load_config
+          run: |
+              grid_version=$(yq e '.grid.version' driver_config.yml)
+              grid_url=$(yq e '.grid.url' driver_config.yml)
+              echo "GRID_VERSION=$grid_version"
+              echo "GRID_URL=$grid_url"
+              echo "grid_version=$grid_version" >> $GITHUB_OUTPUT
+              echo "grid_url=$grid_url" >> $GITHUB_OUTPUT
         - name: Set up Docker Buildx
           uses: docker/setup-buildx-action@v1
         - name: Cache Docker layers
@@ -73,7 +80,7 @@ jobs:
         - uses: paulhatch/semantic-version@v5.0.0-alpha2
           with:
             bump_each_commit: false
-            version_format: "${{ matrix.driver_kind}}-${{ matrix.driver_version }}-sha-${GITHUB_SHA:0:6}"
+            version_format: "${{ matrix.driver_kind}}-${{ steps.load_config.outputs.grid_version }}-sha-${GITHUB_SHA:0:6}"
           id: semver
         - name: 'Check version'
           run: |
@@ -84,7 +91,7 @@ jobs:
             set -x
             echo "tag is: "
             echo ${{ steps.semver.outputs.version }}
-            docker buildx build --build-arg DRIVER_URL=${{ matrix.driver_url }} --build-arg DRIVER_KIND=${{ matrix.driver_kind }} --build-arg DRIVER_VERSION=${{ matrix.driver_version }} --cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new --output=type=docker -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }} .
+            docker buildx build --build-arg DRIVER_URL=${{ steps.load_config.outputs.grid_url }} --build-arg DRIVER_KIND=${{ matrix.driver_kind }} --build-arg DRIVER_VERSION=${{ steps.load_config.outputs.grid_version }} --cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new --output=type=docker -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }} .
             docker images
         - name: Move cache
           run: |

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -13,7 +13,6 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        driver_version: ["${{ needs.load_config.outputs.cuda_version }}"]
         driver_kind: ["cuda"]
     steps:
       - uses: actions/checkout@v2
@@ -62,30 +61,37 @@ jobs:
         run: |
           rm -r /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
-  grid535:
+  grid:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        driver_version: ["535.161.08"]
         driver_kind: ["grid"]
-        driver_url: ["https://download.microsoft.com/download/8/d/a/8da4fb8e-3a9b-4e6a-bc9a-72ff64d7a13c/NVIDIA-Linux-x86_64-535.161.08-grid-azure.run"]
     steps:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
+      - name: Load GRID config
+        id: load_config
+        run: |
+            grid_version=$(yq e '.grid.version' driver_config.yml)
+            grid_url=$(yq e '.grid.url' driver_config.yml)
+            echo "GRID_VERSION=$grid_version"
+            echo "GRID_URL=$grid_url"
+            echo "grid_version=$grid_version" >> $GITHUB_OUTPUT
+            echo "grid_url=$grid_url" >> $GITHUB_OUTPUT
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
       - name: Cache Docker layers
         uses: actions/cache@v2
         with:
           path: /tmp/.buildx-cache
-          key: ${{ runner.os }}-buildx-${{ matrix.driver_kind}}-${{ matrix.driver_version }}-${{ github.sha }}
+          key: ${{ runner.os }}-buildx-${{ matrix.driver_kind}}-${{ steps.load_config.outputs.grid_version }}-${{ github.sha }}
           restore-keys: |
-            ${{ runner.os }}-buildx-${{ matrix.driver_kind}}-${{ matrix.driver_version }}
+            ${{ runner.os }}-buildx-${{ matrix.driver_kind}}-${{ steps.load_config.outputs.grid_version }}
       - uses: paulhatch/semantic-version@v5.0.0-alpha2
         with:
           bump_each_commit: false
-          version_format: "${{ matrix.driver_kind}}-${{ matrix.driver_version }}-sha-${GITHUB_SHA:0:6}"
+          version_format: "${{ matrix.driver_kind}}-${{ steps.load_config.outputs.grid_versionn }}-sha-${GITHUB_SHA:0:6}"
         id: semver
       - name: 'Check version'
         run: |
@@ -102,10 +108,10 @@ jobs:
           set -x
           echo "tag is: "
           echo ${{ steps.semver.outputs.version }}
-          docker buildx build --build-arg DRIVER_URL=${{ matrix.driver_url }} --build-arg DRIVER_KIND=${{ matrix.driver_kind }} --build-arg DRIVER_VERSION=${{ matrix.driver_version }} --cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new --output=type=docker -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }} .
+          docker buildx build --build-arg DRIVER_URL=${{ steps.load_config.outputs.grid_url }} --build-arg DRIVER_KIND=${{ matrix.driver_kind }} --build-arg DRIVER_VERSION=${{ steps.load_config.outputs.grid_version }} --cache-from=type=local,src=/tmp/.buildx-cache --cache-to=type=local,dest=/tmp/.buildx-cache-new --output=type=docker -t ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }} .
           docker images
           az acr login -n ${{ secrets.AZURE_REGISTRY_SERVER }}
-          docker push ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.semver.outputs.version }}
+          docker push ${{ secrets.AZURE_REGISTRY_SERVER }}/public/aks/aks-gpu:${{ steps.load_config.outputs.grid_version }}
       - name: Move cache
         run: |
           rm -r /tmp/.buildx-cache

diff --git a/justfile b/justfile
@@ -1,23 +1,19 @@
-grid_535_url    := "https://download.microsoft.com/download/8/d/a/8da4fb8e-3a9b-4e6a-bc9a-72ff64d7a13c/NVIDIA-Linux-x86_64-535.161.08-grid-azure.run"
-
-grid_535_driver := "535.161.08"
-
 registry := "docker.io/alexeldeib"
 
 default:
 
 pushallcuda: (pushcuda)
 
-pushallgrid: (pushgrid grid_535_driver)
+pushallgrid: (pushgrid)
 
 pushcuda: (buildcuda)
 	docker push {{ registry }}/aks-gpu:$(yq e '.cuda.version' driver_config.yml)-cuda
 
-pushgrid VERSION URL: (buildgrid VERSION URL)
-	docker push {{ registry }}/aks-gpu:{{VERSION}}-grid
+pushgrid: (buildgrid)
+	docker push {{ registry }}/aks-gpu:$(yq e '.grid.version' driver_config.yml)-grid
 
-buildgrid VERSION URL:
-	docker build --build-arg DRIVER_URL={{URL}} --build-arg DRIVER_KIND=grid --build-arg DRIVER_VERSION={{VERSION}} -f Dockerfile -t {{ registry }}/aks-gpu:{{VERSION}}-grid .
+buildgrid:
+	docker build --build-arg DRIVER_URL=$(yq e '.grid.url' driver_config.yml) --build-arg DRIVER_KIND=grid --build-arg DRIVER_VERSION=$(yq e '.grid.version' driver_config.yml) -f Dockerfile -t {{ registry }}/aks-gpu:{{VERSION}}-grid .
 
-buildcuda VERSION:
-	docker build --build-arg DRIVER_KIND=cuda --build-arg DRIVER_VERSION={{VERSION}} -f Dockerfile -t {{ registry }}/aks-gpu:{{VERSION}}-cuda .
+buildcuda:
+	docker build --build-arg DRIVER_KIND=cuda --build-arg DRIVER_VERSION=$(yq e '.cuda.version' driver_config.yml) -f Dockerfile -t {{ registry }}/aks-gpu:$(yq e '.cuda.version' driver_config.yml)-cuda .