Merge branch 'main' into jstjohn/optional_logging_for_predict

NVIDIA · Aug 15, 2024 · ac2075b · ac2075b
2 parents 479e2d8 + c066ef7
commit ac2075b
Show file tree

Hide file tree

Showing 23 changed files with 1,770 additions and 192 deletions.
diff --git a/.github/workflows/_test_template.yml b/.github/workflows/_test_template.yml
@@ -34,39 +34,32 @@ on:
         description: Last 2000 characters of the test step's log
         value: ${{ jobs.main.outputs.log }} 
 jobs:
-  runner-auto-clean:
-    runs-on: ${{ inputs.RUNNER }}
-    steps:
-        - name: Docker system cleanup
-          run: |
-            docker system prune -a --filter "until=48h" --force
-
+
   main:
     runs-on: ${{ inputs.RUNNER }} 
     outputs:
       conclusion: ${{ steps.main.conclusion }}
       log: ${{ steps.main.outputs.log }}
-    container:
-      image: nemoci.azurecr.io/nemo_container_${{ github.run_id }}
-      options: 
-        --device=/dev/nvidia0
-        --gpus all
-        --shm-size=8g
-        --env TRANSFORMERS_OFFLINE=0 
-        --env HYDRA_FULL_ERROR=1
-        --volume /mnt/datadrive/TestData:/home/TestData
     steps:
-        - name: Checkout repository
-          uses: actions/checkout@v4
+        - name: Docker system cleanup
+          run: |
+            docker system prune -a --filter "until=48h" --force
+
+        - name: Docker pull image
+          run: |
+            docker pull nemoci.azurecr.io/nemo_container_${{ github.run_id }}
+
         - id: main
           name: Run main script
           timeout-minutes: ${{ inputs.TIMEOUT }}
           run: |
+            mkdir -p ${{ github.run_id }}
+            cd ${{ github.run_id }}/
             set +e 
             (  
               set -e
 
-              ${{ inputs.SCRIPT }}
+              docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.SCRIPT }}'
             ) 2> >(tee err.log)
 
             EXIT_CODE=$?
@@ -79,4 +72,5 @@ jobs:
           if: failure() && inputs.IS_OPTIONAL == false
         - name: after_script
           if: always() && inputs.AFTER_SCRIPT != ':'
-          run: ${{ inputs.AFTER_SCRIPT }}
+          run: |
+            docker run --rm --device=/dev/nvidia0 --gpus all --shm-size=8g --env TRANSFORMERS_OFFLINE=0 --env HYDRA_FULL_ERROR=1 --volume /mnt/datadrive/TestData:/home/TestData nemoci.azurecr.io/nemo_container_${{ github.run_id }} bash -c '${{ inputs.AFTER_SCRIPT }}'