Cloud runner develop - latest fixes (#524)

game-ci · Mar 27, 2023 · 7abb3a4 · 7abb3a4
1 parent 309d668
commit 7abb3a4
Show file tree

Hide file tree

Showing 91 changed files with 5,204 additions and 3,860 deletions.
diff --git a/.github/workflows/build-tests-ubuntu.yml b/.github/workflows/build-tests-ubuntu.yml
@@ -49,7 +49,7 @@ jobs:
         exclude:
           - targetPlatform: Android
             unityVersion: 2022.2.7f1
-        cloudRunnerCluster:
+        providerStrategy:
           # - local-docker
           - local
         projectPath:
@@ -109,7 +109,7 @@ jobs:
           unityVersion: ${{ matrix.unityVersion }}
           targetPlatform: ${{ matrix.targetPlatform }}
           customParameters: -profile SomeProfile -someBoolean -someValue exampleValue
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
 
       ###########################
       #          Upload         #

diff --git a/.github/workflows/cloud-runner-async-checks.yml b/.github/workflows/cloud-runner-async-checks.yml
@@ -23,7 +23,7 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-github-pipelines
+  AWS_STACK_NAME: game-ci-github-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
   CLOUD_RUNNER_DEBUG: true
   CLOUD_RUNNER_DEBUG_TREE: true
@@ -39,20 +39,21 @@ jobs:
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout (default)
-        uses: actions/checkout@v3
-        with:
-          lfs: false
-      - run: yarn
-      - run: yarn run cli -m checks-update
-        timeout-minutes: 180
+      - timeout-minutes: 180
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
           CLOUD_RUNNER_CLUSTER: local-docker
-          AWS_BASE_STACK_NAME: game-ci-github-pipelines
+          AWS_STACK_NAME: game-ci-github-pipelines
           CHECKS_UPDATE: ${{ github.event.inputs.checksObject }}
+        run: |
+          git clone -b cloud-runner-develop https://github.com/game-ci/unity-builder
+          cd unity-builder
+          yarn
+          ls
+          yarn run cli -m checks-update
diff --git a/.github/workflows/cloud-runner-pipeline.yml → ...ub/workflows/cloud-runner-ci-pipeline.yml b/.github/workflows/cloud-runner-pipeline.yml → ...ub/workflows/cloud-runner-ci-pipeline.yml
@@ -21,81 +21,140 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: eu-west-2
-  AWS_BASE_STACK_NAME: game-ci-team-pipelines
+  AWS_STACK_NAME: game-ci-team-pipelines
   CLOUD_RUNNER_BRANCH: ${{ github.ref }}
-  CLOUD_RUNNER_DEBUG: true
-  CLOUD_RUNNER_DEBUG_TREE: true
   DEBUG: true
   UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
   PROJECT_PATH: test-project
   UNITY_VERSION: 2019.3.15f1
   USE_IL2CPP: false
   USE_GKE_GCLOUD_AUTH_PLUGIN: true
+  GIT_PRIVATE_TOKEN: ${{ secrets.GIT_PRIVATE_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
 jobs:
-  integrationTests:
-    name: Integration Tests
+  smokeTests:
+    name: Smoke Tests
     if: github.event.event_type != 'pull_request_target'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
-          - aws
+        test:
+          #- 'cloud-runner-async-workflow'
+          - 'cloud-runner-caching'
+          # - 'cloud-runner-end2end-caching'
+          # - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          - 'cloud-runner-hooks'
+          - 'cloud-runner-local-persistence'
+          - 'cloud-runner-locking-core'
+          - 'cloud-runner-locking-get-locked'
+        providerStrategy:
+          #- aws
           - local-docker
-          - k8s
+          #- k8s
     steps:
       - name: Checkout (default)
         uses: actions/checkout@v3
         with:
           lfs: false
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
       - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
         with:
           credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
       - name: 'Set up Cloud SDK'
-        uses: 'google-github-actions/setup-gcloud@v1'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
       - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
         run: |
           export USE_GKE_GCLOUD_AUTH_PLUGIN=True
           gcloud components install gke-gcloud-auth-plugin
           gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
-      - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          aws-region: eu-west-2
       - run: yarn
-      - run: yarn run test "cloud-runner-async-workflow" --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster != 'local-docker'
-        timeout-minutes: 180
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 35
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GIT_PRIVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: yarn run test-i --detectOpenHandles --forceExit --runInBand
-        if: matrix.CloudRunnerCluster == 'local-docker'
-        timeout-minutes: 180
+          CLOUD_RUNNER_CLUSTER: ${{ matrix.providerStrategy }}
+  tests:
+    # needs:
+    #   - smokeTests
+    #   - buildTargetTests
+    name: Integration Tests
+    if: github.event.event_type != 'pull_request_target'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        providerStrategy:
+          - aws
+          - local-docker
+          - k8s
+        test:
+          - 'cloud-runner-async-workflow'
+          #- 'cloud-runner-caching'
+          - 'cloud-runner-end2end-locking'
+          - 'cloud-runner-end2end-caching'
+          - 'cloud-runner-end2end-retaining'
+          - 'cloud-runner-environment'
+          #- 'cloud-runner-hooks'
+          - 'cloud-runner-s3-steps'
+          #- 'cloud-runner-local-persistence'
+          #- 'cloud-runner-locking-core'
+          #- 'cloud-runner-locking-get-locked'
+    steps:
+      - name: Checkout (default)
+        uses: actions/checkout@v2
+        with:
+          lfs: false
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-2
+      - uses: google-github-actions/auth@v1
+        if: matrix.providerStrategy == 'k8s'
+        with:
+          credentials_json: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
+      - name: 'Set up Cloud SDK'
+        if: matrix.providerStrategy == 'k8s'
+        uses: 'google-github-actions/setup-gcloud@v1.1.0'
+      - name: Get GKE cluster credentials
+        if: matrix.providerStrategy == 'k8s'
+        run: |
+          export USE_GKE_GCLOUD_AUTH_PLUGIN=True
+          gcloud components install gke-gcloud-auth-plugin
+          gcloud container clusters get-credentials $GKE_CLUSTER --zone $GKE_ZONE --project $GKE_PROJECT
+      - run: yarn
+      - run: yarn run test "${{ matrix.test }}" --detectOpenHandles --forceExit --runInBand
+        timeout-minutes: 60
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
           PROJECT_PATH: test-project
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           TARGET_PLATFORM: StandaloneWindows64
           cloudRunnerTests: true
           versioning: None
-          CLOUD_RUNNER_CLUSTER: ${{ matrix.cloudRunnerCluster }}
-  localBuildTests:
+          PROVIDER_STRATEGY: ${{ matrix.providerStrategy }}
+  buildTargetTests:
     name: Local Build Target Tests
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        cloudRunnerCluster:
+        providerStrategy:
           #- aws
           - local-docker
           #- k8s
@@ -114,20 +173,18 @@ jobs:
       - run: yarn
       - uses: ./
         id: unity-build
-        timeout-minutes: 90
+        timeout-minutes: 30
         env:
           UNITY_LICENSE: ${{ secrets.UNITY_LICENSE }}
         with:
           cloudRunnerTests: true
           versioning: None
-          projectPath: test-project
-          gitPrivateToken: ${{ secrets.GITHUB_TOKEN }}
           targetPlatform: ${{ matrix.targetPlatform }}
-          cloudRunnerCluster: ${{ matrix.cloudRunnerCluster }}
+          providerStrategy: ${{ matrix.providerStrategy }}
       - run: |
           cp ./cloud-runner-cache/cache/${{ steps.unity-build.outputs.CACHE_KEY }}/build/${{ steps.unity-build.outputs.BUILD_ARTIFACT }} ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
       - uses: actions/upload-artifact@v3
         with:
-          name: ${{ matrix.cloudRunnerCluster }} Build (${{ matrix.targetPlatform }})
+          name: ${{ matrix.providerStrategy }} Build (${{ matrix.targetPlatform }})
           path: ${{ steps.unity-build.outputs.BUILD_ARTIFACT }}
           retention-days: 14
diff --git a/action.yml b/action.yml
@@ -118,7 +118,7 @@ inputs:
     description:
       '[CloudRunner] Run a pre build job after the repository setup but before the build job (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  customStepFiles:
+  containerHookFiles:
     required: false
     default: ''
     description:
@@ -130,7 +130,7 @@ inputs:
     description:
       '[CloudRunner] Specify the names (by file name) of custom hooks to run before or after cloud runner jobs, must
       match a yaml step file inside your repo in the folder .game-ci/hooks/'
-  customJobHooks:
+  customCommandHooks:
     required: false
     default: ''
     description: '[CloudRunner] Specify custom commands and trigger hooks (injects commands into jobs)'
@@ -140,11 +140,11 @@ inputs:
     description:
       '[CloudRunner] Run a custom job instead of the standard build automation for cloud runner (in yaml format with the
       keys image, secrets (name, value object array), command line string)'
-  awsBaseStackName:
+  awsStackName:
     default: 'game-ci'
     required: false
     description: '[CloudRunner] The Cloud Formation stack name that must be setup before using this option.'
-  cloudRunnerCluster:
+  providerStrategy:
     default: 'local'
     required: false
     description: