fix: optimize scripts for parallel startup of Looker docker image (#881)

* fix: optimize scripts for parellel startup of Looker docker image * fix: Default value of LOOKERSDK_API_VERSION if it is not set * fix: try to get python-ci working properly * fix: debug python-ci * fix: debug python-ci try again * Make sure repo is checked out first otherwise weird rade conditions happen
looker-open-source · Oct 29, 2021 · 5624c72 · 5624c72
1 parent a4dfb3c
commit 5624c72
Show file tree

Hide file tree

Showing 6 changed files with 126 additions and 116 deletions.
diff --git a/.github/scripts/wait_for_looker.py b/.github/scripts/wait_for_looker.py
diff --git a/.github/scripts/wait_for_looker.sh b/.github/scripts/wait_for_looker.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+check_looker() {
+  status=$(curl --silent --insecure --write "%{http_code}" \
+    --data "client_id=$LOOKERSDK_CLIENT_ID&client_secret=$LOOKERSDK_CLIENT_SECRET"\
+    $LOOKERSDK_BASE_URL/api/${LOOKERSDK_API_VERSION:-4.0}/login\
+    -o /dev/null)
+}
+
+MAX_RETRIES=160
+ATTEMPTS=1
+status=0
+check_looker
+while [ $status -ne 200 ];
+do
+  RETRY_MSG="after $ATTEMPTS attempts: $MAX_RETRIES retries remaining."
+  if [ $ATTEMPTS -ge $MAX_RETRIES ];
+  then
+    echo 'Looker took too long to start'
+    exit 1
+  else
+    if [ $status -ne 0 ];
+    then
+      echo "Received status($status) from Looker $RETRY_MSG"
+    else
+      echo "Looker server connection rejected $RETRY_MSG"
+    fi
+  fi
+
+  sleep 2
+  ATTEMPTS=$(( $ATTEMPTS + 1 ))
+  check_looker
+done
+echo "Looker ready after $ATTEMPTS attempts"
+exit 0
diff --git a/.github/workflows/codegen-ci.yml b/.github/workflows/codegen-ci.yml
@@ -32,13 +32,32 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Repo Checkout
+        uses: actions/checkout@v2
+
       - name: Cancel Previous Runs
         uses: styfle/cancel-workflow-action@0.6.0
         with:
           access_token: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Repo Checkout
-        uses: actions/checkout@v2
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0.2.0
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Authenticate Artifact Repository
+        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
+
+      - name: Pull and run Looker docker image
+        # TODO: can we cache some layers of the image for faster download?
+        # we probably don't want to cache the final image for IP security...
+        run: |
+          docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
+          # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl
+          docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
+          docker logs -f looker-sdk-codegen-ci --until=30s &
 
       - uses: actions/setup-node@v1
         with:
@@ -60,30 +79,9 @@ jobs:
           echo "[Miner]" >> looker.ini
           echo "base_url=." >> looker.ini
 
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0.2.0
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
-          export_default_credentials: true
-
-      - name: Authenticate Artifact Repository
-        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
-      - name: Pull and run Looker docker image
-        # TODO: can we cache some layers of the image for faster download?
-        # we probably don't want to cache the final image for IP security...
+      - name: Check that Looker is ready
         run: |
-          docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
-          # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl
-          docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
-          docker logs -f looker-sdk-codegen-ci --until=30s &
-          python ${{ github.workspace }}/.github/scripts/wait_for_looker.py
+          ${{ github.workspace }}/.github/scripts/wait_for_looker.sh
 
       - name: Run unit tests
         run: yarn jest "packages/sdk-codegen(-utils|-scripts)/src" --reporters=default --reporters=jest-junit

diff --git a/.github/workflows/lerna-publish.yml b/.github/workflows/lerna-publish.yml
@@ -29,6 +29,23 @@ jobs:
         with:
           persist-credentials: false
           ref: ${{  github.event.release.tag_name || github.event.inputs.ref }}
+
+      - name: Set up Cloud SDK
+        uses: google-github-actions/setup-gcloud@v0.2.0
+        with:
+          project_id: ${{ secrets.GCP_PROJECT_ID }}
+          service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
+          export_default_credentials: true
+
+      - name: Authenticate Artifact Repository
+        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
+
+      - name: Pull and run Looker docker image
+        run: |
+          docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
+          docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
+          docker logs -f looker-sdk-codegen-ci --until=30s &
+
       - uses: actions/setup-node@v2
         with:
           node-version: 15
@@ -53,27 +70,10 @@ jobs:
           echo "base_url=https://self-signed.looker.com:19999" >> looker.ini
           echo "verify_ssl=False" >> looker.ini
           echo "timeout=30" >> looker.ini
-      - name: Set up Cloud SDK
-        uses: google-github-actions/setup-gcloud@v0.2.0
-        with:
-          project_id: ${{ secrets.GCP_PROJECT_ID }}
-          service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
-          export_default_credentials: true
-
-      - name: Authenticate Artifact Repository
-        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
-
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
 
-      - name: Pull and run Looker docker image
+      - name: Check that Looker is ready
         run: |
-          docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
-          docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18
-          docker logs -f looker-sdk-codegen-ci --until=30s &
-          python ${{ github.workspace }}/.github/scripts/wait_for_looker.py
+          ${{ github.workspace }}/.github/scripts/wait_for_looker.sh
 
       - name: Publish to NPM registry
         run: $(npm bin)/lerna publish from-package --yes --no-verify-access

diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml
@@ -66,16 +66,20 @@ jobs:
     steps:
       - name: Repo Checkout
         uses: actions/checkout@v2
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install Tox and any other packages
         run: |
           python -m pip install --upgrade pip
           pip install tox
+
       - name: Run Unit Tests
         run: tox -e unit
+
       - name: Upload pytest test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v2
@@ -120,32 +124,25 @@ jobs:
     steps:
       - name: Repo Checkout
         uses: actions/checkout@v2
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install Tox and any other packages
-        run: |
-          python -m pip install --upgrade pip
-          pip install tox
-      - name: Set LOOKERSDK_CLIENT_{ID,SECRET}
-        # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20
-        run: |
-          echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV
-          echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV
+
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v0.2.0
         with:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
           service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }}
           export_default_credentials: true
+
+      - name: Authenticate Artifact Repository
+        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
+
       - name: Install docker on macos
         if: ${{ matrix.os == 'macos' }}
         uses: docker-practice/actions-setup-docker@v1
         with:
           docker_channel: stable
           docker_buildx: false
           docker_cli_experimental: disabled
+
       - name: Bump docker for mac memory
         if: ${{ matrix.os == 'macos' }}
         run: |
@@ -161,8 +158,7 @@ jobs:
           done
           (( i )) && printf '\n'
           echo "-- Docker is ready."
-      - name: Authenticate Artifact Repository
-        run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
+
       - name: Pull and run Looker docker image
         # TODO: can we cache some layers of the image for faster download?
         # we probably don't want to cache the final image for IP security...
@@ -171,9 +167,26 @@ jobs:
           # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl
           docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }}
           docker logs -f looker-sdk-codegen-ci --until=30s &
-          python ${{ github.workspace }}/.github/scripts/wait_for_looker.py
+
+      - name: Install Tox and any other packages
+        run: |
+          python -m pip install --upgrade pip
+          pip install tox
+
+      - name: Set LOOKERSDK_CLIENT_{ID,SECRET}
+        # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20
+        run: |
+          echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV
+          echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV
+
+
+      - name: Check that Looker is ready
+        run: |
+          ${{ github.workspace }}/.github/scripts/wait_for_looker.sh
+
       - name: Run Integration Tests
         run: tox -e integration
+
       - name: Upload pytest test results
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

diff --git a/.github/workflows/tssdk-ci.yml b/.github/workflows/tssdk-ci.yml
@@ -107,22 +107,6 @@ jobs:
       - name: Repo Checkout
         uses: actions/checkout@v2
 
-      - name: Install Node ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: Install dependencies
-        run: |
-          yarn
-          yarn build
-
-      - name: Set LOOKERSDK_CLIENT_{ID,SECRET}
-        # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20
-        run: |
-          echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV
-          echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV
-
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v0.2.0
         with:
@@ -133,11 +117,6 @@ jobs:
       - name: Authenticate Artifact Repository
         run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-
       - name: Pull and run Looker docker image
         # TODO: can we cache some layers of the image for faster download?
         # we probably don't want to cache the final image for IP security...
@@ -146,7 +125,26 @@ jobs:
           # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl
           docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }}
           docker logs -f looker-sdk-codegen-ci --until=30s &
-          python ${{ github.workspace }}/.github/scripts/wait_for_looker.py
+
+      - name: Install Node ${{ matrix.node-version }}
+        uses: actions/setup-node@v2
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: Install dependencies
+        run: |
+          yarn
+          yarn build
+
+      - name: Set LOOKERSDK_CLIENT_{ID,SECRET}
+        # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20
+        run: |
+          echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV
+          echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV
+
+      - name: Check that Looker is ready
+        run: |
+          ${{ github.workspace }}/.github/scripts/wait_for_looker.sh
 
       - name: Run Integration Tests
         run: yarn jest packages/sdk-node/test --reporters=default --reporters=jest-junit