From 5624c72e3e65159bd8ae306c3eca4176ebee3675 Mon Sep 17 00:00:00 2001 From: "Dr. Strangelove" Date: Fri, 29 Oct 2021 15:06:18 -0400 Subject: [PATCH] fix: optimize scripts for parallel startup of Looker docker image (#881) * fix: optimize scripts for parellel startup of Looker docker image * fix: Default value of LOOKERSDK_API_VERSION if it is not set * fix: try to get python-ci working properly * fix: debug python-ci * fix: debug python-ci try again * Make sure repo is checked out first otherwise weird rade conditions happen --- .github/scripts/wait_for_looker.py | 34 -------------------- .github/scripts/wait_for_looker.sh | 35 +++++++++++++++++++++ .github/workflows/codegen-ci.yml | 48 ++++++++++++++--------------- .github/workflows/lerna-publish.yml | 38 +++++++++++------------ .github/workflows/python-ci.yml | 45 +++++++++++++++++---------- .github/workflows/tssdk-ci.yml | 42 ++++++++++++------------- 6 files changed, 126 insertions(+), 116 deletions(-) delete mode 100644 .github/scripts/wait_for_looker.py create mode 100755 .github/scripts/wait_for_looker.sh diff --git a/.github/scripts/wait_for_looker.py b/.github/scripts/wait_for_looker.py deleted file mode 100644 index f4f72d5a2..000000000 --- a/.github/scripts/wait_for_looker.py +++ /dev/null @@ -1,34 +0,0 @@ -import ssl -import sys -import time -import urllib.request - -# turn off ssl checking -ctx = ssl.create_default_context() -ctx.check_hostname = False -ctx.verify_mode = ssl.CERT_NONE - -MAX_RETRIES = 160 -ATTEMPTS = 1 -while MAX_RETRIES: - retry_msg = f"after {ATTEMPTS} attempts: {MAX_RETRIES} retries remaining." - try: - status = urllib.request.urlopen( - "https://localhost:10000/login", context=ctx - ).getcode() - except urllib.error.URLError: - msg = f"Looker server connection rejected {retry_msg}" - else: - if status == 200: - print(f"Looker ready after {ATTEMPTS} attempts.") - sys.exit(0) - else: - msg = f"Received status({status}) from Looker {retry_msg}" - - ATTEMPTS += 1 - MAX_RETRIES -= 1 - time.sleep(2) - print(msg) - -print(f"Looker took too long to start {retry_msg}") -sys.exit(1) diff --git a/.github/scripts/wait_for_looker.sh b/.github/scripts/wait_for_looker.sh new file mode 100755 index 000000000..2942fe482 --- /dev/null +++ b/.github/scripts/wait_for_looker.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +check_looker() { + status=$(curl --silent --insecure --write "%{http_code}" \ + --data "client_id=$LOOKERSDK_CLIENT_ID&client_secret=$LOOKERSDK_CLIENT_SECRET"\ + $LOOKERSDK_BASE_URL/api/${LOOKERSDK_API_VERSION:-4.0}/login\ + -o /dev/null) +} + +MAX_RETRIES=160 +ATTEMPTS=1 +status=0 +check_looker +while [ $status -ne 200 ]; +do + RETRY_MSG="after $ATTEMPTS attempts: $MAX_RETRIES retries remaining." + if [ $ATTEMPTS -ge $MAX_RETRIES ]; + then + echo 'Looker took too long to start' + exit 1 + else + if [ $status -ne 0 ]; + then + echo "Received status($status) from Looker $RETRY_MSG" + else + echo "Looker server connection rejected $RETRY_MSG" + fi + fi + + sleep 2 + ATTEMPTS=$(( $ATTEMPTS + 1 )) + check_looker +done +echo "Looker ready after $ATTEMPTS attempts" +exit 0 diff --git a/.github/workflows/codegen-ci.yml b/.github/workflows/codegen-ci.yml index cd8064677..f13503537 100644 --- a/.github/workflows/codegen-ci.yml +++ b/.github/workflows/codegen-ci.yml @@ -32,13 +32,32 @@ jobs: runs-on: ubuntu-latest steps: + - name: Repo Checkout + uses: actions/checkout@v2 + - name: Cancel Previous Runs uses: styfle/cancel-workflow-action@0.6.0 with: access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Repo Checkout - uses: actions/checkout@v2 + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }} + export_default_credentials: true + + - name: Authenticate Artifact Repository + run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet + + - name: Pull and run Looker docker image + # TODO: can we cache some layers of the image for faster download? + # we probably don't want to cache the final image for IP security... + run: | + docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 + # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl + docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 + docker logs -f looker-sdk-codegen-ci --until=30s & - uses: actions/setup-node@v1 with: @@ -60,30 +79,9 @@ jobs: echo "[Miner]" >> looker.ini echo "base_url=." >> looker.ini - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.2.0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }} - export_default_credentials: true - - - name: Authenticate Artifact Repository - run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - - name: Pull and run Looker docker image - # TODO: can we cache some layers of the image for faster download? - # we probably don't want to cache the final image for IP security... + - name: Check that Looker is ready run: | - docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 - # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl - docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 - docker logs -f looker-sdk-codegen-ci --until=30s & - python ${{ github.workspace }}/.github/scripts/wait_for_looker.py + ${{ github.workspace }}/.github/scripts/wait_for_looker.sh - name: Run unit tests run: yarn jest "packages/sdk-codegen(-utils|-scripts)/src" --reporters=default --reporters=jest-junit diff --git a/.github/workflows/lerna-publish.yml b/.github/workflows/lerna-publish.yml index d8c687268..dab726b92 100644 --- a/.github/workflows/lerna-publish.yml +++ b/.github/workflows/lerna-publish.yml @@ -29,6 +29,23 @@ jobs: with: persist-credentials: false ref: ${{ github.event.release.tag_name || github.event.inputs.ref }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v0.2.0 + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }} + export_default_credentials: true + + - name: Authenticate Artifact Repository + run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet + + - name: Pull and run Looker docker image + run: | + docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 + docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 + docker logs -f looker-sdk-codegen-ci --until=30s & + - uses: actions/setup-node@v2 with: node-version: 15 @@ -53,27 +70,10 @@ jobs: echo "base_url=https://self-signed.looker.com:19999" >> looker.ini echo "verify_ssl=False" >> looker.ini echo "timeout=30" >> looker.ini - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.2.0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }} - export_default_credentials: true - - - name: Authenticate Artifact Repository - run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet - - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Pull and run Looker docker image + - name: Check that Looker is ready run: | - docker pull --quiet us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 - docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/21_18 - docker logs -f looker-sdk-codegen-ci --until=30s & - python ${{ github.workspace }}/.github/scripts/wait_for_looker.py + ${{ github.workspace }}/.github/scripts/wait_for_looker.sh - name: Publish to NPM registry run: $(npm bin)/lerna publish from-package --yes --no-verify-access diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 63e7c4a67..f1b1c1837 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -66,16 +66,20 @@ jobs: steps: - name: Repo Checkout uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} + - name: Install Tox and any other packages run: | python -m pip install --upgrade pip pip install tox + - name: Run Unit Tests run: tox -e unit + - name: Upload pytest test results if: ${{ always() }} uses: actions/upload-artifact@v2 @@ -120,25 +124,17 @@ jobs: steps: - name: Repo Checkout uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Install Tox and any other packages - run: | - python -m pip install --upgrade pip - pip install tox - - name: Set LOOKERSDK_CLIENT_{ID,SECRET} - # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20 - run: | - echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV - echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV + - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0.2.0 with: project_id: ${{ secrets.GCP_PROJECT_ID }} service_account_key: ${{ secrets.GCP_AR_READER_SA_KEY }} export_default_credentials: true + + - name: Authenticate Artifact Repository + run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet + - name: Install docker on macos if: ${{ matrix.os == 'macos' }} uses: docker-practice/actions-setup-docker@v1 @@ -146,6 +142,7 @@ jobs: docker_channel: stable docker_buildx: false docker_cli_experimental: disabled + - name: Bump docker for mac memory if: ${{ matrix.os == 'macos' }} run: | @@ -161,8 +158,7 @@ jobs: done (( i )) && printf '\n' echo "-- Docker is ready." - - name: Authenticate Artifact Repository - run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet + - name: Pull and run Looker docker image # TODO: can we cache some layers of the image for faster download? # we probably don't want to cache the final image for IP security... @@ -171,9 +167,26 @@ jobs: # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }} docker logs -f looker-sdk-codegen-ci --until=30s & - python ${{ github.workspace }}/.github/scripts/wait_for_looker.py + + - name: Install Tox and any other packages + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Set LOOKERSDK_CLIENT_{ID,SECRET} + # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20 + run: | + echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV + echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV + + + - name: Check that Looker is ready + run: | + ${{ github.workspace }}/.github/scripts/wait_for_looker.sh + - name: Run Integration Tests run: tox -e integration + - name: Upload pytest test results if: ${{ always() }} uses: actions/upload-artifact@v2 diff --git a/.github/workflows/tssdk-ci.yml b/.github/workflows/tssdk-ci.yml index 38a38992d..47cb0a5bd 100644 --- a/.github/workflows/tssdk-ci.yml +++ b/.github/workflows/tssdk-ci.yml @@ -107,22 +107,6 @@ jobs: - name: Repo Checkout uses: actions/checkout@v2 - - name: Install Node ${{ matrix.node-version }} - uses: actions/setup-node@v2 - with: - node-version: ${{ matrix.node-version }} - - - name: Install dependencies - run: | - yarn - yarn build - - - name: Set LOOKERSDK_CLIENT_{ID,SECRET} - # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20 - run: | - echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV - echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV - - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0.2.0 with: @@ -133,11 +117,6 @@ jobs: - name: Authenticate Artifact Repository run: gcloud auth configure-docker us-west1-docker.pkg.dev --quiet - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.9 - - name: Pull and run Looker docker image # TODO: can we cache some layers of the image for faster download? # we probably don't want to cache the final image for IP security... @@ -146,7 +125,26 @@ jobs: # set $LOOKER_OPTS to --no-ssl if we want to turn off ssl docker run --name looker-sdk-codegen-ci -d -p 10000:9999 -p 20000:19999 us-west1-docker.pkg.dev/cloud-looker-sdk-codegen-cicd/looker/${{ matrix.looker }} docker logs -f looker-sdk-codegen-ci --until=30s & - python ${{ github.workspace }}/.github/scripts/wait_for_looker.py + + - name: Install Node ${{ matrix.node-version }} + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + run: | + yarn + yarn build + + - name: Set LOOKERSDK_CLIENT_{ID,SECRET} + # api client/secret stored as github secret per release, e.g. LOOKERSDK_CLIENT_ID__7_20 + run: | + echo "LOOKERSDK_CLIENT_ID=${{ secrets[format('LOOKERSDK_CLIENT_ID__{0}', matrix.looker)] }}" >> $GITHUB_ENV + echo "LOOKERSDK_CLIENT_SECRET=${{ secrets[format('LOOKERSDK_CLIENT_SECRET__{0}', matrix.looker)] }}" >> $GITHUB_ENV + + - name: Check that Looker is ready + run: | + ${{ github.workspace }}/.github/scripts/wait_for_looker.sh - name: Run Integration Tests run: yarn jest packages/sdk-node/test --reporters=default --reporters=jest-junit