diff --git a/.github/workflows/builders.yml b/.github/workflows/builders.yml index 75763d28..650ce655 100644 --- a/.github/workflows/builders.yml +++ b/.github/workflows/builders.yml @@ -1,39 +1,43 @@ +# yamllint --format github .github/workflows/builders.yml +--- name: builders on: push: - branches: [ master ] + branches: + - master paths: - - images/extension-builders/** + - images/extension-builders/** pull_request: - branches: [ master ] + branches: + - master paths: - - images/extension-builders/** + - images/extension-builders/** jobs: builders: runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v2 - - - name: Build the Docker image - run: make builders BUILDERS_TAG=${{ github.sha }} - - - uses: azure/docker-login@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} - if: github.event_name == 'push' - - - name: Push the Docker image - run: make builders.push BUILDERS_TAG=${{ github.sha }} - if: github.event_name == 'push' - - - name: Tag the Docker image as 'latest' - run: make builders.tag BUILDERS_TAG=${{ github.sha }} EXTRA_TAG=latest - if: github.ref == 'refs/heads/master' && github.event_name == 'push' - - - name: Push the 'latest' Docker image - run: make builders.push BUILDERS_TAG=latest - if: github.ref == 'refs/heads/master' && github.event_name == 'push' + - uses: actions/checkout@v2 + + - name: Build the Docker image + run: make builders BUILDERS_TAG=${{ github.sha }} + + - uses: azure/docker-login@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + if: github.event_name == 'push' + + - name: Push the Docker image + run: make builders.push BUILDERS_TAG=${{ github.sha }} + if: github.event_name == 'push' + + - name: Tag the Docker image as 'latest' + run: make builders.tag BUILDERS_TAG=${{ github.sha }} EXTRA_TAG=latest + if: github.ref == 'refs/heads/master' && github.event_name == 'push' + + - name: Push the 'latest' Docker image + run: make builders.push BUILDERS_TAG=latest + if: github.ref == 'refs/heads/master' && github.event_name == 'push' diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index 2c8b736d..eddf3019 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -1,85 +1,91 @@ # `name` value will appear "as is" in the badge. # See https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#adding-a-workflow-status-badge-to-your-repository +# yamllint --format github .github/workflows/commit.yaml +--- name: "build" on: push: branches: - - master + - master pull_request: branches: - - '**' + - '**' + # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. + # This is important because sometimes the macos build fails due to Docker installation. + # See https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: jobs: bin: name: "Build `getenvoy` and `e2e` binaries for use in e2e tests" runs-on: ubuntu-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 + - name: "Checkout" + uses: actions/checkout@v2 - - name: "Install Go" - uses: actions/setup-go@v2 - with: - go-version: '1.16.2' + - name: "Install Go" + uses: actions/setup-go@v2 + with: + go-version: '1.16.2' - - name: "Init on first use" - run: make init + - name: "Init on first use" + run: make init - - name: "Build `getenvoy` and `e2e` binaries" - run: make bin + - name: "Build `getenvoy` and `e2e` binaries" + run: make bin - - name: "Share `getenvoy` and `e2e` binaries with the downstream jobs" - uses: actions/upload-artifact@v2 - with: - name: bin - path: build/bin + - name: "Share `getenvoy` and `e2e` binaries with the downstream jobs" + uses: actions/upload-artifact@v2 + with: + name: bin + path: build/bin e2e_linux: name: "Run e2e tests on Linux" needs: - - bin + - bin runs-on: ubuntu-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 + - name: "Checkout" + uses: actions/checkout@v2 - - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" - uses: actions/download-artifact@v2 - with: - name: bin - path: build/bin + - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" + uses: actions/download-artifact@v2 + with: + name: bin + path: build/bin - - name: "Build language-specific Docker build images" - run: make builders + - name: "Build language-specific Docker build images" + run: make builders - - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" - run: ./ci/e2e/linux/run_tests.sh + - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" + run: ./ci/e2e/linux/run_tests.sh e2e_macos: name: "Run e2e tests on MacOS" needs: - - bin + - bin runs-on: macos-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 + - name: "Checkout" + uses: actions/checkout@v2 - - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" - uses: actions/download-artifact@v2 - with: - name: bin - path: build/bin + - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" + uses: actions/download-artifact@v2 + with: + name: bin + path: build/bin - - name: "Install 'Docker for Mac' (an older version that can be installed in CI environment)" - run: ./ci/e2e/macos/install_docker.sh + - name: "Install 'Docker for Mac' (an older version that can be installed in CI environment)" + run: ./ci/e2e/macos/install_docker.sh - - name: "Build language-specific Docker build images" - env: - # don't use `DOCKER_BUILDKIT=1`, `--build-arg BUILDKIT_INLINE_CACHE=1` and `--cache-from` - # options when using `Docker for Mac` in CI environment - USE_DOCKER_BUILDKIT_CACHE: 'no' - run: make builders + - name: "Build language-specific Docker build images" + env: + # don't use `DOCKER_BUILDKIT=1`, `--build-arg BUILDKIT_INLINE_CACHE=1` and `--cache-from` + # options when using `Docker for Mac` in CI environment + USE_DOCKER_BUILDKIT_CACHE: 'no' + run: make builders - - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" - run: ./ci/e2e/macos/run_tests.sh + - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" + run: ./ci/e2e/macos/run_tests.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 8d7a0907..974bc627 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,11 +1,17 @@ # `name` value will appear "as is" in the badge. # See https://docs.github.com/en/actions/configuring-and-managing-workflows/configuring-a-workflow#adding-a-workflow-status-badge-to-your-repository +# yamllint --format github .github/workflows/release.yaml +--- name: "release" on: push: tags: - - '**' + - '**' + # workflow_dispatch will let us manually trigger the workflow from GitHub actions dashboard. + # This is important because sometimes the macos build fails due to Docker installation. + # See https://docs.github.com/en/free-pro-team@latest/actions/managing-workflow-runs/manually-running-a-workflow + workflow_dispatch: jobs: getenvoy: @@ -42,129 +48,130 @@ jobs: name: "Release extension builder images" runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - # fetch all history for all tags and branches (needed to determine the right cache source image) - fetch-depth: 0 + - uses: actions/checkout@v2 + with: + # fetch all history for all tags and branches (needed to determine the right cache source image) + fetch-depth: 0 - - name: "Get tag name" - run: | # Trim "v" prefix in the release tag - RELEASE_TAG=${GITHUB_REF#refs/*/} - if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi - echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV + - name: "Get tag name" + run: | # Trim "v" prefix in the release tag + RELEASE_TAG=${GITHUB_REF#refs/*/} + if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV - - name: "Build extension builder images" - run: make builders BUILDERS_TAG=${{ env.RELEASE_VERSION }} + - name: "Build extension builder images" + run: make builders BUILDERS_TAG=${{ env.RELEASE_VERSION }} - - name: "Login into DockerHub" - uses: azure/docker-login@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: "Login into DockerHub" + uses: azure/docker-login@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} - - name: "Push extension builder images" - run: make builders.push BUILDERS_TAG=${{ env.RELEASE_VERSION }} + - name: "Push extension builder images" + run: make builders.push BUILDERS_TAG=${{ env.RELEASE_VERSION }} e2e_bin: name: "Build `e2e` binaries for use in e2e tests" runs-on: ubuntu-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 + - name: "Checkout" + uses: actions/checkout@v2 - - name: "Install Go" - uses: actions/setup-go@v2 - with: - go-version: '1.16.2' + - name: "Install Go" + uses: actions/setup-go@v2 + with: + go-version: '1.16.2' - - name: "Build `e2e` binaries" - run: make build/bin/linux/amd64/e2e build/bin/darwin/amd64/e2e + - name: "Build `e2e` binaries" + run: make build/bin/linux/amd64/e2e build/bin/darwin/amd64/e2e - - name: "Share `e2e` binaries with the downstream jobs" - uses: actions/upload-artifact@v2 - with: - name: bin - path: build/bin + - name: "Share `e2e` binaries with the downstream jobs" + uses: actions/upload-artifact@v2 + with: + name: bin + path: build/bin e2e_linux: name: "Run e2e tests on Linux" needs: - - getenvoy - - builders - - e2e_bin + - getenvoy + - builders + - e2e_bin runs-on: ubuntu-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 - - - name: "Get tag name" - run: | # Trim "v" prefix in the release tag - RELEASE_TAG=${GITHUB_REF#refs/*/} - if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi - echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV - echo "RELEASE_TAG=${RELEASE_TAG}" >> $GITHUB_ENV - - - name: "Download `e2e` binary pre-built by the upstream job" - uses: actions/download-artifact@v2 - with: - name: bin - path: build/bin - - - name: "Download `getenvoy` binary from GithHub release assets" - env: - INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Linux_x86_64.tar.gz - INPUT_VERSION: tags/${{ env.RELEASE_TAG }} - run: | - curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash - mkdir -p build/bin/linux/amd64 - tar -C build/bin/linux/amd64 -xf ${INPUT_FILE} getenvoy - - - name: "Pull extension builder images" - run: make builders.pull BUILDERS_TAG=${{ env.RELEASE_VERSION }} # pull Docker images in advance to make output of - # `getenvoy extension build | test | run` stable - - - name: "Run e2e tests using released `getenvoy` binary and published extension builder images" - run: ./ci/e2e/linux/run_tests.sh + - name: "Checkout" + uses: actions/checkout@v2 + + - name: "Get tag name" + run: | # Trim "v" prefix in the release tag + RELEASE_TAG=${GITHUB_REF#refs/*/} + if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV + echo "RELEASE_TAG=${RELEASE_TAG}" >> $GITHUB_ENV + + - name: "Download `e2e` binary pre-built by the upstream job" + uses: actions/download-artifact@v2 + with: + name: bin + path: build/bin + + - name: "Download `getenvoy` binary from GithHub release assets" + env: + INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Linux_x86_64.tar.gz + INPUT_VERSION: tags/${{ env.RELEASE_TAG }} + run: | + curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash + mkdir -p build/bin/linux/amd64 + tar -C build/bin/linux/amd64 -xf ${INPUT_FILE} getenvoy + + - name: "Pull extension builder images" + # pull Docker images in advance to make output of `getenvoy extension build | test | run` stable + run: make builders.pull BUILDERS_TAG=${{ env.RELEASE_VERSION }} + + - name: "Run e2e tests using released `getenvoy` binary and published extension builder images" + run: ./ci/e2e/linux/run_tests.sh e2e_macos: name: "Run e2e tests on MacOS" needs: - - getenvoy - - builders - - e2e_bin + - getenvoy + - builders + - e2e_bin runs-on: macos-latest steps: - - name: "Checkout" - uses: actions/checkout@v2 - - - name: "Get tag name" - run: | # Trim "v" prefix in the release tag - RELEASE_TAG=${GITHUB_REF#refs/*/} - if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi - echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV - echo "RELEASE_TAG=${RELEASE_TAG}" >> $GITHUB_ENV - - - name: "Download `e2e` binary pre-built by the upstream job" - uses: actions/download-artifact@v2 - with: - name: bin - path: build/bin - - - name: "Download `getenvoy` binary from GithHub release assets" - env: - INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Darwin_x86_64.tar.gz - INPUT_VERSION: tags/${{ env.RELEASE_TAG }} - run: | - curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash - mkdir -p build/bin/darwin/amd64 - tar -C build/bin/darwin/amd64 -xf ${INPUT_FILE} getenvoy - - - name: "Install 'Docker for Mac' (an older version that can be installed in CI environment)" - run: ./ci/e2e/macos/install_docker.sh - - - name: "Pull extension builder images" - run: make builders.pull BUILDERS_TAG=${{ env.RELEASE_VERSION }} # pull Docker images in advance to make output of - # `getenvoy extension build | test | run` stable - - - name: "Run e2e tests using released `getenvoy` binary and published extension builder images" - run: ./ci/e2e/macos/run_tests.sh + - name: "Checkout" + uses: actions/checkout@v2 + + - name: "Get tag name" + run: | # Trim "v" prefix in the release tag + RELEASE_TAG=${GITHUB_REF#refs/*/} + if [[ "${RELEASE_TAG}" = v* ]]; then RELEASE_VERSION="${RELEASE_TAG:1}"; else RELEASE_VERSION="${RELEASE_TAG}"; fi + echo "RELEASE_VERSION=${RELEASE_VERSION}" >> $GITHUB_ENV + echo "RELEASE_TAG=${RELEASE_TAG}" >> $GITHUB_ENV + + - name: "Download `e2e` binary pre-built by the upstream job" + uses: actions/download-artifact@v2 + with: + name: bin + path: build/bin + + - name: "Download `getenvoy` binary from GithHub release assets" + env: + INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Darwin_x86_64.tar.gz + INPUT_VERSION: tags/${{ env.RELEASE_TAG }} + run: | + curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash + mkdir -p build/bin/darwin/amd64 + tar -C build/bin/darwin/amd64 -xf ${INPUT_FILE} getenvoy + + - name: "Install 'Docker for Mac' (an older version that can be installed in CI environment)" + run: ./ci/e2e/macos/install_docker.sh + + - name: "Pull extension builder images" + # pull Docker images in advance to make output of + # `getenvoy extension build | test | run` stable + run: make builders.pull BUILDERS_TAG=${{ env.RELEASE_VERSION }} + + - name: "Run e2e tests using released `getenvoy` binary and published extension builder images" + run: ./ci/e2e/macos/run_tests.sh diff --git a/Makefile b/Makefile index 8373a8fe..98c91f56 100644 --- a/Makefile +++ b/Makefile @@ -196,8 +196,9 @@ lint: generate $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml ## Run the l # The goimports tool does not arrange imports in 3 blocks if there are already more than three blocks. # To avoid that, before running it, we collapse all imports in one block, then run the formatter. .PHONY: format -format: $(GOIMPORTS) ## Format all Go code +format: $(GOIMPORTS) $(SHFMT) ## Format all Go code @echo "--- format ---" + @$(SHFMT) -w . @$(LICENSER) apply -r "Tetrate" @find . -type f -name '*.go' | xargs gofmt -s -w @for f in `find . -name '*.go'`; do \ diff --git a/ci/e2e/macos/.licenserignore b/ci/e2e/macos/.licenserignore new file mode 100644 index 00000000..2ef951d6 --- /dev/null +++ b/ci/e2e/macos/.licenserignore @@ -0,0 +1,3 @@ +# install_docker.sh a copy of https://github.com/play-with-go/play-with-go/blob/d2a13db0ed4ac80b39ce727cd54f2438c93096dc/_scripts/macCISetup.sh +# Copyright (c) 2020, The play-with-go.dev Authors. All rights reserved. BSD-3-Clause License +install_docker.sh diff --git a/ci/e2e/macos/install_docker.sh b/ci/e2e/macos/install_docker.sh index 4f6464b0..bc8eb44f 100755 --- a/ci/e2e/macos/install_docker.sh +++ b/ci/e2e/macos/install_docker.sh @@ -1,41 +1,61 @@ #!/usr/bin/env bash -# Copyright 2020 Tetrate -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -TMP_DIR=$(mktemp -d) - -# Docker for Mac 2.0.0.3-ce-mac81,31259 (the last version of 'Docker for Mac' that can be installed in CI environment) -E2E_MACOS_DOCKER_CASK_VERSION="${E2E_MACOS_DOCKER_CASK_VERSION:-8ce4e89d10716666743b28c5a46cd54af59a9cc2}" - -# install Docker for Mac -pushd "${TMP_DIR}" -curl -L https://raw.githubusercontent.com/Homebrew/homebrew-cask/${E2E_MACOS_DOCKER_CASK_VERSION}/Casks/docker.rb > docker.rb -brew install --cask docker.rb -popd - -# follow instructions from: -# https://github.com/microsoft/azure-pipelines-image-generation/issues/738#issuecomment-496211237 -# https://github.com/microsoft/azure-pipelines-image-generation/issues/738#issuecomment-522301481 -sudo /Applications/Docker.app/Contents/MacOS/Docker --quit-after-install --unattended -nohup /Applications/Docker.app/Contents/MacOS/Docker --unattended > /dev/stdout & -while ! docker info 2> /dev/null; do - sleep 5 - echo "Waiting for docker service to be in the running state" +# Below is a copy of https://github.com/play-with-go/play-with-go/blob/d2a13db0ed4ac80b39ce727cd54f2438c93096dc/_scripts/macCISetup.sh +# The last change to this file was in response to this comment https://github.com/docker/for-mac/issues/2359#issuecomment-793595407 + +set -euo pipefail + +# We can't have this line because the default version of bash on mac os too old +# shopt -s inherit_errexit + +# With thanks/credit to https://github.com/docker/for-mac/issues/2359#issuecomment-607154849 + +# Update brew to make sure we're using the latest formulae +brew update + +############################### +# General +brew install bash +brew install gsed +brew install findutils +ln -s /usr/local/bin/gsed /usr/local/bin/sed +ln -s /usr/local/bin/gfind /usr/local/bin/find +hash -r +which sed +which find + +############################### +# Docker + +# Install Docker +brew install --cask docker + +# Allow the app to run without confirmation +xattr -d -r com.apple.quarantine /Applications/Docker.app + +# preemptively do docker.app's setup to avoid any gui prompts +sudo /bin/cp /Applications/Docker.app/Contents/Library/LaunchServices/com.docker.vmnetd /Library/PrivilegedHelperTools +sudo /bin/cp /Applications/Docker.app/Contents/Resources/com.docker.vmnetd.plist /Library/LaunchDaemons/ +sudo /bin/chmod 544 /Library/PrivilegedHelperTools/com.docker.vmnetd +sudo /bin/chmod 644 /Library/LaunchDaemons/com.docker.vmnetd.plist +sudo /bin/launchctl load /Library/LaunchDaemons/com.docker.vmnetd.plist + +# Run +[[ $(uname) == 'Darwin' ]] || { + echo "This function only runs on macOS." >&2 + exit 2 +} + +echo "-- Starting Docker.app, if necessary..." + +open -g -a /Applications/Docker.app || exit + +# Wait for the server to start up, if applicable. +i=0 +while ! docker system info &> /dev/null; do + ((i++ == 0)) && printf %s '-- Waiting for Docker to finish starting up...' || printf '.' + sleep 1 done +((i)) && printf '\n' -# sanity check -docker run --rm -t busybox date +echo "-- Docker is ready." diff --git a/pkg/binary/envoy/debug/node_test.go b/pkg/binary/envoy/debug/node_test.go index 15c89d99..d725000d 100644 --- a/pkg/binary/envoy/debug/node_test.go +++ b/pkg/binary/envoy/debug/node_test.go @@ -23,6 +23,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/tetratelabs/getenvoy/pkg/binary/envoy" "github.com/tetratelabs/getenvoy/pkg/binary/envoytest" ) @@ -38,17 +40,13 @@ func Test_debugging_outputs(t *testing.T) { for _, file := range files { path := filepath.Join(r.DebugStore(), file) f, err := os.Stat(path) - if err != nil { - t.Errorf("error stating %v: %v", path, err) - } + require.NoError(t, err, "error stating %v", path) if f.Size() < 1 { t.Errorf("file %v was empty", path) } if strings.HasSuffix(file, ".json") { raw, err := ioutil.ReadFile(path) - if err != nil { - t.Errorf("error to read the file %v: %v", path, err) - } + require.NoError(t, err, "error to read the file %v", path) var is []interface{} if err := json.Unmarshal(raw, &is); err != nil { t.Errorf("error to unmarshal json string, %v: \"%v\"", err, raw)