ci: use azp to generate docs (envoyproxy#13481)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

ci use azp to sync filter example (envoyproxy#13501)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

ci: use azp for api and go-control-plane sync (envoyproxy#13550)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

ci: use multiple stage (envoyproxy#13557)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

ci: various improvements (envoyproxy#13660)

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

mac ci: try ignoring update failure (envoyproxy#13658)

Signed-off-by: Asra Ali <asraa@google.com>

revert run_docker.sh

fix

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

fix

Signed-off-by: Lizan Zhou <lizan@tetrate.io>

.azure-pipelines/pipelines.yml

@@ -10,115 +10,282 @@ trigger:
 # PR build config is manually overridden in Azure pipelines UI with different secrets
 pr: none
 
-jobs:
-  - job: format
-    dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
-    pool:
-      vmImage: "ubuntu-16.04"
-    steps:
-      - task: CacheBeta@1
-        inputs:
-          key: "format | ./WORKSPACE | **/*.bzl"
-          path: $(Build.StagingDirectory)/repository_cache
-
-      - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh'
-        workingDirectory: $(Build.SourcesDirectory)
-        env:
-          ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
-          BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-          BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
-          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-        displayName: "Run check format scripts"
-
-      - task: PublishBuildArtifacts@1
-        inputs:
-          pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff"
-          artifactName: format
-        condition: failed()
-
-  - job: release
-    displayName: "Linux-x64 release"
-    dependsOn: ["format"]
-    # For master builds, continue even if format fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
-    timeoutInMinutes: 360
-    pool:
-      vmImage: "ubuntu-16.04"
-    steps:
-      - template: bazel.yml
-        parameters:
-          ciTarget: bazel.release
-
-  - job: bazel
-    displayName: "Linux-x64"
-    dependsOn: ["release"]
-    # For master builds, continue even if format fails
-    condition: and(not(canceled()), or(succeeded(), ne(variables['Build.Reason'], 'PullRequest')))
-    strategy:
-      maxParallel: 3
-      matrix:
-        gcc:
-          CI_TARGET: "bazel.gcc"
-        clang_tidy:
-          CI_TARGET: "bazel.clang_tidy"
-        asan:
-          CI_TARGET: "bazel.asan"
-        tsan:
-          CI_TARGET: "bazel.tsan"
-        compile_time_options:
-          CI_TARGET: "bazel.compile_time_options"
-    timeoutInMinutes: 360
-    pool:
-      vmImage: "Ubuntu 16.04"
-    steps:
-      - template: bazel.yml
-        parameters:
-          ciTarget: $(CI_TARGET)
-
-  - job: docker
-    displayName: "Linux-x64 docker"
-    dependsOn: ["release"]
-    condition: and(succeeded(), eq(variables['PostSubmit'], 'true'), ne(variables['Build.Reason'], 'PullRequest'))
-    pool:
-      vmImage: "ubuntu-16.04"
-    steps:
-      - task: DownloadBuildArtifacts@0
-        inputs:
-          buildType: current
-          artifactName: "bazel.release"
-          itemPattern: "bazel.release/envoy_binary.tar.gz"
-          downloadType: single
-          targetPath: $(Build.StagingDirectory)
-
-      - bash: |
-          set -e
-          tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
-          ci/docker_ci.sh
-        workingDirectory: $(Build.SourcesDirectory)
-        env:
-          AZP_BRANCH: $(Build.SourceBranch)
-          AZP_SHA1: $(Build.SourceVersion)
-          DOCKERHUB_USERNAME: $(DockerUsername)
-          DOCKERHUB_PASSWORD: $(DockerPassword)
-
-  - job: macOS
-    dependsOn: ["format"]
-    timeoutInMinutes: 360
-    pool:
-      vmImage: "macos-latest"
-    steps:
-      - script: ./ci/mac_ci_setup.sh
-        displayName: "Install dependencies"
-
-      - script: ./ci/mac_ci_steps.sh
-        displayName: "Run Mac CI"
-        env:
-          BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
-          BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
-          GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
-
-      - task: PublishTestResults@2
-        inputs:
-          testResultsFiles: "**/bazel-testlogs/**/test.xml"
-          testRunTitle: "macOS"
-        condition: always()
+stages:
+  - stage: precheck
+    jobs:
+      - job: format
+        dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - task: Cache@2
+            inputs:
+              key: "format | ./WORKSPACE | **/*.bzl"
+              path: $(Build.StagingDirectory)/repository_cache
+            continueOnError: true
+
+          - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+            displayName: "Run check format scripts"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff"
+              artifactName: format
+            condition: failed()
+
+      - job: docs
+        dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel.
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - task: Cache@2
+            inputs:
+              key: "docs | ./WORKSPACE | **/*.bzl"
+              path: $(Build.StagingDirectory)/repository_cache
+            continueOnError: true
+
+          - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs'
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+            displayName: "Generate docs"
+
+          - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs'
+            displayName: "Upload Docs to GCS"
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              pathtoPublish: "$(Build.SourcesDirectory)/generated/docs"
+              artifactName: docs
+
+          - task: InstallSSHKey@0
+            inputs:
+              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+              sshPublicKey: "$(DocsPublicKey)"
+              sshPassphrase: "$(SshDeployKeyPassphrase)"
+              sshKeySecureFile: "$(DocsPrivateKey)"
+            condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
+
+          - script: docs/publish.sh
+            displayName: "Publish to GitHub"
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              AZP_BRANCH: $(Build.SourceBranch)
+              AZP_SHA1: $(Build.SourceVersion)
+            condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
+
+  - stage: sync
+    condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true))
+    dependsOn: []
+    jobs:
+      - job: filter_example
+        dependsOn: []
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - task: InstallSSHKey@0
+            inputs:
+              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+              sshPublicKey: "$(FilterExamplePublicKey)"
+              sshPassphrase: "$(SshDeployKeyPassphrase)"
+              sshKeySecureFile: "$(FilterExamplePrivateKey)"
+
+          - bash: ci/filter_example_mirror.sh
+            displayName: "Sync envoy-filter-example"
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              AZP_BRANCH: $(Build.SourceBranch)
+
+      - job: data_plane_api
+        dependsOn: []
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - task: InstallSSHKey@0
+            inputs:
+              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+              sshPublicKey: "$(DataPlaneApiPublicKey)"
+              sshPassphrase: "$(SshDeployKeyPassphrase)"
+              sshKeySecureFile: "$(DataPlaneApiPrivateKey)"
+
+          - bash: ci/api_mirror.sh
+            displayName: "Sync data-plane-api"
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              AZP_BRANCH: $(Build.SourceBranch)
+
+      - job: go_control_plane
+        dependsOn: []
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - task: InstallSSHKey@0
+            inputs:
+              hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ=="
+              sshPublicKey: "$(GoControlPlanePublicKey)"
+              sshPassphrase: "$(SshDeployKeyPassphrase)"
+              sshKeySecureFile: "$(GoControlPlanePrivateKey)"
+
+          - bash: |
+              cp -a ~/.ssh $(Build.StagingDirectory)/
+              ci/run_envoy_docker.sh 'ci/go_mirror.sh'
+            displayName: "Sync go-control-plane"
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              AZP_BRANCH: $(Build.SourceBranch)
+
+  - stage: linux_x64
+    dependsOn: ["precheck"]
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
+    jobs:
+      - job: release
+        timeoutInMinutes: 120
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - template: bazel.yml
+            parameters:
+              ciTarget: bazel.release
+
+  - stage: linux_arm64
+    dependsOn: ["precheck"]
+    # For post-submit builds, continue even if precheck fails
+    condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true)))
+    jobs:
+      - job: release
+        timeoutInMinutes: 120
+        pool: "arm-large"
+        steps:
+          - template: bazel.yml
+            parameters:
+              managedAgent: false
+              ciTarget: bazel.release.server_only
+              rbe: false
+              artifactSuffix: ".arm64"
+              bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base"
+
+  - stage: check
+    dependsOn: ["linux_x64"]
+    jobs:
+      - job: bazel
+        displayName: "linux_x64"
+        dependsOn: []
+        strategy:
+          maxParallel: 3
+          matrix:
+            api:
+              CI_TARGET: "bazel.api"
+            gcc:
+              CI_TARGET: "bazel.gcc"
+            clang_tidy:
+              CI_TARGET: "bazel.clang_tidy"
+            asan:
+              CI_TARGET: "bazel.asan"
+            tsan:
+              CI_TARGET: "bazel.tsan"
+            compile_time_options:
+              CI_TARGET: "bazel.compile_time_options"
+        timeoutInMinutes: 120
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - template: bazel.yml
+            parameters:
+              ciTarget: $(CI_TARGET)
+
+      - job: coverage
+        displayName: "linux_x64"
+        dependsOn: []
+        timeoutInMinutes: 120
+        pool: "x64-large"
+        strategy:
+          matrix:
+            coverage:
+              CI_TARGET: "coverage"
+        steps:
+          - template: bazel.yml
+            parameters:
+              managedAgent: false
+              ciTarget: bazel.$(CI_TARGET)
+              rbe: false
+              # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces
+              bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base"
+
+          - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)'
+            displayName: "Upload $(CI_TARGET) Report to GCS"
+            env:
+              ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory)
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+              GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket)
+            condition: always()
+
+  - stage: docker
+    dependsOn: ["linux_x64"]
+    jobs:
+      - job: docker
+        displayName: "linux multiarch"
+        pool:
+          vmImage: "ubuntu-16.04"
+        steps:
+          - bash: .azure-pipelines/cleanup.sh
+            displayName: "Removing tools from agent"
+          - task: DownloadBuildArtifacts@0
+            inputs:
+              buildType: current
+              artifactName: "bazel.release"
+              itemPattern: "bazel.release/envoy_binary.tar.gz"
+              downloadType: single
+              targetPath: $(Build.StagingDirectory)
+          - bash: |
+              set -e
+              tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz
+              ci/docker_ci.sh
+            workingDirectory: $(Build.SourcesDirectory)
+            env:
+              AZP_BRANCH: $(Build.SourceBranch)
+              AZP_SHA1: $(Build.SourceVersion)
+              DOCKERHUB_USERNAME: $(DockerUsername)
+              DOCKERHUB_PASSWORD: $(DockerPassword)
+
+  - stage: macos
+    dependsOn: ["precheck"]
+    jobs:
+      - job: test
+        timeoutInMinutes: 180
+        pool:
+          vmImage: "macos-latest"
+        steps:
+          - script: ./ci/mac_ci_setup.sh
+            displayName: "Install dependencies"
+
+          - script: ./ci/mac_ci_steps.sh
+            displayName: "Run Mac CI"
+            env:
+              BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2"
+              BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com
+              BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance
+              GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey)
+
+          - task: PublishTestResults@2
+            inputs:
+              testResultsFiles: "**/bazel-testlogs/**/test.xml"
+              testRunTitle: "macOS"
+            condition: always()