Skip to content

Add sinks to write to GCP Bigquery #2

Add sinks to write to GCP Bigquery

Add sinks to write to GCP Bigquery #2

Workflow file for this run

# K8s E2E Suite
#
# This workflow runs under any of the following conditions:
# - manual dispatch in GH UI
# - on a PR commit if the kubernetes_logs source was changed
# - in the merge queue
# - on a schedule at midnight UTC Tue-Sat
# - on demand by either of the following comments in a PR:
# - '/ci-run-k8s'
# - '/ci-run-all'
#
# If the workflow trigger is the nightly schedule, all the k8s versions
# are run in the matrix, otherwise, only the latest is run.
name: K8S E2E Suite
on:
workflow_dispatch:
workflow_call:
pull_request:
merge_group:
types: [checks_requested]
schedule:
# At midnight UTC Tue-Sat
- cron: '0 0 * * 2-6'
concurrency:
# In flight runs will be canceled through re-trigger in the merge queue, scheduled run, or if
# additional PR commits are pushed. The comment.html_url should always be unique.
#
# Note that technically this workflow can run on PRs which have code changes that affect K8s. Choosing not to add the PR commit to
# the concurrency group settings- since that would result in new PR commits canceling out manual runs on any PR that doesn't flag
# change detection. This is a "conservative" approach that means we may have some runs that could be canceled, but it's safer than
# having user's runs canceled when they shouldn't be. In practice this shouldn't happen very often given this component does not change
# often so any increased cost from the conservative approach should be negligible.
group: ${{ github.workflow }}-${{ github.event.comment.html_url || github.ref || github.event.schedule }}
cancel-in-progress: true
env:
AWS_ACCESS_KEY_ID: "dummy"
AWS_SECRET_ACCESS_KEY: "dummy"
CONTAINER_TOOL: "docker"
RUST_BACKTRACE: full
TEST_LOG: vector=debug
VERBOSE: true
DISABLE_MOLD: true
CI: true
PROFILE: debug
jobs:
changes:
# Only evaluate files changed on pull request trigger
if: github.event_name == 'pull_request'
uses: ./.github/workflows/changes.yml
with:
base_ref: ${{ github.event.pull_request.base.ref }}
head_ref: ${{ github.event.pull_request.head.ref }}
secrets: inherit
build-x86_64-unknown-linux-gnu:
name: Build - x86_64-unknown-linux-gnu
runs-on: ubuntu-20.04-4core
timeout-minutes: 45
needs: changes
# Run this job even if `changes` job is skipped (non- pull request trigger)
if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }}
# cargo-deb requires a release build, but we don't need optimizations for tests
env:
CARGO_PROFILE_RELEASE_OPT_LEVEL: 0
CARGO_PROFILE_RELEASE_CODEGEN_UNITS: 256
CARGO_INCREMENTAL: 0
steps:
- name: (PR review) Set latest commit status as pending
if: ${{ github.event_name == 'pull_request_review' }}
uses: myrotvorets/set-commit-status-action@v2.0.1
with:
sha: ${{ github.event.review.commit_id }}
token: ${{ secrets.GITHUB_TOKEN }}
status: pending
- name: (PR review) Checkout PR branch
if: ${{ github.event_name == 'pull_request_review' }}
uses: actions/checkout@v3
with:
ref: ${{ github.event.review.commit_id }}
- name: Checkout branch
if: ${{ github.event_name != 'pull_request_review' }}
uses: actions/checkout@v3
- uses: actions/cache@v4
with:
path: |
~/.cargo/registry
~/.cargo/git
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- run: sudo -E bash scripts/environment/bootstrap-ubuntu-20.04.sh
- run: bash scripts/environment/prepare.sh
- run: echo "::add-matcher::.github/matchers/rust.json"
- run: VECTOR_VERSION="$(cargo vdev version)" make package-deb-x86_64-unknown-linux-gnu
- uses: actions/upload-artifact@v3
with:
name: e2e-test-deb-package
path: target/artifacts/*
- name: (PR review) Set latest commit status as 'failure'
uses: myrotvorets/set-commit-status-action@v2.0.1
if: failure() && github.event_name == 'pull_request_review'
with:
sha: ${{ github.event.review.commit_id }}
token: ${{ secrets.GITHUB_TOKEN }}
status: 'failure'
# GitHub Actions don't support `matrix` at the job-level `if:` condition.
# We apply this workaround - compute `matrix` in a preceding job, and assign
# it's value dynamically at the actual test job.
# This approach can be advanced further by, for instance, dynamically
# detecting versions of various components, or reading them from `.meta`.
# See https://git.luolix.topmunity/t/feature-request-and-use-case-example-to-allow-matrix-in-if-s/126067
compute-k8s-test-plan:
name: Compute K8s test plan
runs-on: ubuntu-latest
timeout-minutes: 5
needs: changes
# Run this job even if `changes` job is skipped
if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }}
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- uses: actions/github-script@v7.0.1
id: set-matrix
with:
script: |
// Parameters.
const minikube_version = [
"v1.33.1",
]
// Aim to test against oldest supported k8s cloud-provider versions
// https://docs.aws.amazon.com/eks/latest/userguide/kubernetes-versions.html
// https://cloud.google.com/kubernetes-engine/docs/release-notes
// https://docs.microsoft.com/en-us/azure/aks/supported-kubernetes-versions?tabs=azure-cli#aks-kubernetes-release-calendar
const kubernetes_version = [
{ version: "v1.23.3", is_essential: true },
{ version: "v1.22.5", is_essential: false },
{ version: "v1.21.8", is_essential: false },
{ version: "v1.20.14", is_essential: false },
{ version: "v1.19.8", is_essential: false },
]
const container_runtime = [
"docker",
"containerd",
// https://github.com/kubernetes/minikube/issues/12928
// "crio",
]
// Run all versions if triggered by nightly schedule. Otherwise only run latest.
const run_all = context.eventName == "schedule";
const filter_targets = array => array.filter(val => run_all || val.is_essential)
const matrix = {
minikube_version,
kubernetes_version: filter_targets(kubernetes_version).map(e => ({
version: e.version,
role: e.is_essential ? "essential" : "extra",
})),
container_runtime,
}
core.setOutput('matrix', matrix)
- name: Dump matrix context
env:
MATRIX_CONTEXT: ${{ toJson(steps.set-matrix.outputs.matrix) }}
run: echo "$MATRIX_CONTEXT"
test-e2e-kubernetes:
name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }})
runs-on: ubuntu-20.04-4core
timeout-minutes: 45
needs:
- build-x86_64-unknown-linux-gnu
- compute-k8s-test-plan
# because `changes` job might be skipped
if: always() && needs.build-x86_64-unknown-linux-gnu.result == 'success' && needs.compute-k8s-test-plan.result == 'success'
strategy:
matrix: ${{ fromJson(needs.compute-k8s-test-plan.outputs.matrix) }}
fail-fast: false
steps:
- name: (PR review) Get PR branch
if: ${{ github.event_name == 'pull_request_review' }}
uses: xt0rted/pull-request-comment-branch@v2
id: comment-branch
- name: (PR review) Checkout PR branch
if: ${{ github.event_name == 'pull_request_review' }}
uses: actions/checkout@v3
with:
ref: ${{ steps.comment-branch.outputs.head_ref }}
- name: Checkout branch
if: ${{ github.event_name != 'pull_request_review' }}
uses: actions/checkout@v3
- uses: actions/download-artifact@v3
with:
name: e2e-test-deb-package
path: target/artifacts
- name: Setup Minikube
run: scripts/ci-setup-minikube.sh
env:
KUBERNETES_VERSION: ${{ matrix.kubernetes_version.version }}
MINIKUBE_VERSION: ${{ matrix.minikube_version }}
CONTAINER_RUNTIME: ${{ matrix.container_runtime }}
- run: make test-e2e-kubernetes
env:
USE_MINIKUBE_CACHE: "true"
SKIP_PACKAGE_DEB: "true"
CARGO_INCREMENTAL: 0
- name: (PR review) Set latest commit status as failure
uses: myrotvorets/set-commit-status-action@v2.0.1
if: failure() && github.event_name == 'pull_request_review'
with:
sha: ${{ github.event.review.commit_id }}
token: ${{ secrets.GITHUB_TOKEN }}
status: 'failure'
final-result:
name: K8s E2E Suite
runs-on: ubuntu-latest
timeout-minutes: 5
needs:
- changes
- build-x86_64-unknown-linux-gnu
- compute-k8s-test-plan
- test-e2e-kubernetes
if: always()
env:
FAILED: ${{ contains(needs.*.result, 'failure') }}
steps:
- name: (PR review) Get PR branch
if: github.event_name == 'pull_request_review' && env.FAILED != 'true'
uses: xt0rted/pull-request-comment-branch@v2
id: comment-branch
- name: (PR review) Submit PR result as success
if: github.event_name == 'pull_request_review' && env.FAILED != 'true'
uses: myrotvorets/set-commit-status-action@v2.0.1
with:
sha: ${{ github.event.review.commit_id }}
token: ${{ secrets.GITHUB_TOKEN }}
status: 'success'
- run: |
echo "failed=${{ env.FAILED }}"
if [[ "$FAILED" == "true" ]] ; then
exit 1
else
exit 0
fi