diff --git a/.github/ISSUE_TEMPLATE/minor-release.md b/.github/ISSUE_TEMPLATE/minor-release.md index 0ea25a30c0432..71ba54de40652 100644 --- a/.github/ISSUE_TEMPLATE/minor-release.md +++ b/.github/ISSUE_TEMPLATE/minor-release.md @@ -41,8 +41,8 @@ On the day of release: - [ ] Merge release preparation branch into the release branch - `git co v0. && git merge --ff-only prepare-v0.` - [ ] Tag new release - - [ ] `git tag v0..0 -a -m v0..0`` - - [ ] `git push origin v0..0 + - [ ] `git tag v0..0 -a -m v0..0` + - [ ] `git push origin v0..0` - [ ] Wait for release workflow to complete - Discoverable via [https://github.com/timberio/vector/actions/workflows/release.yml](https://github.com/timberio/vector/actions/workflows/release.yml) - [ ] Release updated Helm chart. See [releasing Helm chart](https://github.com/vectordotdev/helm-charts#releasing). diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index d9b13fcf742c0..4ccce23a22e37 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -36,6 +36,7 @@ Celkon Ceph Chromecast Citrix +Cloudflare Cloudfone Cmx Coby @@ -165,6 +166,7 @@ Samsung Sega Segoe Shopify +SIGINTs Simvalley Skype Skytex @@ -244,6 +246,7 @@ compiletime coredns corejs coreutils +curta daemonset databend datacenter @@ -295,6 +298,9 @@ gpg gql grafana graphiql +greptime +greptimecloud +greptimedb gvisor gws hadoop diff --git a/.github/actions/spelling/expect.txt b/.github/actions/spelling/expect.txt index ed5c241d38b0f..7b81b278e8665 100644 --- a/.github/actions/spelling/expect.txt +++ b/.github/actions/spelling/expect.txt @@ -1,9 +1,7 @@ abcd abcdefghijklm abcdefghijklmnopqrstuvwxyzand -abced abortable -acb ack'ing acking Acq @@ -37,7 +35,6 @@ ARNOTAREALIDD arshiyasolei asdf asdfasdf -ASMS assertverify Asterix asynk @@ -140,7 +137,6 @@ casttype castvalue cbe CBOR -cbs cddl cdeab cdylib @@ -304,7 +300,6 @@ ebfcee edenhill edns eeyun -efg efgh Elhage emca @@ -325,7 +320,6 @@ ENVARS envsubst EOIG EOL'ed -Erfxl Err'ing errorf Errorsfor @@ -469,7 +463,6 @@ hannes Hashbang hashbrown hashindex -hashlink hashring hashset hashsum @@ -778,6 +771,7 @@ NQTP nresamples nullishness numbackends +oahd oap OKD omfwd @@ -821,6 +815,7 @@ pathgen peekable PEMS pgmajfault +pgrep PII Pitbull pkc @@ -1053,7 +1048,6 @@ supertrait suser sustainability svalue -Sya sysfs sysinit syslogng @@ -1114,7 +1108,6 @@ Tomola tonydanza toolbars toolchains -TOOLSDIRECTORY toolset toor topdir diff --git a/.github/workflows/changes.yml b/.github/workflows/changes.yml index db6c4225073e0..653d3395bdfda 100644 --- a/.github/workflows/changes.yml +++ b/.github/workflows/changes.yml @@ -70,6 +70,8 @@ on: value: ${{ jobs.int_tests.outputs.fluent }} gcp: value: ${{ jobs.int_tests.outputs.gcp }} + greptimedb: + value: ${{ jobs.int_tests.outputs.greptimedb }} humio: value: ${{ jobs.int_tests.outputs.humio }} http-client: @@ -194,6 +196,7 @@ jobs: eventstoredb: ${{ steps.filter.outputs.eventstoredb }} fluent: ${{ steps.filter.outputs.fluent }} gcp: ${{ steps.filter.outputs.gcp }} + greptimedb: ${{ steps.filter.outputs.greptimedb }} humio: ${{ steps.filter.outputs.humio }} http-client: ${{ steps.filter.outputs.http-client }} influxdb: ${{ steps.filter.outputs.influxdb }} diff --git a/.github/workflows/comment-trigger.yml b/.github/workflows/comment-trigger.yml index 84cabd825ccf3..8cb2c3780887e 100644 --- a/.github/workflows/comment-trigger.yml +++ b/.github/workflows/comment-trigger.yml @@ -15,6 +15,7 @@ # /ci-run-unit-windows : runs Unit - Windows # /ci-run-environment : runs Environment Suite # /ci-run-regression : runs Regression Detection Suite +# /ci-run-k8s : runs K8s E2E Suite name: Comment Trigger @@ -55,15 +56,22 @@ jobs: || contains(github.event.comment.body, '/ci-run-unit-windows') || contains(github.event.comment.body, '/ci-run-environment') || contains(github.event.comment.body, '/ci-run-regression') + || contains(github.event.comment.body, '/ci-run-k8s') ) steps: + - name: Generate authentication token + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR comment author id: comment uses: tspascoal/get-user-teams-membership@v2 with: username: ${{ github.actor }} team: 'Vector' - GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership if: steps.comment.outputs.isTeamMember == 'false' @@ -116,3 +124,9 @@ jobs: if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-regression') uses: ./.github/workflows/regression.yml secrets: inherit + + k8s: + needs: validate + if: contains(github.event.comment.body, '/ci-run-all') || contains(github.event.comment.body, '/ci-run-k8s') + uses: ./.github/workflows/k8s_e2e.yml + secrets: inherit diff --git a/.github/workflows/environment.yml b/.github/workflows/environment.yml index cdddb0a980db8..7aa76a5cc2bbb 100644 --- a/.github/workflows/environment.yml +++ b/.github/workflows/environment.yml @@ -42,7 +42,7 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@v2.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.1 - name: Login to DockerHub uses: docker/login-action@v2.1.0 if: github.ref == 'refs/heads/master' diff --git a/.github/workflows/gardener_issue_comment.yml b/.github/workflows/gardener_issue_comment.yml index 8ea89fb315145..e066f40f49c87 100644 --- a/.github/workflows/gardener_issue_comment.yml +++ b/.github/workflows/gardener_issue_comment.yml @@ -20,50 +20,74 @@ jobs: GH_TOKEN: ${{ secrets.GH_PROJECT_PAT }} run: | issue_id=${{ github.event.issue.node_id }} + echo "issue_id: $issue_id" # IDs fetched from https://docs.github.com/en/graphql/overview/explorer project_id="PVT_kwDOAQFeYs4AAsTr" # Gardener status_field_id="PVTF_lADOAQFeYs4AAsTrzgAXRuU" # Status triage_option_id="2a08fafa" - # ensures that the issue is already on board but also seems to be the only way to fetch - # the item id - item_id="$(gh api graphql -f query=' - mutation($project_id: ID!, $content_id: ID!) { - addProjectV2ItemById(input: {projectId: $project_id, contentId: $content_id}) { - item { - id - } - } - }' -f project_id="$project_id" -f content_id="$issue_id" -q '.data.addProjectV2ItemById.item.id' - )" - - echo "item_id: $item_id" - - if [ -z "$item_id" ] ; then - echo "Issue not found in Gardener board" - exit 0 - else - echo "Found issue on Gardener board" - fi - - current_status="$(gh api graphql -f query=' + # Query for project items for the given issue + project_items="$(gh api graphql -f query=' query($item_id: ID!) { node(id: $item_id) { - ... on ProjectV2Item { - fieldValueByName(name: "Status") { - ... on ProjectV2ItemFieldSingleSelectValue { - name + ... on Issue { + projectItems(first: 50) { + ... on ProjectV2ItemConnection { + nodes { + fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { + name + } + } + ... on ProjectV2Item { + id + project { + ... on ProjectV2 { + id + } + } + } + } + } + } + } + ... on PullRequest { + projectItems(first: 50) { + ... on ProjectV2ItemConnection { + nodes { + fieldValueByName(name: "Status") { + ... on ProjectV2ItemFieldSingleSelectValue { + name + } + } + ... on ProjectV2Item { + id + project { + ... on ProjectV2 { + id + } + } + } + } } } } } - }' -f item_id="$item_id" + }' -f item_id="$issue_id" )" - current_status=$(echo $current_status | jq -c -r '.["data"]["node"]["fieldValueByName"]["name"]') + # Extract the item in the Gardener project + project=$(echo $project_items | jq -c -r --arg project_id $project_id '.data.node.projectItems.nodes[] | select(.project.id == $project_id)') + current_status=$(echo $project | jq -c '.fieldValueByName.name') + item_id=$(echo $project | jq -c '.id') - echo "Current issue status is: '${current_status}'" + if [ -z "$current_status" ] ; then + echo "Issue not found in Gardener board" + exit 0 + else + echo "Found issue on Gardener board. Current issue status is: '${current_status}'" + fi if [ "$current_status" = "Blocked / Waiting" ] ; then echo "Moving issue from 'Blocked / Waiting' to 'Triage'" diff --git a/.github/workflows/gardener_open_pr.yml b/.github/workflows/gardener_open_pr.yml index e9b0fef67ba05..4fc84c07cc5ea 100644 --- a/.github/workflows/gardener_open_pr.yml +++ b/.github/workflows/gardener_open_pr.yml @@ -13,12 +13,18 @@ jobs: runs-on: ubuntu-latest if: ${{ github.actor != 'dependabot[bot]' }} steps: + - name: Generate authentication token + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - uses: tspascoal/get-user-teams-membership@v2 id: checkVectorMember with: username: ${{ github.actor }} team: vector - GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.5.0 if: ${{ steps.checkVectorMember.outputs.isTeamMember == 'false' }} with: diff --git a/.github/workflows/integration-comment.yml b/.github/workflows/integration-comment.yml index a35f994f82f25..f6ed21bda71b5 100644 --- a/.github/workflows/integration-comment.yml +++ b/.github/workflows/integration-comment.yml @@ -31,6 +31,7 @@ env: AWS_SECRET_ACCESS_KEY: "dummy" AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} + TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} @@ -46,13 +47,19 @@ jobs: runs-on: ubuntu-latest if: contains(github.event.comment.body, '/ci-run-integration') || contains(github.event.comment.body, '/ci-run-all') steps: + - name: Generate authentication token + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Get PR comment author id: comment uses: tspascoal/get-user-teams-membership@v2 with: username: ${{ github.actor }} team: 'Vector' - GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: Validate author membership if: steps.comment.outputs.isTeamMember == 'false' @@ -69,107 +76,330 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} status: pending - test-integration: - uses: ./.github/workflows/integration-test.yml - with: - if: ${{ matrix.run.if }} - test_name: ${{ matrix.run.test_name }} + integration-tests: needs: prep-pr - secrets: inherit - strategy: - fail-fast: false - matrix: - run: - - test_name: 'amqp' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-amqp') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'appsignal' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-appsignal') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'aws' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-aws') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'axiom' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-axiom') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'azure' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-azure') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'clickhouse' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-clickhouse') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'databend' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-databend') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'datadog-agent' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'datadog-logs' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'datadog-metrics' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'datadog-traces' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'dnstap' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'docker-logs' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-docker-logs') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'elasticsearch' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-elasticsearch') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'eventstoredb' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-eventstoredb') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'fluent' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-fluent') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'gcp' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-gcp') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'humio' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-humio') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'http-client' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-http-client') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'influxdb' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-influxdb') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'kafka' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-kafka') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'logstash' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-logstash') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'loki' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-loki') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'mongodb' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-mongodb') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'nats' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-nats') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'nginx' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-nginx') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'opentelemetry' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-opentelemetry') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'postgres' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-postgres') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'prometheus' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-prometheus') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'pulsar' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-pulsar') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'redis' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-redis') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'shutdown' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-shutdown') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'splunk' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-splunk') || contains(github.event.comment.body, '/ci-run-all') }} - - test_name: 'webhdfs' - if: ${{ contains(github.event.comment.body, '/ci-run-integration-webhdfs') || contains(github.event.comment.body, '/ci-run-all') }} + runs-on: [linux, ubuntu-20.04-4core] + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - run: sudo npm -g install @datadog/datadog-ci + + - run: docker image prune -af ; docker container prune -f + + - name: amqp + if: ${{ contains(github.event.comment.body, '/ci-run-integration-amqp') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + # First one requires more time, as we need to build the image from scratch + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh amqp + + - name: appsignal + if: ${{ contains(github.event.comment.body, '/ci-run-integration-appsignal') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh appsignal + + - name: aws + if: ${{ contains(github.event.comment.body, '/ci-run-integration-aws') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh aws + + - name: axiom + if: ${{ contains(github.event.comment.body, '/ci-run-integration-axiom') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh axiom + + - name: azure + if: ${{ contains(github.event.comment.body, '/ci-run-integration-azure') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh azure + + - name: clickhouse + if: ${{ contains(github.event.comment.body, '/ci-run-integration-clickhouse') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh clickhouse + + - name: databend + if: ${{ contains(github.event.comment.body, '/ci-run-integration-databend') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh databend + + - name: datadog-agent + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-agent') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-agent + + - name: datadog-logs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-logs') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-logs + + - name: datadog-metrics + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-metrics') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-metrics + + - name: datadog-traces + if: ${{ contains(github.event.comment.body, '/ci-run-integration-datadog-traces') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-traces + + - name: dnstap + if: ${{ contains(github.event.comment.body, '/ci-run-integration-dnstap') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh dnstap + + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + + - name: docker-logs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-docker-logs') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh docker-logs + + - name: elasticsearch + if: ${{ contains(github.event.comment.body, '/ci-run-integration-elasticsearch') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh elasticsearch + + - name: eventstoredb + if: ${{ contains(github.event.comment.body, '/ci-run-integration-eventstoredb') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh eventstoredb + + - name: fluent + if: ${{ contains(github.event.comment.body, '/ci-run-integration-fluent') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh fluent + + - name: gcp + if: ${{ contains(github.event.comment.body, '/ci-run-integration-gcp') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh gcp + + - name: greptimedb + if: ${{ contains(github.event.comment.body, '/ci-run-integration-greptimedb') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh greptimedb + + - name: humio + if: ${{ contains(github.event.comment.body, '/ci-run-integration-humio') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh humio + + - name: http-client + if: ${{ contains(github.event.comment.body, '/ci-run-integration-http-client') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh http-client + + - name: influxdb + if: ${{ contains(github.event.comment.body, '/ci-run-integration-influxdb') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh influxdb + + - name: kafka + if: ${{ contains(github.event.comment.body, '/ci-run-integration-kafka') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh kafka + + - name: logstash + if: ${{ contains(github.event.comment.body, '/ci-run-integration-logstash') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh logstash + + - name: loki + if: ${{ contains(github.event.comment.body, '/ci-run-integration-loki') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh loki + + - name: mongodb + if: ${{ contains(github.event.comment.body, '/ci-run-integration-mongodb') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh mongodb + + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + + - name: nats + if: ${{ contains(github.event.comment.body, '/ci-run-integration-nats') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh nats + + - name: nginx + if: ${{ contains(github.event.comment.body, '/ci-run-integration-nginx') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh nginx + + - name: opentelemetry + if: ${{ contains(github.event.comment.body, '/ci-run-integration-opentelemetry') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh opentelemetry + + - name: postgres + if: ${{ contains(github.event.comment.body, '/ci-run-integration-postgres') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh postgres + + - name: prometheus + if: ${{ contains(github.event.comment.body, '/ci-run-integration-prometheus') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh prometheus + + - name: pulsar + if: ${{ contains(github.event.comment.body, '/ci-run-integration-pulsar') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh pulsar + + - name: redis + if: ${{ contains(github.event.comment.body, '/ci-run-integration-redis') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh redis + + - name: shutdown + if: ${{ contains(github.event.comment.body, '/ci-run-integration-shutdown') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh shutdown + + - name: splunk + if: ${{ contains(github.event.comment.body, '/ci-run-integration-splunk') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh splunk + + - name: webhdfs + if: ${{ contains(github.event.comment.body, '/ci-run-integration-webhdfs') || contains(github.event.comment.body, '/ci-run-all') }} + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh webhdfs update-pr-status: name: Signal result to PR runs-on: ubuntu-latest - needs: test-integration + needs: integration-tests if: always() && (contains(github.event.comment.body, '/ci-run-integration') || contains(github.event.comment.body, '/ci-run-all')) steps: + - name: Generate authentication token + id: generate_token + uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 + with: + app_id: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + private_key: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_PRIVATE_KEY }} - name: Validate issue comment if: github.event_name == 'issue_comment' uses: tspascoal/get-user-teams-membership@v2 with: username: ${{ github.actor }} team: 'Vector' - GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} + GITHUB_TOKEN: ${{ steps.generate_token.outputs.token }} - name: (PR comment) Get PR branch uses: xt0rted/pull-request-comment-branch@v2 id: comment-branch - - name: (PR comment) Submit PR result as ${{ needs.test-integration.result }} + - name: (PR comment) Submit PR result as ${{ needs.integration-tests.result }} uses: myrotvorets/set-commit-status-action@v1.1.7 with: sha: ${{ steps.comment-branch.outputs.head_sha }} token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ needs.test-integration.result }} + status: ${{ needs.integration-tests.result }} diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4d8d635fcb477..ce0dbb195ab4b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -1,22 +1,12 @@ -# This workflow is used to run an integration test. -# The most common use case is that it is triggered by another workflow, -# such as the Master Merge Queue Suite, or the Integration Comment. +# Integration Test # -# It can also be triggered on manual dispatch in CI however. -# In that use case, an input for the test name needs to be provided. +# This workflow is used to run an integration test on demand. +# An input for the test name needs to be provided. # TODO: check if the input is "all" , and run all, without a timeout? name: Integration Test on: - workflow_call: - inputs: - if: - required: false - type: boolean - test_name: - required: true - type: string workflow_dispatch: inputs: test_name: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 7f85870e7dff4..01a2a7dabca3a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -22,6 +22,9 @@ env: CONTAINER_TOOL: "docker" DD_ENV: "ci" DD_API_KEY: ${{ secrets.DD_API_KEY }} + TEST_DATADOG_API_KEY: ${{ secrets.CI_TEST_DATADOG_API_KEY }} + TEST_APPSIGNAL_PUSH_API_KEY: ${{ secrets.TEST_APPSIGNAL_PUSH_API_KEY }} + AXIOM_TOKEN: ${{ secrets.AXIOM_TOKEN }} RUST_BACKTRACE: full TEST_LOG: vector=debug VERBOSE: true @@ -34,6 +37,7 @@ env: jobs: changes: + if: github.event_name == 'pull_request' uses: ./.github/workflows/changes.yml with: base_ref: ${{ github.event.pull_request.base.ref }} @@ -42,94 +46,365 @@ jobs: int_tests: true secrets: inherit - # Calls the Integration Test workflow for each integration that was detected to have files changed that impact it. - integration-matrix: - uses: ./.github/workflows/integration-test.yml - with: - if: ${{ matrix.run.if }} - test_name: ${{ matrix.run.test_name }} - secrets: inherit + integration-tests: + name: Integration Tests + runs-on: [linux, ubuntu-20.04-4core] needs: changes - strategy: - fail-fast: false - matrix: - run: - - test_name: 'amqp' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.amqp == 'true' }} - - test_name: 'appsignal' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.appsignal == 'true' }} - - test_name: 'aws' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.aws == 'true' }} - - test_name: 'axiom' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.axiom == 'true' }} - - test_name: 'azure' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.azure == 'true' }} - - test_name: 'clickhouse' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.clickhouse == 'true' }} - - test_name: 'databend' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.databend == 'true' }} - - test_name: 'datadog-agent' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} - - test_name: 'datadog-logs' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} - - test_name: 'datadog-metrics' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} - - test_name: 'datadog-traces' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.datadog == 'true' }} - - test_name: 'dnstap' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.dnstap == 'true' }} - - test_name: 'docker-logs' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.docker-logs == 'true' }} - - test_name: 'elasticsearch' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.elasticsearch == 'true' }} - - test_name: 'eventstoredb' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.eventstoredb == 'true' }} - - test_name: 'fluent' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.fluent == 'true' }} - - test_name: 'gcp' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.gcp == 'true' }} - - test_name: 'humio' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.humio == 'true' }} - - test_name: 'http-client' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.http-client == 'true' }} - - test_name: 'influxdb' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.influxdb == 'true' }} - - test_name: 'kafka' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.kafka == 'true' }} - - test_name: 'logstash' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.logstash == 'true' }} - - test_name: 'loki' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.loki == 'true' }} - - test_name: 'mongodb' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.mongodb == 'true' }} - - test_name: 'nats' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.nats == 'true' }} - - test_name: 'nginx' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.nginx == 'true' }} - - test_name: 'opentelemetry' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.opentelemetry == 'true' }} - - test_name: 'postgres' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.postgres == 'true' }} - - test_name: 'prometheus' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.prometheus == 'true' }} - - test_name: 'pulsar' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.pulsar == 'true' }} - - test_name: 'redis' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.redis == 'true' }} - - test_name: 'shutdown' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' }} - - test_name: 'splunk' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.splunk == 'true' }} - - test_name: 'webhdfs' - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.int-all == 'true' || needs.changes.outputs.webhdfs == 'true' }} - - # This is a required status check, so it always needs to run if prior jobs failed, in order to mark the status correctly. - integration: + if: always() && ( + github.event_name == 'merge_group' || ( + needs.changes.outputs.all-int == 'true' + || needs.changes.outputs.amqp == 'true' + || needs.changes.outputs.appsignal == 'true' + || needs.changes.outputs.aws == 'true' + || needs.changes.outputs.axiom == 'true' + || needs.changes.outputs.azure == 'true' + || needs.changes.outputs.clickhouse == 'true' + || needs.changes.outputs.databend == 'true' + || needs.changes.outputs.datadog == 'true' + || needs.changes.outputs.dnstap == 'true' + || needs.changes.outputs.docker-logs == 'true' + || needs.changes.outputs.elasticsearch == 'true' + || needs.changes.outputs.eventstoredb == 'true' + || needs.changes.outputs.fluent == 'true' + || needs.changes.outputs.gcp == 'true' + || needs.changes.outputs.greptimedb == 'true' + || needs.changes.outputs.humio == 'true' + || needs.changes.outputs.http-client == 'true' + || needs.changes.outputs.influxdb == 'true' + || needs.changes.outputs.kafka == 'true' + || needs.changes.outputs.logstash == 'true' + || needs.changes.outputs.loki == 'true' + || needs.changes.outputs.mongodb == 'true' + || needs.changes.outputs.nats == 'true' + || needs.changes.outputs.nginx == 'true' + || needs.changes.outputs.opentelemetry == 'true' + || needs.changes.outputs.postgres == 'true' + || needs.changes.outputs.prometheus == 'true' + || needs.changes.outputs.pulsar == 'true' + || needs.changes.outputs.redis == 'true' + || needs.changes.outputs.splunk == 'true' + || needs.changes.outputs.webhdfs == 'true' + ) + ) + timeout-minutes: 75 + steps: + - uses: actions/checkout@v3 + with: + submodules: "recursive" + + - run: sudo npm -g install @datadog/datadog-ci + + - run: docker image prune -af ; docker container prune -f + + - name: Determine if secrets are defined (PR author is team member). + if: github.event_name == 'pull_request' + env: + GH_APP_DATADOG_VECTOR_CI_APP_ID: ${{ secrets.GH_APP_DATADOG_VECTOR_CI_APP_ID }} + run: | + if [[ "$GH_APP_DATADOG_VECTOR_CI_APP_ID" != "" ]] ; then + echo "PR_HAS_ACCESS_TO_SECRETS=true" >> "$GITHUB_ENV" + else + echo "PR_HAS_ACCESS_TO_SECRETS=false" >> "$GITHUB_ENV" + fi + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.amqp == 'true' }} + name: amqp + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh amqp + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.appsignal == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: appsignal + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh appsignal + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.aws == 'true' }} + name: aws + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh aws + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.axiom == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: axiom + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh axiom + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.azure == 'true' }} + name: azure + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh azure + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.clickhouse == 'true' }} + name: clickhouse + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh clickhouse + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.databend == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: databend + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh databend + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-agent + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-agent + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-logs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-logs + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-metrics + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-metrics + + - if: (github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.datadog == 'true') && + (github.event_name != 'pull_request' || env.PR_HAS_ACCESS_TO_SECRETS == 'true') + name: datadog-traces + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh datadog-traces + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.dnstap == 'true' }} + name: dnstap + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh dnstap + + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.docker-logs == 'true' }} + name: docker-logs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh docker-logs + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.elasticsearch == 'true' }} + name: elasticsearch + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh elasticsearch + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.eventstoredb == 'true' }} + name: eventstoredb + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh eventstoredb + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.fluent == 'true' }} + name: fluent + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh fluent + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.gcp == 'true' }} + name: gcp + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh gcp + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.greptimedb == 'true' }} + name: greptimedb + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh greptimedb + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.humio == 'true' }} + name: humio + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh humio + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.http-client == 'true' }} + name: http-client + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh http-client + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.influxdb == 'true' }} + name: influxdb + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh influxdb + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.kafka == 'true' }} + name: kafka + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh kafka + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.logstash == 'true' }} + name: logstash + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh logstash + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.loki == 'true' }} + name: loki + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh loki + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.mongodb == 'true' }} + name: mongodb + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh mongodb + + - run: docker image prune -af --filter=label!=vector-test-runner=true ; docker container prune -f + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nats == 'true' }} + name: nats + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh nats + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.nginx == 'true' }} + name: nginx + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh nginx + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.opentelemetry == 'true' }} + name: opentelemetry + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh opentelemetry + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.postgres == 'true' }} + name: postgres + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh postgres + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.prometheus == 'true' }} + name: prometheus + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh prometheus + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.pulsar == 'true' }} + name: pulsar + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh pulsar + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.redis == 'true' }} + name: redis + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh redis + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' }} + name: shutdown + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh shutdown + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.splunk == 'true' }} + name: splunk + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh splunk + + - if: ${{ github.event_name == 'merge_group' || needs.changes.outputs.all-int == 'true' || needs.changes.outputs.webhdfs == 'true' }} + name: webhdfs + uses: nick-fields/retry@v2 + with: + timeout_minutes: 30 + max_attempts: 3 + command: bash scripts/ci-integration-test.sh webhdfs + + integration-test-suite: name: Integration Test Suite runs-on: ubuntu-latest if: always() needs: - - integration-matrix + - changes + - integration-tests env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: diff --git a/.github/workflows/k8s_e2e.yml b/.github/workflows/k8s_e2e.yml index e7faa3a59555a..a2c09a760a8cf 100644 --- a/.github/workflows/k8s_e2e.yml +++ b/.github/workflows/k8s_e2e.yml @@ -16,9 +16,8 @@ name: K8S E2E Suite on: workflow_dispatch: + workflow_call: pull_request: - issue_comment: - types: [created] merge_group: types: [checks_requested] schedule: @@ -27,7 +26,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.number || github.event.comment.html_url || github.event.merge_group.head_sha || github.event.schedule || github.sha }} - cancel-in-progress: true env: @@ -42,31 +40,9 @@ env: PROFILE: debug jobs: - validate: - name: Validate comment - runs-on: ubuntu-latest - if: | - github.event_name != 'issue_comment' || - ( github.event.issue.pull_request && - ( contains(github.event.comment.body, '/ci-run-all') || - contains(github.event.comment.body, '/ci-run-k8s') - ) - ) - steps: - - name: Get PR comment author - id: comment - uses: tspascoal/get-user-teams-membership@v2 - with: - username: ${{ github.actor }} - team: 'Vector' - GITHUB_TOKEN: ${{ secrets.GH_PAT_ORG }} - - - name: Validate author membership - if: steps.comment.outputs.isTeamMember == 'false' - run: exit 1 - changes: - needs: validate + # Only evaluate files changed on pull request trigger + if: github.event_name == 'pull_request' uses: ./.github/workflows/changes.yml with: base_ref: ${{ github.event.pull_request.base.ref }} @@ -76,8 +52,9 @@ jobs: build-x86_64-unknown-linux-gnu: name: Build - x86_64-unknown-linux-gnu runs-on: [linux, ubuntu-20.04-4core] - needs: [changes, validate] - if: github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true' + needs: changes + # Run this job even if `changes` job is skipped (non- pull request trigger) + if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} # cargo-deb requires a release build, but we don't need optimizations for tests env: CARGO_PROFILE_RELEASE_OPT_LEVEL: 0 @@ -141,8 +118,9 @@ jobs: compute-k8s-test-plan: name: Compute K8s test plan runs-on: ubuntu-latest - needs: [changes, validate] - if: github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true' + needs: changes + # Run this job even if `changes` job is skipped + if: ${{ !failure() && !cancelled() && (github.event_name != 'pull_request' || needs.changes.outputs.k8s == 'true') }} outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: @@ -195,9 +173,10 @@ jobs: name: K8s ${{ matrix.kubernetes_version.version }} / ${{ matrix.container_runtime }} (${{ matrix.kubernetes_version.role }}) runs-on: [linux, ubuntu-20.04-4core] needs: - - validate - build-x86_64-unknown-linux-gnu - compute-k8s-test-plan + # because `changes` job might be skipped + if: always() && needs.build-x86_64-unknown-linux-gnu.result == 'success' && needs.compute-k8s-test-plan.result == 'success' strategy: matrix: ${{ fromJson(needs.compute-k8s-test-plan.outputs.matrix) }} fail-fast: false @@ -246,20 +225,22 @@ jobs: final-result: name: K8s E2E Suite runs-on: ubuntu-latest - needs: [test-e2e-kubernetes, validate] - if: | - always() && (github.event_name != 'issue_comment' || (github.event.issue.pull_request - && (contains(github.event.comment.body, '/ci-run-k8s') || contains(github.event.comment.body, '/ci-run-all')))) + needs: + - changes + - build-x86_64-unknown-linux-gnu + - compute-k8s-test-plan + - test-e2e-kubernetes + if: always() env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: - name: (PR comment) Get PR branch - if: success() && github.event_name == 'issue_comment' + if: github.event_name == 'issue_comment' && env.FAILED != 'true' uses: xt0rted/pull-request-comment-branch@v2 id: comment-branch - name: (PR comment) Submit PR result as success - if: success() && github.event_name == 'issue_comment' + if: github.event_name == 'issue_comment' && env.FAILED != 'true' uses: myrotvorets/set-commit-status-action@v1.1.7 with: sha: ${{ steps.comment-branch.outputs.head_sha }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2dc4c4939eb37..6447afd489fc6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -435,7 +435,7 @@ jobs: platforms: all - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.1 with: version: latest install: true diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml index 4d770d496486f..28669fb9b13ca 100644 --- a/.github/workflows/regression.yml +++ b/.github/workflows/regression.yml @@ -37,6 +37,9 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.merge_group.head_sha || github.sha }} cancel-in-progress: true +env: + SINGLE_MACHINE_PERFORMANCE_API: ${{ secrets.SINGLE_MACHINE_PERFORMANCE_API }} + jobs: # Only run this workflow if files changed in areas that could possibly introduce a regression @@ -229,7 +232,7 @@ jobs: export REPLICAS="10" export TOTAL_SAMPLES="600" export P_VALUE="0.1" - export SMP_CRATE_VERSION="0.7.3" + export SMP_CRATE_VERSION="0.9.1" export LADING_VERSION="0.12.0" echo "warmup seconds: ${WARMUP_SECONDS}" @@ -295,7 +298,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.1 - name: Build 'vector' target image uses: docker/build-push-action@v4.1.1 @@ -332,7 +335,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2.8.0 + uses: docker/setup-buildx-action@v2.9.1 - name: Build 'vector' target image uses: docker/build-push-action@v4.1.1 @@ -496,6 +499,7 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job submit \ + --use-curta \ --lading-version ${{ needs.compute-metadata.outputs.lading-version }} \ --total-samples ${{ needs.compute-metadata.outputs.total-samples }} \ --warmup-seconds ${{ needs.compute-metadata.outputs.warmup-seconds }} \ @@ -525,6 +529,7 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job status \ + --use-curta \ --wait \ --wait-delay-seconds 60 \ --wait-timeout-minutes 90 \ @@ -537,6 +542,7 @@ jobs: run: | chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job cancel \ + --use-curta \ --submission-metadata ${{ runner.temp }}/submission-metadata - name: Check status, cancelled @@ -617,6 +623,7 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job result \ + --use-curta \ --submission-metadata ${{ runner.temp }}/submission-metadata - name: Check status, cancelled @@ -708,6 +715,7 @@ jobs: chmod +x ${{ runner.temp }}/bin/smp ${{ runner.temp }}/bin/smp --team-id ${{ secrets.SINGLE_MACHINE_PERFORMANCE_TEAM_ID }} job sync \ + --use-curta \ --submission-metadata ${{ runner.temp }}/submission-metadata \ --output-path "${{ runner.temp }}/outputs" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca5184b5041c2..dda403b3b6030 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -127,7 +127,7 @@ jobs: name: Test Suite runs-on: ubuntu-20.04 if: always() - needs: checks + needs: [changes, checks] env: FAILED: ${{ contains(needs.*.result, 'failure') }} steps: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c993c8de4dc1a..e9403d814ef22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -230,7 +230,7 @@ When deprecating functionality in Vector, see [DEPRECATION.md](DEPRECATION.md). When adding, modifying, or removing a dependency in Vector you may find that you need to update the inventory of third-party licenses maintained in `LICENSE-3rdparty.csv`. This file is generated using -[rust-license-tool](https://github.com/DataDog/rust-license-tool.git) and can be updated using +[dd-rust-license-tool](https://github.com/DataDog/rust-license-tool.git) and can be updated using `cargo vdev build licenses`. ## Next steps diff --git a/Cargo.lock b/Cargo.lock index b4643639d00ee..917be847aa2f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -73,15 +73,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - [[package]] name = "aho-corasick" version = "1.0.1" @@ -183,9 +174,9 @@ checksum = "41ed9a86bf92ae6580e0a31281f65a1b1d867c0cc68d5346e2ae128dddfa6a7d" [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" [[package]] name = "anymap" @@ -195,9 +186,9 @@ checksum = "8f1f8f5a6f3d50d89e3797d7593a50f96bb2aaa20ca0cc7be1fb673232c91d72" [[package]] name = "apache-avro" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf4144857f9e4d7dd6cc4ba4c78efd2a46bad682b029bd0d91e76a021af1b2a" +checksum = "9c0fdddc3fdac97394ffcc5c89c634faa9c1c166ced54189af34e407c97b6ee7" dependencies = [ "byteorder", "digest 0.10.7", @@ -213,7 +204,7 @@ dependencies = [ "strum", "strum_macros", "thiserror", - "typed-builder", + "typed-builder 0.14.0", "uuid", "zerocopy", ] @@ -260,7 +251,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6368f9ae5c6ec403ca910327ae0c9437b0a85255b6950c90d497e6177f6e5e" dependencies = [ "proc-macro-hack", - "quote 1.0.29", + "quote 1.0.31", "syn 1.0.109", ] @@ -314,12 +305,12 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d6b683edf8d1119fe420a94f8a7e389239666aa72e65495d91c00462510151" +checksum = "88903cb14723e4d4003335bb7f8a14f27691649105346a0f0957466c096adfe6" dependencies = [ "anstyle 1.0.0", - "bstr 1.5.0", + "bstr 1.6.0", "doc-comment", "predicates", "predicates-core", @@ -366,16 +357,16 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0122885821398cc923ece939e24d1056a2384ee719432397fa9db87230ff11" +checksum = "62b74f44609f0f91493e3082d3734d98497e094777144380ea4db9f9905dd5b6" dependencies = [ "flate2", "futures-core", "memchr", "pin-project-lite", "tokio", - "zstd 0.12.3+zstd.1.5.2", + "zstd 0.12.4", "zstd-safe 6.0.3+zstd.1.5.2", ] @@ -472,8 +463,8 @@ dependencies = [ "async-graphql-parser", "darling 0.14.2", "proc-macro-crate 1.2.1", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "thiserror", ] @@ -592,9 +583,9 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -614,9 +605,9 @@ version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -627,13 +618,13 @@ checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524" [[package]] name = "async-trait" -version = "0.1.68" +version = "0.1.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -1156,9 +1147,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8175979259124331c1d7bf6586ee7e0da434155e4b2d48ec2c8386281d8df39" +checksum = "a6a1de45611fdb535bfde7b7de4fd54f4fd2b17b1737c0a59b69bf9b92074b8c" dependencies = [ "async-trait", "axum-core", @@ -1384,15 +1375,9 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" dependencies = [ - "bit-vec 0.6.3", + "bit-vec", ] -[[package]] -name = "bit-vec" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b4ff8b16e6076c3e14220b39fbc1fabb6737522281a388998046859400895f" - [[package]] name = "bit-vec" version = "0.6.3" @@ -1413,12 +1398,12 @@ checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" [[package]] name = "bitmask-enum" -version = "2.1.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9e32d7420c85055e8107e5b2463c4eeefeaac18b52359fe9f9c08a18f342b2" +checksum = "78d456f91b4c1fdebf2698214e599fec3d7f8b46e3140fb254a9ea88c970ab0a" dependencies = [ - "quote 1.0.29", - "syn 1.0.109", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -1475,12 +1460,12 @@ dependencies = [ ] [[package]] -name = "bloom" -version = "0.3.2" +name = "bloomy" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d00ac8e5056d6d65376a3c1aa5c7c34850d6949ace17f0266953a254eb3d6fe8" +checksum = "489d2af57852b78a86478273ac6a1ef912061b6af3a439694c49f309f6ea3bdd" dependencies = [ - "bit-vec 0.4.4", + "siphasher", ] [[package]] @@ -1538,7 +1523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "822462c1e7b17b31961798a6874b36daea6818e99e0cb7d3b7b0fa3c477751c3" dependencies = [ "borsh-derive", - "hashbrown 0.13.2", + "hashbrown 0.13.1", ] [[package]] @@ -1550,7 +1535,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", "syn 1.0.109", ] @@ -1560,8 +1545,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61820b4c5693eafb998b1e67485423c923db4a75f72585c247bdee32bad81e7b" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1571,8 +1556,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c76cdbfa13def20d1f8af3ae7b3c6771f06352a74221d8851262ac384c122b8e" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1605,18 +1590,17 @@ checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223" dependencies = [ "lazy_static", "memchr", - "regex-automata", + "regex-automata 0.1.10", ] [[package]] name = "bstr" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a246e68bb43f6cd9db24bea052a53e40405417c5fb372e3d1a8a7f770a564ef5" +checksum = "6798148dccfbff0fae41c7574d2fa8f1ef3492fba0face179de5d8d447d67b05" dependencies = [ "memchr", - "once_cell", - "regex-automata", + "regex-automata 0.3.2", "serde", ] @@ -1642,8 +1626,8 @@ version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13e576ebe98e605500b3c8041bb888e966653577172df6dd97398714eb30b9bf" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1710,7 +1694,7 @@ dependencies = [ "cached_proc_macro", "cached_proc_macro_types", "futures 0.3.28", - "hashbrown 0.13.2", + "hashbrown 0.13.1", "instant", "once_cell", "thiserror", @@ -1725,8 +1709,8 @@ checksum = "b48814962d2fd604c50d2b9433c2a41a0ab567779ee2c02f7fba6eca1221f082" dependencies = [ "cached_proc_macro_types", "darling 0.14.2", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -1936,9 +1920,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.3.1" +version = "4.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6b5c519bab3ea61843a7923d074b04245624bb84a64a8c150f5deb014e388b" +checksum = "5fc443334c81a804575546c5a8a79b4913b50e28d69232903604cada1de817ce" dependencies = [ "clap 4.1.14", ] @@ -1949,10 +1933,10 @@ version = "4.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81d7dc0031c3a59a04fc2ba395c8e2dd463cba1859275f065d225f6122221b45" dependencies = [ - "heck 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -2028,13 +2012,13 @@ dependencies = [ [[package]] name = "colored" -version = "2.0.0" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3616f750b84d8f0de8a58bda93e08e2a81ad3f523089b05f1dffecab48c6cbd" +checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" dependencies = [ - "atty", + "is-terminal", "lazy_static", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -2122,9 +2106,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57ab2224a0311582eb03adba4caaf18644f7b1f10a760803a803b9b605187fc7" +checksum = "d4cf42660ac07fcebed809cfe561dd8730bcd35b075215e6479c516bcd0d11cb" dependencies = [ "console-api", "crossbeam-channel", @@ -2405,7 +2389,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.29", + "quote 1.0.31", "syn 1.0.109", ] @@ -2458,8 +2442,8 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "scratch", "syn 1.0.109", ] @@ -2476,8 +2460,8 @@ version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -2501,6 +2485,16 @@ dependencies = [ "darling_macro 0.14.2", ] +[[package]] +name = "darling" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0209d94da627ab5605dcccf08bb18afa5009cfbef48d8a8b7d7bdbc79be25c5e" +dependencies = [ + "darling_core 0.20.3", + "darling_macro 0.20.3", +] + [[package]] name = "darling_core" version = "0.13.4" @@ -2509,8 +2503,8 @@ checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "strsim 0.10.0", "syn 1.0.109", ] @@ -2523,12 +2517,26 @@ checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "strsim 0.10.0", "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "177e3443818124b357d8e76f53be906d60937f0d3a90773a664fa63fa253e621" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2 1.0.66", + "quote 1.0.31", + "strsim 0.10.0", + "syn 2.0.26", +] + [[package]] name = "darling_macro" version = "0.13.4" @@ -2536,7 +2544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ "darling_core 0.13.4", - "quote 1.0.29", + "quote 1.0.31", "syn 1.0.109", ] @@ -2547,18 +2555,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" dependencies = [ "darling_core 0.14.2", - "quote 1.0.29", + "quote 1.0.31", "syn 1.0.109", ] +[[package]] +name = "darling_macro" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" +dependencies = [ + "darling_core 0.20.3", + "quote 1.0.31", + "syn 2.0.26", +] + [[package]] name = "dashmap" -version = "5.4.0" +version = "5.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" +checksum = "6943ae99c34386c84a470c499d3414f66502a41340aa895406e0d2e4a207b91d" dependencies = [ "cfg-if", - "hashbrown 0.12.3", + "hashbrown 0.14.0", "lock_api", "once_cell", "parking_lot_core", @@ -2622,8 +2641,8 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -2633,9 +2652,9 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53e0efad4403bfc52dc201159c4b842a246a14b98c64b55dfd0f2d89729dfeb8" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -2645,8 +2664,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" dependencies = [ "convert_case 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "rustc_version 0.4.0", "syn 1.0.109", ] @@ -2816,9 +2835,9 @@ checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" [[package]] name = "dyn-clone" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b0cf012f1230e43cd00ebb729c6bb58707ecfa8ad08b52ef3a4ccd2697fc30" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" [[package]] name = "ed25519" @@ -2900,9 +2919,9 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "21cdad81446a7f7dc43f6a77409efeb9733d2fa65553efef6018ef257c959b73" dependencies = [ - "heck 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -2912,22 +2931,22 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" dependencies = [ - "heck 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] [[package]] name = "enum_dispatch" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11f36e95862220b211a6e2aa5eca09b4fa391b13cd52ceb8035a24bf65a79de2" +checksum = "8f33313078bb8d4d05a2733a94ac4c2d8a0df9a2b84424ebf4f33bfc224a890e" dependencies = [ "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 1.0.109", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -2945,9 +2964,9 @@ version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e9a1f9f7d83e59740248a6e14ecf93929ade55027844dfcea78beafccc15745" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -2987,9 +3006,9 @@ checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" [[package]] name = "erased-serde" -version = "0.3.23" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54558e0ba96fbe24280072642eceb9d7d442e32c7ec0ea9e7ecd7b4ea2cf4e11" +checksum = "f94c0e13118e7d7533271f754a168ae8400e6a1cc043f2bfd53cc7290f1a1de3" dependencies = [ "serde", ] @@ -3073,8 +3092,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f47da3a72ec598d9c8937a7ebca8962a5c7a1f28444e38c2b33c771ba3f55f05" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3117,7 +3136,7 @@ dependencies = [ name = "file-source" version = "0.1.0" dependencies = [ - "bstr 1.5.0", + "bstr 1.6.0", "bytes 1.4.0", "chrono", "crc", @@ -3169,8 +3188,8 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4c81935e123ab0741c4c4f0d9b8377e5fb21d3de7e062fa4b1263b1fbcba1ea" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -3350,9 +3369,9 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -3427,17 +3446,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "ghost" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb19fe8de3ea0920d282f7b77dd4227aea6b8b999b42cdf0ca41b2472b14443a" -dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 1.0.109", -] - [[package]] name = "gimli" version = "0.27.3" @@ -3484,9 +3492,9 @@ dependencies = [ [[package]] name = "governor" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5" +checksum = "821239e5672ff23e2a7060901fa622950bbd80b649cdaadd78d1c1767ed14eb4" dependencies = [ "cfg-if", "dashmap", @@ -3537,10 +3545,10 @@ checksum = "a40f793251171991c4eb75bd84bc640afa8b68ff6907bc89d3b712a22f700506" dependencies = [ "graphql-introspection-query", "graphql-parser", - "heck 0.4.0", + "heck 0.4.1", "lazy_static", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde", "serde_json", "syn 1.0.109", @@ -3553,10 +3561,43 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00bda454f3d313f909298f626115092d348bc231025699f557b27e248475f48c" dependencies = [ "graphql_client_codegen", - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", "syn 1.0.109", ] +[[package]] +name = "greptime-proto" +version = "0.1.0" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?tag=0.2.1#4398d20c56d5f7939cc2960789cb1fa7dd18e6fe" +dependencies = [ + "prost", + "serde", + "serde_json", + "tonic", + "tonic-build", +] + +[[package]] +name = "greptimedb-client" +version = "0.1.0" +source = "git+https://github.com/GreptimeTeam/greptimedb-client-rust.git?rev=bc32362adf0df17a41a95bae4221d6d8f1775656#bc32362adf0df17a41a95bae4221d6d8f1775656" +dependencies = [ + "dashmap", + "enum_dispatch", + "futures 0.3.28", + "futures-util", + "greptime-proto", + "parking_lot", + "prost", + "rand 0.8.5", + "snafu", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tower", +] + [[package]] name = "grok" version = "2.0.0" @@ -3609,9 +3650,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.13.2" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" +checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ "ahash 0.8.2", ] @@ -3675,9 +3716,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "heim" @@ -4125,9 +4166,9 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f2cb48b81b1dc9f39676bf99f5499babfec7cd8fe14307f7b3d747208fb5690" +checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" [[package]] name = "infer" @@ -4137,9 +4178,9 @@ checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" [[package]] name = "infer" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb78f4c4a058ef30a9ff77322e758f7e60f871274b602d7fdc1b0956b0cb88e" +checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" [[package]] name = "inotify" @@ -4182,12 +4223,9 @@ dependencies = [ [[package]] name = "inventory" -version = "0.3.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0539b5de9241582ce6bd6b0ba7399313560151e58c9aaf8b74b711b1bdce644" -dependencies = [ - "ghost", -] +checksum = "25b1d6b4b9fb75fc419bdef998b689df5080a32931cb3395b86202046b56a9ea" [[package]] name = "io-lifetimes" @@ -4488,7 +4526,7 @@ dependencies = [ "secrecy", "serde", "serde_json", - "serde_yaml 0.9.22", + "serde_yaml 0.9.25", "thiserror", "tokio", "tokio-util", @@ -4554,7 +4592,7 @@ dependencies = [ "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.7.2", + "regex-syntax 0.7.3", "string_cache", "term", "tiny-keccak", @@ -4569,9 +4607,9 @@ checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" [[package]] name = "lapin" -version = "2.2.1" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc13beaa09eed710f406201f46b961345b4d061dd90ec3d3ccc70721e70342a" +checksum = "5f3067a1fcfbc3fc46455809c023e69b8f6602463201010f4ae5a3b572adb9dc" dependencies = [ "amq-protocol", "async-global-executor-trait", @@ -4688,9 +4726,9 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" dependencies = [ "autocfg", "scopeguard", @@ -4728,9 +4766,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718e8fae447df0c7e1ba7f5189829e63fd536945c8988d61444c19039f16b670" +checksum = "eedb2bdbad7e0634f83989bf596f497b070130daaa398ab22d84c39e266deec5" [[package]] name = "lru-cache" @@ -4824,7 +4862,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" dependencies = [ - "regex-automata", + "regex-automata 0.1.10", ] [[package]] @@ -4910,9 +4948,9 @@ dependencies = [ [[package]] name = "metrics" -version = "0.21.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa8ebbd1a9e57bbab77b9facae7f5136aea44c356943bf9a198f647da64285d6" +checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ "ahash 0.8.2", "metrics-macros", @@ -4925,9 +4963,9 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddece26afd34c31585c74a4db0630c376df271c285d682d1e55012197830b6df" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -4948,14 +4986,14 @@ dependencies = [ [[package]] name = "metrics-util" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "111cb375987443c3de8d503580b536f77dc8416d32db62d9456db5d93bd7ac47" +checksum = "4de2ed6e491ed114b40b732e4d1659a9d53992ebd87490c44a6ffe23739d973e" dependencies = [ - "aho-corasick 0.7.20", + "aho-corasick", "crossbeam-epoch", "crossbeam-utils", - "hashbrown 0.13.2", + "hashbrown 0.13.1", "indexmap 1.9.3", "metrics", "num_cpus", @@ -5081,7 +5119,7 @@ dependencies = [ "tokio-util", "trust-dns-proto 0.21.2", "trust-dns-resolver", - "typed-builder", + "typed-builder 0.10.0", "uuid", "webpki-roots", ] @@ -5451,8 +5489,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate 1.2.1", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -5463,9 +5501,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" dependencies = [ "proc-macro-crate 1.2.1", - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -5572,9 +5610,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "opendal" -version = "0.38.0" +version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "717f47be1760a6a651f81eeba8239444a077d0d229409a016298d2b2483c408c" +checksum = "4df645b6012162c04c8949e9b96ae2ef002e79189cfb154e507e51ac5be76a09" dependencies = [ "anyhow", "async-compat", @@ -5646,8 +5684,8 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -5797,15 +5835,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.4" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dc9e0dc2adc1c69d09143aff38d3d30c5c3f0df0dad82e6d25547af174ebec0" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.3.5", "smallvec", - "windows-sys 0.42.0", + "windows-targets 0.48.0", ] [[package]] @@ -5819,9 +5857,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.12" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" [[package]] name = "pbkdf2" @@ -5890,8 +5928,8 @@ checksum = "75a1ef20bf3193c15ac345acb32e26b3dc3223aff4d77ae4fc5359567683796b" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -5965,22 +6003,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e138fdd8263907a2b0e1b4e80b7e58c721126479b6e6eedfb1b402acea7b9bd" +checksum = "030ad2bc4db10a8944cb0d837f158bdfec4d4a4873ab701a95046770d11f8842" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1fef411b303e3e12d534fb6e7852de82da56edd937d895125821fb7c09436c7" +checksum = "ec2e072ecce94ec471b13398d5402c188e76ac03cf74dd1a975161b23a3f6d9c" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -6199,7 +6237,7 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" dependencies = [ - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", "syn 1.0.109", ] @@ -6244,8 +6282,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "version_check", ] @@ -6256,8 +6294,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "version_check", ] @@ -6284,9 +6322,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" dependencies = [ "unicode-ident", ] @@ -6342,7 +6380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "119533552c9a7ffacc21e099c24a0ac8bb19c2a2a3f363de84cd9b844feab270" dependencies = [ "bytes 1.4.0", - "heck 0.4.0", + "heck 0.4.1", "itertools 0.10.5", "lazy_static", "log", @@ -6365,8 +6403,8 @@ checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" dependencies = [ "anyhow", "itertools 0.10.5", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6405,8 +6443,8 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6417,7 +6455,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6eb95b2e36b92d3e0536be87eaf7accb17db39f5a44452759b43f1328e82dc9" dependencies = [ "async-trait", - "bit-vec 0.6.3", + "bit-vec", "bytes 1.4.0", "chrono", "crc", @@ -6504,8 +6542,8 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b22a693222d716a9587786f37ac3f6b4faedb5b80c23914e7303ff5a1d8016e9" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -6520,11 +6558,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.29" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "5fe8a65d69dd0808184ebb5f836ab526bb259db23c657efa38711b1072ee47f0" dependencies = [ - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", ] [[package]] @@ -6688,9 +6726,9 @@ dependencies = [ [[package]] name = "rdkafka" -version = "0.32.2" +version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8733bc5dc0b192d1a4b28073f9bff1326ad9e4fecd4d9b025d6fc358d1c3e79" +checksum = "da18026aad1c24033da3da726200de7e911e75c2e2cc2f77ffb9b4502720faae" dependencies = [ "futures-channel", "futures-util", @@ -6785,13 +6823,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.8.4" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0ab3ca65655bb1e41f2a8c8cd662eb4fb035e67c3f78da1d61dffe89d07300f" +checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" dependencies = [ - "aho-corasick 1.0.1", + "aho-corasick", "memchr", - "regex-syntax 0.7.2", + "regex-automata 0.3.2", + "regex-syntax 0.7.3", ] [[package]] @@ -6803,6 +6842,17 @@ dependencies = [ "regex-syntax 0.6.29", ] +[[package]] +name = "regex-automata" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d3daa6976cffb758ec878f108ba0e062a45b2d6ca3a2cca965338855476caf" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.7.3", +] + [[package]] name = "regex-syntax" version = "0.6.29" @@ -6811,9 +6861,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436b050e76ed2903236f032a59761c1eb99e1b0aead2c257922771dab1fc8c78" +checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846" [[package]] name = "rend" @@ -6919,8 +6969,8 @@ version = "0.7.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -7038,7 +7088,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.17", + "semver 1.0.18", ] [[package]] @@ -7208,9 +7258,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.13" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" [[package]] name = "same-file" @@ -7245,11 +7295,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys 0.48.0", ] [[package]] @@ -7340,9 +7390,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" dependencies = [ "serde", ] @@ -7355,9 +7405,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.164" +version = "1.0.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e8c8cf938e98f769bc164923b06dce91cea1751522f46f8466461af04c9027d" +checksum = "e91f70896d6720bc714a4a57d22fc91f1db634680e65c8efe13323f1fa38d53f" dependencies = [ "serde_derive", ] @@ -7368,7 +7418,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78072b550e5c20bc4a9d1384be28809cbdb7b25b2b4707ddc6d908b7e6de3bf" dependencies = [ - "toml 0.7.5", + "toml 0.7.6", ] [[package]] @@ -7406,40 +7456,40 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" +checksum = "ab33ec92f677585af6d88c65593ae2375adde54efdbf16d597f2cbc7a6d368ff" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.164" +version = "1.0.173" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9735b638ccc51c28bf6914d90a2e9725b377144fc612c49a611fddd1b631d68" +checksum = "a6250dde8342e0232232be9ca3db7aa40aceb5a3e5dd9bddbc00d99a007cde49" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] name = "serde_derive_internals" -version = "0.26.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +checksum = "e578a843d40b4189a4d66bba51d7684f57da5bd7c304c64e14bd63efbef49509" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 1.0.109", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] name = "serde_json" -version = "1.0.99" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46266871c240a00b8f503b877622fe33430b3c7d963bdc0f2adc511e54a1eae3" +checksum = "d03b412469450d4404fe8499a268edd7f8b79fecb074b0d812ad64ca21f4031b" dependencies = [ "indexmap 2.0.0", "itoa", @@ -7482,8 +7532,8 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fe39d9fbb0ebf5eb2c7cb7e2a47e4f462fad1379f1166b8ae49ad9eae89a7ca" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -7530,7 +7580,22 @@ dependencies = [ "indexmap 1.9.3", "serde", "serde_json", - "serde_with_macros 2.3.2", + "time", +] + +[[package]] +name = "serde_with" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21e47d95bc83ed33b2ecf84f4187ad1ab9685d18ff28db000c99deac8ce180e3" +dependencies = [ + "base64 0.21.2", + "chrono", + "hex", + "indexmap 1.9.3", + "serde", + "serde_json", + "serde_with_macros 3.1.0", "time", ] @@ -7541,21 +7606,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ "darling 0.13.4", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] [[package]] name = "serde_with_macros" -version = "2.3.2" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "859011bddcc11f289f07f467cc1fe01c7a941daa4d8f6c40d4d1c92eb6d9319c" +checksum = "ea3cee93715c2e266b9338b7544da68a9f24e227722ba482bd1c024367c77c65" dependencies = [ - "darling 0.14.2", - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 1.0.109", + "darling 0.20.3", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -7572,9 +7637,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.22" +version = "0.9.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" +checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" dependencies = [ "indexmap 2.0.0", "itoa", @@ -7764,9 +7829,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" dependencies = [ "serde", ] @@ -7807,9 +7872,9 @@ dependencies = [ [[package]] name = "snafu" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0656e7e3ffb70f6c39b3c2a86332bb74aa3c679da781642590f3c1118c5045" +checksum = "e4de37ad025c587a29e8f3f5605c00f70b98715ef90b9061a815b9e59e9042d6" dependencies = [ "doc-comment", "futures-core", @@ -7819,13 +7884,13 @@ dependencies = [ [[package]] name = "snafu-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "475b3bbe5245c26f2d8a6f62d67c1f30eb9fffeccee721c45d162c3ebbdf81b2" +checksum = "990079665f075b699031e9c08fd3ab99be5029b96f3b78dc0709e8f77e4efebf" dependencies = [ - "heck 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -7965,28 +8030,28 @@ checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ "heck 0.3.3", "proc-macro-error", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] [[package]] name = "strum" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "6069ca09d878a33f883cc06aaa9718ede171841d3832450354410b718b097232" dependencies = [ - "heck 0.4.0", - "proc-macro2 1.0.63", - "quote 1.0.29", + "heck 0.4.1", + "proc-macro2 1.0.66", + "quote 1.0.31", "rustversion", - "syn 1.0.109", + "syn 2.0.26", ] [[package]] @@ -8022,19 +8087,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "unicode-ident", ] [[package]] name = "syn" -version = "2.0.10" +version = "2.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40" +checksum = "45c3457aacde3c65315de5031ec191ce46604304d2446e803d71ade03308d970" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "unicode-ident", ] @@ -8050,8 +8115,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -8191,22 +8256,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" +checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.40" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" +checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -8304,9 +8369,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.29.0" +version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374442f06ee49c3a28a8fc9f01a2596fed7559c6b99b31279c3261778e77d84f" +checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" dependencies = [ "autocfg", "backtrace", @@ -8350,9 +8415,9 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -8498,9 +8563,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebafdf5ad1220cb59e7d17cf4d2c72015297b75b19a10472f99b89225089240" +checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542" dependencies = [ "serde", "serde_spanned", @@ -8519,9 +8584,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.11" +version = "0.19.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266f016b7f039eec8a1a80dfe6156b633d208b9fccca5e4db1d6775b0c4e34a7" +checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" dependencies = [ "indexmap 2.0.0", "serde", @@ -8570,9 +8635,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6fdaae4c2c638bb70fe42803a26fbd6fc6ac8c72f5c59f67ecc2a2dcabf4b07" dependencies = [ "prettyplease", - "proc-macro2 1.0.63", + "proc-macro2 1.0.66", "prost-build", - "quote 1.0.29", + "quote 1.0.31", "syn 1.0.109", ] @@ -8675,8 +8740,8 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -8946,8 +9011,19 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89851716b67b937e393b3daa8423e67ddfc4bbbf1654bcf05488e95e0828db0c" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 1.0.109", +] + +[[package]] +name = "typed-builder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cba322cb9b7bc6ca048de49e83918223f35e7a86311267013afff257004870" +dependencies = [ + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -8959,9 +9035,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" [[package]] name = "typetag" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6898cc6f6a32698cc3e14d5632a14d2b23ed9f7b11e6b8e05ce685990acc22" +checksum = "7a66aafcfb982bf1f9a28755ac6bcbdcd4631ff516cb038fa61299201ebb4364" dependencies = [ "erased-serde", "inventory", @@ -8972,13 +9048,13 @@ dependencies = [ [[package]] name = "typetag-impl" -version = "0.2.8" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c3e1c30cedd24fc597f7d37a721efdbdc2b1acae012c1ef1218f4c7c2c0f3e7" +checksum = "d836cd032f71d90cbaa3c1f85ce84266af23659766d8c0b1c4c6524a0fb4c36f" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", ] [[package]] @@ -9131,9 +9207,9 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" [[package]] name = "uuid" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d023da39d1fde5a8a3fe1f3e01ca9632ada0a63e9797de55a879d6e2236277be" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" dependencies = [ "getrandom 0.2.10", "rand 0.8.5", @@ -9181,10 +9257,10 @@ dependencies = [ "reqwest", "serde", "serde_json", - "serde_yaml 0.9.22", + "serde_yaml 0.9.25", "sha2 0.10.7", "tempfile", - "toml 0.7.5", + "toml 0.7.6", ] [[package]] @@ -9195,14 +9271,14 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "vector" -version = "0.31.0" +version = "0.32.0" dependencies = [ "apache-avro", "approx", "arc-swap", "arr_macro", "assert_cmd", - "async-compression 0.4.0", + "async-compression 0.4.1", "async-graphql", "async-graphql-warp", "async-stream", @@ -9230,7 +9306,7 @@ dependencies = [ "azure_storage", "azure_storage_blobs", "base64 0.21.2", - "bloom", + "bloomy", "bollard", "bytes 1.4.0", "bytesize", @@ -9259,6 +9335,7 @@ dependencies = [ "glob", "goauth", "governor", + "greptimedb-client", "grok", "h2", "hash_hasher", @@ -9274,7 +9351,7 @@ dependencies = [ "hyper-proxy", "indexmap 2.0.0", "indoc", - "infer 0.14.0", + "infer 0.15.0", "inventory", "itertools 0.11.0", "k8s-openapi 0.18.0", @@ -9328,13 +9405,13 @@ dependencies = [ "rmpv", "roaring", "seahash", - "semver 1.0.17", + "semver 1.0.18", "serde", "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 2.3.2", - "serde_yaml 0.9.22", + "serde_with 3.1.0", + "serde_yaml 0.9.25", "sha2 0.10.7", "similar-asserts", "smallvec", @@ -9355,7 +9432,7 @@ dependencies = [ "tokio-test", "tokio-tungstenite 0.19.0", "tokio-util", - "toml 0.7.5", + "toml 0.7.6", "tonic", "tonic-build", "tower", @@ -9385,7 +9462,7 @@ dependencies = [ "warp", "windows-service", "wiremock", - "zstd 0.12.3+zstd.1.5.2", + "zstd 0.12.4", ] [[package]] @@ -9438,7 +9515,7 @@ dependencies = [ "rand 0.8.5", "rkyv", "serde", - "serde_yaml 0.9.22", + "serde_yaml 0.9.25", "snafu", "temp-dir", "tokio", @@ -9501,9 +9578,9 @@ dependencies = [ "once_cell", "serde", "serde_json", - "serde_with 2.3.2", + "serde_with 3.1.0", "snafu", - "toml 0.7.5", + "toml 0.7.6", "tracing 0.1.37", "url", "vector-config-common", @@ -9516,13 +9593,13 @@ name = "vector-config-common" version = "0.1.0" dependencies = [ "convert_case 0.6.0", - "darling 0.13.4", + "darling 0.20.3", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde", "serde_json", - "syn 1.0.109", + "syn 2.0.26", "tracing 0.1.37", ] @@ -9530,12 +9607,12 @@ dependencies = [ name = "vector-config-macros" version = "0.1.0" dependencies = [ - "darling 0.13.4", - "proc-macro2 1.0.63", - "quote 1.0.29", + "darling 0.20.3", + "proc-macro2 1.0.66", + "quote 1.0.31", "serde", "serde_derive_internals", - "syn 1.0.109", + "syn 2.0.26", "vector-config", "vector-config-common", ] @@ -9593,7 +9670,7 @@ dependencies = [ "security-framework", "serde", "serde_json", - "serde_with 2.3.2", + "serde_with 3.1.0", "similar-asserts", "smallvec", "snafu", @@ -9603,7 +9680,7 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-util", - "toml 0.7.5", + "toml 0.7.6", "tonic", "tower", "tracing 0.1.37", @@ -9771,7 +9848,7 @@ dependencies = [ "uuid", "webbrowser", "woothee", - "zstd 0.12.3+zstd.1.5.2", + "zstd 0.12.4", ] [[package]] @@ -9791,8 +9868,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d257817081c7dffcdbab24b9e62d2def62e2ff7d00b1c20062551e6cccc145ff" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", ] [[package]] @@ -9892,9 +9969,9 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", "wasm-bindgen-shared", ] @@ -9916,7 +9993,7 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "quote 1.0.29", + "quote 1.0.31", "wasm-bindgen-macro-support", ] @@ -9926,9 +10003,9 @@ version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", - "syn 2.0.10", + "proc-macro2 1.0.66", + "quote 1.0.31", + "syn 2.0.26", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -10337,8 +10414,8 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6505e6815af7de1746a08f69c69606bb45695a17149517680f3b2149713b19a3" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", ] @@ -10357,8 +10434,8 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f8f187641dad4f680d25c4bfc4225b418165984179f26ca76ec4fb6441d3a17" dependencies = [ - "proc-macro2 1.0.63", - "quote 1.0.29", + "proc-macro2 1.0.66", + "quote 1.0.31", "syn 1.0.109", "synstructure", ] @@ -10374,9 +10451,9 @@ dependencies = [ [[package]] name = "zstd" -version = "0.12.3+zstd.1.5.2" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" dependencies = [ "zstd-safe 6.0.3+zstd.1.5.2", ] diff --git a/Cargo.toml b/Cargo.toml index dd4227faab120..da1973d061d62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vector" -version = "0.31.0" +version = "0.32.0" authors = ["Vector Contributors "] edition = "2021" description = "A lightweight and ultra-fast tool for building observability pipelines" @@ -143,13 +143,13 @@ loki-logproto = { path = "lib/loki-logproto", optional = true } # Tokio / Futures async-stream = { version = "0.3.5", default-features = false } -async-trait = { version = "0.1.68", default-features = false } +async-trait = { version = "0.1.71", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["compat", "io-compat"], package = "futures" } -tokio = { version = "1.29.0", default-features = false, features = ["full"] } +tokio = { version = "1.29.1", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1.14", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } -console-subscriber = { version = "0.1.9", default-features = false, optional = true } +console-subscriber = { version = "0.1.10", default-features = false, optional = true } # Tracing tracing = { version = "0.1.34", default-features = false } @@ -159,7 +159,7 @@ tracing-subscriber = { version = "0.3.17", default-features = false, features = tracing-tower = { git = "https://github.com/tokio-rs/tracing", default-features = false, rev = "e0642d949891546a3bb7e47080365ee7274f05cd" } # Metrics -metrics = "0.21.0" +metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } # AWS - Official SDK @@ -194,12 +194,12 @@ opendal = {version = "0.38", default-features = false, features = ["native-tls", tower = { version = "0.4.13", default-features = false, features = ["buffer", "limit", "retry", "timeout", "util", "balance", "discover"] } tower-http = { version = "0.4.1", default-features = false, features = ["decompression-gzip"]} # Serde -serde = { version = "1.0.164", default-features = false, features = ["derive"] } +serde = { version = "1.0.173", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } -serde_bytes = { version = "0.11.9", default-features = false, features = ["std"], optional = true } -serde_json = { version = "1.0.99", default-features = false, features = ["raw_value"] } -serde_with = { version = "2.3.2", default-features = false, features = ["macros", "std"] } -serde_yaml = { version = "0.9.22", default-features = false } +serde_bytes = { version = "0.11.12", default-features = false, features = ["std"], optional = true } +serde_json = { version = "1.0.103", default-features = false, features = ["raw_value"] } +serde_with = { version = "3.1.0", default-features = false, features = ["macros", "std"] } +serde_yaml = { version = "0.9.25", default-features = false } # Messagepack rmp-serde = { version = "1.1.1", default-features = false, optional = true } @@ -215,7 +215,7 @@ goauth = { version = "0.13.1", optional = true } smpl_jwt = { version = "0.7.1", default-features = false, optional = true } # AMQP -lapin = { version = "2.2.1", default-features = false, features = ["native-tls"], optional = true } +lapin = { version = "2.3.1", default-features = false, features = ["native-tls"], optional = true } # API async-graphql = { version = "5.0.10", default-features = false, optional = true, features = ["chrono"] } @@ -233,31 +233,34 @@ tui = { version = "0.19.0", optional = true, default-features = false, features hex = { version = "0.4.3", default-features = false, optional = true } sha2 = { version = "0.10.7", default-features = false, optional = true } +# GreptimeDB +greptimedb-client = { git = "https://github.com/GreptimeTeam/greptimedb-client-rust.git", rev = "bc32362adf0df17a41a95bae4221d6d8f1775656", optional = true } + # External libs arc-swap = { version = "1.6", default-features = false, optional = true } -async-compression = { version = "0.4.0", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } -apache-avro = { version = "0.14.0", default-features = false, optional = true } -axum = { version = "0.6.18", default-features = false } +async-compression = { version = "0.4.1", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true } +apache-avro = { version = "0.15.0", default-features = false, optional = true } +axum = { version = "0.6.19", default-features = false } base64 = { version = "0.21.2", default-features = false, optional = true } -bloom = { version = "0.3.2", default-features = false, optional = true } +bloomy = { version = "1.2.0", default-features = false, optional = true } bollard = { version = "0.14.0", default-features = false, features = ["ssl", "chrono"], optional = true } bytes = { version = "1.4.0", default-features = false, features = ["serde"] } bytesize = { version = "1.2.0", default-features = false } chrono = { version = "0.4.26", default-features = false, features = ["serde"] } cidr-utils = { version = "0.5.10", default-features = false } clap = { version = "4.1.14", default-features = false, features = ["derive", "error-context", "env", "help", "std", "string", "usage", "wrap_help"] } -colored = { version = "2.0.0", default-features = false } +colored = { version = "2.0.4", default-features = false } csv = { version = "1.2", default-features = false } derivative = { version = "2.2.0", default-features = false } dirs-next = { version = "2.0.0", default-features = false, optional = true } -dyn-clone = { version = "1.0.11", default-features = false } +dyn-clone = { version = "1.0.12", default-features = false } encoding_rs = { version = "0.8.32", default-features = false, features = ["serde"] } -enum_dispatch = { version = "0.3.11", default-features = false } +enum_dispatch = { version = "0.3.12", default-features = false } exitcode = { version = "1.1.2", default-features = false } flate2 = { version = "1.0.26", default-features = false, features = ["default"] } futures-util = { version = "0.3.28", default-features = false } glob = { version = "0.3.1", default-features = false } -governor = { version = "0.5.1", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } +governor = { version = "0.6.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } grok = { version = "2.0.0", default-features = false, optional = true } h2 = { version = "0.3.20", default-features = false, optional = true } hash_hasher = { version = "2.0.0", default-features = false } @@ -270,14 +273,14 @@ hyper = { version = "0.14.27", default-features = false, features = ["client", " hyper-openssl = { version = "0.9.2", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap = { version = "~2.0.0", default-features = false, features = ["serde", "std"] } -infer = { version = "0.14.0", default-features = false, optional = true} -indoc = { version = "2.0.1", default-features = false } -inventory = { version = "0.3.6", default-features = false } +infer = { version = "0.15.0", default-features = false, optional = true} +indoc = { version = "2.0.3", default-features = false } +inventory = { version = "0.3.9", default-features = false } k8s-openapi = { version = "0.18.0", default-features = false, features = ["api", "v1_26"], optional = true } kube = { version = "0.82.0", default-features = false, features = ["client", "openssl-tls", "runtime"], optional = true } listenfd = { version = "1.0.1", default-features = false, optional = true } logfmt = { version = "0.0.2", default-features = false, optional = true } -lru = { version = "0.10.1", default-features = false, optional = true } +lru = { version = "0.11.0", default-features = false, optional = true } maxminddb = { version = "0.23.0", default-features = false, optional = true } md-5 = { version = "0.10", default-features = false, optional = true } mongodb = { version = "2.6.0", default-features = false, features = ["tokio-runtime"], optional = true } @@ -289,21 +292,21 @@ once_cell = { version = "1.18", default-features = false } openssl = { version = "0.10.55", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } ordered-float = { version = "3.7.0", default-features = false } -paste = "1.0.12" +paste = "1.0.14" percent-encoding = { version = "2.3.0", default-features = false } -pin-project = { version = "1.1.1", default-features = false } +pin-project = { version = "1.1.2", default-features = false } postgres-openssl = { version = "0.5.0", default-features = false, features = ["runtime"], optional = true } pulsar = { version = "6.0.1", default-features = false, features = ["tokio-runtime", "auth-oauth2", "flate2", "lz4", "snap", "zstd"], optional = true } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } rand_distr = { version = "0.4.3", default-features = false } -rdkafka = { version = "0.32.2", default-features = false, features = ["tokio", "libz", "ssl", "zstd"], optional = true } +rdkafka = { version = "0.33.2", default-features = false, features = ["tokio", "libz", "ssl", "zstd"], optional = true } redis = { version = "0.23.0", default-features = false, features = ["connection-manager", "tokio-comp", "tokio-native-tls-comp"], optional = true } -regex = { version = "1.8.4", default-features = false, features = ["std", "perf"] } +regex = { version = "1.9.1", default-features = false, features = ["std", "perf"] } roaring = { version = "0.10.1", default-features = false, optional = true } seahash = { version = "4.1.0", default-features = false } -semver = { version = "1.0.17", default-features = false, features = ["serde", "std"], optional = true } +semver = { version = "1.0.18", default-features = false, features = ["serde", "std"], optional = true } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } -snafu = { version = "0.7.4", default-features = false, features = ["futures"] } +snafu = { version = "0.7.5", default-features = false, features = ["futures"] } snap = { version = "1.1.0", default-features = false, optional = true } socket2 = { version = "0.5.3", default-features = false } stream-cancel = { version = "0.8.1", default-features = false } @@ -312,14 +315,14 @@ syslog = { version = "6.1.0", default-features = false, optional = true } tikv-jemallocator = { version = "0.5.0", default-features = false, optional = true } tokio-postgres = { version = "0.7.7", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = {version = "0.19.0", default-features = false, features = ["connect"], optional = true} -toml = { version = "0.7.5", default-features = false, features = ["parse", "display"] } +toml = { version = "0.7.6", default-features = false, features = ["parse", "display"] } tonic = { version = "0.9", optional = true, default-features = false, features = ["transport", "codegen", "prost", "tls", "tls-roots", "gzip"] } trust-dns-proto = { version = "0.22.0", default-features = false, features = ["dnssec"], optional = true } -typetag = { version = "0.2.8", default-features = false } +typetag = { version = "0.2.10", default-features = false } url = { version = "2.4.0", default-features = false, features = ["serde"] } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } warp = { version = "0.3.5", default-features = false } -zstd = { version = "0.12.3", default-features = false } +zstd = { version = "0.12.4", default-features = false } arr_macro = { version = "0.2.1" } # depending on fork for bumped nix dependency @@ -343,7 +346,7 @@ openssl-src = { version = "111", default-features = false, features = ["force-en [dev-dependencies] approx = "0.5.1" -assert_cmd = { version = "2.0.11", default-features = false } +assert_cmd = { version = "2.0.12", default-features = false } azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest", "azurite_workaround"] } azure_identity = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["enable_reqwest"] } azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "b4544d4920fa3064eb921340054cd9cc130b7664", default-features = false, features = ["azurite_workaround"] } @@ -359,11 +362,11 @@ reqwest = { version = "0.11", features = ["json"] } tempfile = "3.6.0" test-generator = "0.3.1" tokio-test = "0.4.2" -tokio = { version = "1.29.0", features = ["test-util"] } +tokio = { version = "1.29.1", features = ["test-util"] } tower-test = "0.4.0" vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } wiremock = "0.5.19" -zstd = { version = "0.12.3", default-features = false } +zstd = { version = "0.12.4", default-features = false } [patch.crates-io] # Removes dependency on `time` v0.1 @@ -593,7 +596,7 @@ transforms-reduce = [] transforms-remap = [] transforms-route = [] transforms-sample = [] -transforms-tag_cardinality_limit = ["dep:bloom", "dep:hashbrown"] +transforms-tag_cardinality_limit = ["dep:bloomy", "dep:hashbrown"] transforms-throttle = ["dep:governor"] # Sinks @@ -646,6 +649,7 @@ sinks-metrics = [ "sinks-blackhole", "sinks-console", "sinks-datadog_metrics", + "sinks-greptimedb", "sinks-humio", "sinks-influxdb", "sinks-kafka", @@ -679,6 +683,7 @@ sinks-datadog_traces = ["protobuf-build", "dep:rmpv", "dep:rmp-serde", "dep:serd sinks-elasticsearch = ["aws-core", "transforms-metric_to_log"] sinks-file = ["dep:async-compression"] sinks-gcp = ["dep:base64", "gcp"] +sinks-greptimedb = ["dep:greptimedb-client"] sinks-honeycomb = [] sinks-http = [] sinks-humio = ["sinks-splunk_hec", "transforms-metric_to_log"] @@ -739,6 +744,7 @@ all-integration-tests = [ "gcp-cloud-storage-integration-tests", "gcp-integration-tests", "gcp-pubsub-integration-tests", + "greptimedb-integration-tests", "http-client-integration-tests", "humio-integration-tests", "influxdb-integration-tests", @@ -800,6 +806,7 @@ fluent-integration-tests = ["docker", "sources-fluent"] gcp-cloud-storage-integration-tests = ["sinks-gcp"] gcp-integration-tests = ["sinks-gcp"] gcp-pubsub-integration-tests = ["sinks-gcp", "sources-gcp_pubsub"] +greptimedb-integration-tests = ["sinks-greptimedb"] humio-integration-tests = ["sinks-humio"] http-client-integration-tests = ["sources-http_client"] influxdb-integration-tests = ["sinks-influxdb"] diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index fd759a75ab82a..d6634f953937a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -87,7 +87,7 @@ bitvec,https://github.com/bitvecto-rs/bitvec,MIT,The bitvec Authors block-buffer,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers block-padding,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers blocking,https://github.com/smol-rs/blocking,Apache-2.0 OR MIT,Stjepan Glavina -bloom,https://github.com/nicklan/bloom-rs,GPL-2.0,Nick Lanham +bloomy,https://docs.rs/bloomy/,MIT,"Aleksandr Bezobchuk , Alexis Sellier " bollard,https://github.com/fussybeaver/bollard,Apache-2.0,Bollard contributors borsh,https://github.com/near/borsh-rs,MIT OR Apache-2.0,Near Inc borsh-derive,https://github.com/nearprotocol/borsh,Apache-2.0,Near Inc @@ -215,7 +215,6 @@ futures-timer,https://github.com/async-rs/futures-timer,MIT OR Apache-2.0,Alex C futures-util,https://github.com/rust-lang/futures-rs,MIT OR Apache-2.0,The futures-util Authors generic-array,https://github.com/fizyk20/generic-array,MIT,"Bartłomiej Kamiński , Aaron Trent " getrandom,https://github.com/rust-random/getrandom,MIT OR Apache-2.0,The Rand Project Developers -ghost,https://github.com/dtolnay/ghost,MIT OR Apache-2.0,David Tolnay gimli,https://github.com/gimli-rs/gimli,MIT OR Apache-2.0,The gimli Authors glob,https://github.com/rust-lang/glob,MIT OR Apache-2.0,The Rust Project Developers goauth,https://github.com/durch/rust-goauth,MIT,Drazen Urch @@ -225,6 +224,8 @@ graphql-parser,https://github.com/graphql-rust/graphql-parser,MIT OR Apache-2.0, graphql_client,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé graphql_client_codegen,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé graphql_query_derive,https://github.com/graphql-rust/graphql-client,Apache-2.0 OR MIT,Tom Houlé +greptime-proto,https://github.com/GreptimeTeam/greptime-proto,Apache-2.0,The greptime-proto Authors +greptimedb-client,https://github.com/GreptimeTeam/greptimedb-client-rust,Apache-2.0,The greptimedb-client Authors grok,https://github.com/daschl/grok,Apache-2.0,Michael Nitschinger h2,https://github.com/hyperium/h2,MIT,"Carl Lerche , Sean McArthur " hash_hasher,https://github.com/Fraser999/Hash-Hasher,Apache-2.0 OR MIT,Fraser Hutchison @@ -421,8 +422,11 @@ rdkafka,https://github.com/fede1024/rust-rdkafka,MIT,Federico Giraud redox_users,https://gitlab.redox-os.org/redox-os/users,MIT,"Jose Narvaez , Wesley Hershberger " -regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers +regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant +regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " +regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers +regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name @@ -470,6 +474,7 @@ serde-toml-merge,https://github.com/jdrouet/serde-toml-merge,MIT,Jeremie Drouet serde-value,https://github.com/arcnmx/serde-value,MIT,arcnmx serde-xml-rs,https://github.com/RReverser/serde-xml-rs,MIT,Ingvar Stepanyan serde_bytes,https://github.com/serde-rs/bytes,MIT OR Apache-2.0,David Tolnay +serde_derive,https://github.com/serde-rs/serde,MIT OR Apache-2.0,David Tolnay serde_json,https://github.com/serde-rs/json,MIT OR Apache-2.0,"Erick Tryzelaar , David Tolnay " serde_nanos,https://github.com/caspervonb/serde_nanos,MIT OR Apache-2.0,Casper Beyer serde_path_to_error,https://github.com/dtolnay/path-to-error,MIT OR Apache-2.0,David Tolnay diff --git a/Makefile b/Makefile index d988725b612f8..c1c2970767305 100644 --- a/Makefile +++ b/Makefile @@ -332,7 +332,7 @@ test-behavior: test-behavior-transforms test-behavior-formats test-behavior-conf test-integration: ## Runs all integration tests test-integration: test-integration-amqp test-integration-appsignal test-integration-aws test-integration-axiom test-integration-azure test-integration-chronicle test-integration-clickhouse test-integration: test-integration-databend test-integration-docker-logs test-integration-elasticsearch -test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-humio test-integration-http-client test-integration-influxdb +test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-greptimedb test-integration-humio test-integration-http-client test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar test-integration: test-integration-redis test-integration-splunk test-integration-dnstap test-integration-datadog-agent test-integration-datadog-logs diff --git a/STYLE.md b/STYLE.md index 4e3e70818f708..51b454afe8914 100644 --- a/STYLE.md +++ b/STYLE.md @@ -327,3 +327,22 @@ you'll need to use an asynchronous-specific synchronization primitives, namely t itself. The documentation on `tokio`'s own [`Mutex`](https://docs.rs/tokio/latest/tokio/sync/struct.Mutex.html), for example, calls out the specifics of when and where you might need to use it vs the one from `std::sync`. + + +## New Configuration Fields vs CLI flags + +Vector makes the distinction between configuration items that are essential to understand data +pipelines and runtime flags that determine the details of the runtime behavior. The main configuration +generally lives in a file in the current directory or in `/etc/vector`. + +Examples of main configuration fields are source, transformation, and sink declaration, as well as +information about where any disk buffers should be persisted. + +For configuration items that purely inform details of Vector's runtime behavior, CLI flags without +corresponding configuration fields should be used. + +An example of a runtime flag is +`vector run --no-graceful-shutdown-limit`, which tells Vector to ignore SIGINTs and to continue running +as normal until a SIGKILL is received. In this case, as the configuration describes the desired runtime +behavior in a specific environment and not to the underlying data pipeline, no corresponding field in +the configuration file should exist. diff --git a/build.rs b/build.rs index 31c151f4b61d7..4e8c4f4986304 100644 --- a/build.rs +++ b/build.rs @@ -1,11 +1,4 @@ -use std::{ - collections::HashSet, - env, - fs::File, - io::Write, - path::{Path, PathBuf}, - process::Command, -}; +use std::{collections::HashSet, env, fs::File, io::Write, path::Path, process::Command}; struct TrackedEnv { tracked: HashSet, @@ -137,7 +130,7 @@ fn main() { // in a type-safe way, which is necessary for incrementally building certain payloads, like // the ones generated in the `datadog_metrics` sink. let protobuf_fds_path = - PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set")) + Path::new(&std::env::var("OUT_DIR").expect("OUT_DIR environment variable not set")) .join("protobuf-fds.bin"); let mut prost_build = prost_build::Config::new(); diff --git a/distribution/install.sh b/distribution/install.sh index e2903c7b11e7e..534d261c96d1e 100755 --- a/distribution/install.sh +++ b/distribution/install.sh @@ -12,7 +12,7 @@ set -u # If PACKAGE_ROOT is unset or empty, default it. PACKAGE_ROOT="${PACKAGE_ROOT:-"https://packages.timber.io/vector"}" -VECTOR_VERSION="0.30.0" +VECTOR_VERSION="0.31.0" _divider="--------------------------------------------------------------------------------" _prompt=">>>" _indent=" " @@ -152,6 +152,16 @@ install_from_archive() { x86_64-*linux*-musl) _archive_arch="x86_64-unknown-linux-musl" ;; + aarch64-apple-darwin) + # This if statement can be removed when Vector publishes aarch64-apple-darwin builds + if /usr/bin/pgrep oahd >/dev/null 2>&1; then + echo "Rosetta is installed, installing x86_64-apple-darwin archive" + _archive_arch="x86_64-apple-darwin" + else + echo "Builds for Apple Silicon are not published today, please install Rosetta" + err "unsupported arch: $_arch" + fi + ;; aarch64-*linux*) _archive_arch="aarch64-unknown-linux-musl" ;; diff --git a/distribution/kubernetes/vector-agent/README.md b/distribution/kubernetes/vector-agent/README.md index 58bc8ff221193..531ce3abac357 100644 --- a/distribution/kubernetes/vector-agent/README.md +++ b/distribution/kubernetes/vector-agent/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.22.0 with the following `values.yaml`: +version 0.23.0 with the following `values.yaml`: ```yaml role: Agent diff --git a/distribution/kubernetes/vector-agent/configmap.yaml b/distribution/kubernetes/vector-agent/configmap.yaml index d7c928b159f7c..cc4d0a701a446 100644 --- a/distribution/kubernetes/vector-agent/configmap.yaml +++ b/distribution/kubernetes/vector-agent/configmap.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" data: agent.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-agent/daemonset.yaml b/distribution/kubernetes/vector-agent/daemonset.yaml index 8508420956313..5deb05f8a834b 100644 --- a/distribution/kubernetes/vector-agent/daemonset.yaml +++ b/distribution/kubernetes/vector-agent/daemonset.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: {} spec: selector: @@ -16,6 +16,7 @@ spec: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent + minReadySeconds: 0 template: metadata: annotations: {} @@ -29,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.30.0-distroless-libc" + image: "timberio/vector:0.31.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir @@ -51,6 +52,8 @@ spec: value: "/host/proc" - name: SYSFS_ROOT value: "/host/sys" + - name: VECTOR_LOG + value: "info" ports: - name: prom-exporter containerPort: 9090 @@ -61,17 +64,17 @@ spec: - name: config mountPath: "/etc/vector/" readOnly: true - - name: var-log - mountPath: "/var/log/" + - mountPath: /var/log/ + name: var-log readOnly: true - - name: var-lib - mountPath: "/var/lib" + - mountPath: /var/lib + name: var-lib readOnly: true - - name: procfs - mountPath: "/host/proc" + - mountPath: /host/proc + name: procfs readOnly: true - - name: sysfs - mountPath: "/host/sys" + - mountPath: /host/sys + name: sysfs readOnly: true terminationGracePeriodSeconds: 60 volumes: @@ -83,15 +86,15 @@ spec: - name: data hostPath: path: "/var/lib/vector" - - name: var-log - hostPath: - path: "/var/log/" - - name: var-lib - hostPath: - path: "/var/lib/" - - name: procfs - hostPath: - path: "/proc" - - name: sysfs - hostPath: - path: "/sys" + - hostPath: + path: /var/log/ + name: var-log + - hostPath: + path: /var/lib/ + name: var-lib + - hostPath: + path: /proc + name: procfs + - hostPath: + path: /sys + name: sysfs diff --git a/distribution/kubernetes/vector-agent/rbac.yaml b/distribution/kubernetes/vector-agent/rbac.yaml index 2ad572473e7d7..2309c07b8b9d2 100644 --- a/distribution/kubernetes/vector-agent/rbac.yaml +++ b/distribution/kubernetes/vector-agent/rbac.yaml @@ -10,7 +10,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" rules: - apiGroups: - "" @@ -31,7 +31,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/distribution/kubernetes/vector-agent/service-headless.yaml b/distribution/kubernetes/vector-agent/service-headless.yaml index 18ea854b8f9e7..1ef8fb122e0e3 100644 --- a/distribution/kubernetes/vector-agent/service-headless.yaml +++ b/distribution/kubernetes/vector-agent/service-headless.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-agent/serviceaccount.yaml b/distribution/kubernetes/vector-agent/serviceaccount.yaml index 18d7093b583f8..389773e9101a3 100644 --- a/distribution/kubernetes/vector-agent/serviceaccount.yaml +++ b/distribution/kubernetes/vector-agent/serviceaccount.yaml @@ -8,5 +8,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Agent - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/README.md b/distribution/kubernetes/vector-aggregator/README.md index 194d80cc3ebb1..4c5a3b4fcb1b4 100644 --- a/distribution/kubernetes/vector-aggregator/README.md +++ b/distribution/kubernetes/vector-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.22.0 with the following `values.yaml`: +version 0.23.0 with the following `values.yaml`: ```yaml diff --git a/distribution/kubernetes/vector-aggregator/configmap.yaml b/distribution/kubernetes/vector-aggregator/configmap.yaml index 028a2f273a075..3f82d196965e4 100644 --- a/distribution/kubernetes/vector-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-aggregator/configmap.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-aggregator/service-headless.yaml b/distribution/kubernetes/vector-aggregator/service-headless.yaml index 06cad3551b635..643102caccd3c 100644 --- a/distribution/kubernetes/vector-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-aggregator/service-headless.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-aggregator/service.yaml b/distribution/kubernetes/vector-aggregator/service.yaml index 449a24950bc1e..023e8dfd0f557 100644 --- a/distribution/kubernetes/vector-aggregator/service.yaml +++ b/distribution/kubernetes/vector-aggregator/service.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml index 0bf2da2d58d3b..024e2f65b3b1b 100644 --- a/distribution/kubernetes/vector-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-aggregator/serviceaccount.yaml @@ -8,5 +8,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" automountServiceAccountToken: true diff --git a/distribution/kubernetes/vector-aggregator/statefulset.yaml b/distribution/kubernetes/vector-aggregator/statefulset.yaml index 2eef56ffd5ad9..ad5a0d0086a2b 100644 --- a/distribution/kubernetes/vector-aggregator/statefulset.yaml +++ b/distribution/kubernetes/vector-aggregator/statefulset.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -32,7 +32,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.30.0-distroless-libc" + image: "timberio/vector:0.31.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/README.md b/distribution/kubernetes/vector-stateless-aggregator/README.md index 2703746f5d435..69466743c76ec 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/README.md +++ b/distribution/kubernetes/vector-stateless-aggregator/README.md @@ -1,6 +1,6 @@ The kubernetes manifests found in this directory have been automatically generated from the [helm chart `vector/vector`](https://github.com/vectordotdev/helm-charts/tree/master/charts/vector) -version 0.22.0 with the following `values.yaml`: +version 0.23.0 with the following `values.yaml`: ```yaml role: Stateless-Aggregator diff --git a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml index 766c693669f7b..38de88fe03a06 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/configmap.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" data: aggregator.yaml: | data_dir: /vector-data-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml index 6ff20dc958816..6072b5464fa86 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/deployment.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: {} spec: replicas: 1 @@ -30,7 +30,7 @@ spec: dnsPolicy: ClusterFirst containers: - name: vector - image: "timberio/vector:0.30.0-distroless-libc" + image: "timberio/vector:0.31.0-distroless-libc" imagePullPolicy: IfNotPresent args: - --config-dir diff --git a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml index 3230af57fbd76..a7d86afb480c5 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service-headless.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: spec: clusterIP: None diff --git a/distribution/kubernetes/vector-stateless-aggregator/service.yaml b/distribution/kubernetes/vector-stateless-aggregator/service.yaml index a22d86ff0925f..9d9f59d239018 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/service.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/service.yaml @@ -8,7 +8,7 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" annotations: spec: ports: diff --git a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml index 50bba163cf9f5..d46fee95872d7 100644 --- a/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml +++ b/distribution/kubernetes/vector-stateless-aggregator/serviceaccount.yaml @@ -8,5 +8,5 @@ metadata: app.kubernetes.io/name: vector app.kubernetes.io/instance: vector app.kubernetes.io/component: Stateless-Aggregator - app.kubernetes.io/version: "0.30.0-distroless-libc" + app.kubernetes.io/version: "0.31.0-distroless-libc" automountServiceAccountToken: true diff --git a/docs/DEVELOPING.md b/docs/DEVELOPING.md index 2eb9fc1d63fab..dca85c077b338 100644 --- a/docs/DEVELOPING.md +++ b/docs/DEVELOPING.md @@ -121,9 +121,10 @@ To build Vector on your own host will require a fairly complete development envi Loosely, you'll need the following: - **To build Vector:** Have working Rustup, Protobuf tools, C++/C build tools (LLVM, GCC, or MSVC), Python, and Perl, `make` (the GNU one preferably), `bash`, `cmake`, `GNU coreutils`, and `autotools`. +- **To run `make test`:** Install [`cargo-nextest`](https://nexte.st/) - **To run integration tests:** Have `docker` available, or a real live version of that service. (Use `AUTOSPAWN=false`) - **To run `make check-component-features`:** Have `remarshal` installed. -- **To run `make check-licenses` or `cargo vdev build licenses`:** Have `rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). +- **To run `make check-licenses` or `cargo vdev build licenses`:** Have `dd-rust-license-tool` [installed](https://github.com/DataDog/rust-license-tool). - **To run `cargo vdev build component-docs`:** Have `cue` [installed](https://cuelang.org/docs/install/). If you find yourself needing to run something inside the Docker environment described above, that's totally fine, they won't collide or hurt each other. In this case, you'd just run `make environment-generate`. @@ -141,19 +142,19 @@ cargo build make build-dev # Validate your test pass cargo test sources::example -make test scope="sources::example" +make test SCOPE="sources::example" # Validate tests (that do not require other services) pass cargo test make test # Validate your tests pass (starting required services in Docker) -make test-integration scope="sources::example" +make test-integration SCOPE="sources::example" # Validate your tests pass against a live service. -make test-integration scope="sources::example" autospawn=false +make test-integration SCOPE="sources::example" autospawn=false cargo test --features docker sources::example # Validate all tests pass (starting required services in Docker) make test-integration # Run your benchmarks -make bench scope="transforms::example" +make bench SCOPE="transforms::example" cargo bench transforms::example # Format your code before pushing! make fmt diff --git a/docs/REVIEWING.md b/docs/REVIEWING.md index f8356f7a2f271..26dfd8943d896 100644 --- a/docs/REVIEWING.md +++ b/docs/REVIEWING.md @@ -36,6 +36,42 @@ following items should also be checked: - [ ] Does it comply with [component spec](specs/component.md)? - [ ] Does it comply with the [instrumentation spec](specs/instrumentation.md)? + +### Checklist - new sink + +This checklist is specific for Vector's sink code. + +#### Logic + +- [ ] Does it work? Do you understand what it is supposed to be doing? +- [ ] Does the retry logic make sense? +- [ ] Are the tests testing that the sink is emitting the correct metrics? +- [ ] Are there integration tests? + +#### Code structure + +- [ ] Is it using the sink prelude (`use crate::sinks::prelude::*`)? +- [ ] Is the sink a stream based sink? + Check that the return value from `SinkConfig::build` is the return from `VectorSink::from_event_streamsink`. +- [ ] Is it gated by sensible feature flags? +- [ ] Is the code modularized into `mod.rs`, `config.rs`, `sink.rs`, `request_builder.rs`, `service.rs` +- [ ] Does the code follow our [style guidelines]. + +#### Documentation + +- [ ] Look at the doc preview on Netlify. Does it look good? +- [ ] Is there a `cue` file linking to `base`? +- [ ] Is there a markdown file under `/website/content/en/docs/reference/configuration/sinks/`? +- [ ] Are module comments included in `mod.rs` linking to any relevant areas in the external services documentation? + +#### Configuration + +- [ ] Are TLS settings configurable? +- [ ] Are the Request settings configurable? +- [ ] Should it have proxy settings? If so, are they in place? +- [ ] Does it need batch settings? If so, are they used? + + ## Expectations We endeavour to review all PRs within 2 working days (Monday to Friday) of submission. @@ -131,3 +167,5 @@ your best judgment, some code requires more testing than others depending on its importance. For integrations, consider whether the code could be integration tested. + +[style guidelines]: https://github.com/vectordotdev/vector/blob/master/STYLE.md diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 85cd4cb6813a3..0e34db9025356 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" publish = false [dependencies] -apache-avro = { version = "0.14.0", default-features = false } +apache-avro = { version = "0.15.0", default-features = false } bytes = { version = "1", default-features = false } chrono = { version = "0.4", default-features = false } csv = { version = "1.2", default-features = false } @@ -17,11 +17,11 @@ memchr = { version = "2", default-features = false } once_cell = { version = "1.18", default-features = false } ordered-float = { version = "3.7.0", default-features = false } prost = { version = "0.11.8", default-features = false, features = ["std"] } -regex = { version = "1.8.4", default-features = false, features = ["std", "perf"] } +regex = { version = "1.9.1", default-features = false, features = ["std", "perf"] } serde = { version = "1", default-features = false, features = ["derive"] } serde_json = { version = "1", default-features = false } smallvec = { version = "1", default-features = false, features = ["union"] } -snafu = { version = "0.7.4", default-features = false, features = ["futures"] } +snafu = { version = "0.7.5", default-features = false, features = ["futures"] } syslog_loose = { version = "0.18", default-features = false, optional = true } tokio-util = { version = "0.7", default-features = false, features = ["codec"] } tracing = { version = "0.1", default-features = false } diff --git a/lib/docs-renderer/Cargo.toml b/lib/docs-renderer/Cargo.toml index 998d19d95bc3d..f391dc8f63139 100644 --- a/lib/docs-renderer/Cargo.toml +++ b/lib/docs-renderer/Cargo.toml @@ -6,10 +6,10 @@ edition = "2021" publish = false [dependencies] -anyhow = { version = "1.0.71", default-features = false, features = ["std"] } +anyhow = { version = "1.0.72", default-features = false, features = ["std"] } serde = { version = "1.0", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["std"] } -snafu = { version = "0.7.4", default-features = false } +snafu = { version = "0.7.5", default-features = false } tracing = { version = "0.1.34", default-features = false } tracing-subscriber = { version = "0.3.17", default-features = false, features = ["ansi", "env-filter", "fmt", "json", "registry", "tracing-log"] } vector-config = { path = "../vector-config" } diff --git a/lib/enrichment/Cargo.toml b/lib/enrichment/Cargo.toml index 1a8e2467f0fed..f6674c8c96ec3 100644 --- a/lib/enrichment/Cargo.toml +++ b/lib/enrichment/Cargo.toml @@ -8,5 +8,5 @@ publish = false [dependencies] arc-swap = { version = "1.6.0", default-features = false } chrono = { version = "0.4.19", default-features = false } -dyn-clone = { version = "1.0.11", default-features = false } +dyn-clone = { version = "1.0.12", default-features = false } vrl.workspace = true diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index 50631a6765175..594d2f7b36f56 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -19,7 +19,7 @@ vector-config-common = { path = "../vector-config-common", default-features = fa vector-config-macros = { path = "../vector-config-macros", default-features = false } [dependencies.bstr] -version = "1.5" +version = "1.6" default-features = false features = [] @@ -34,7 +34,7 @@ default-features = false features = ["clock", "serde"] [dependencies.dashmap] -version = "5.2" +version = "5.5" default-features = false features = [] @@ -69,7 +69,7 @@ default-features = false features = [] [dependencies.tokio] -version = "1.29.0" +version = "1.29.1" default-features = false features = ["full"] diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index fd1a5aa7ab672..9abd68601fc39 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,8 +14,8 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.11.18", features = ["json"] } serde_json = "1" -tokio = { version = "1.29.0", features = ["full"] } -indoc = "2.0.1" +tokio = { version = "1.29.1", features = ["full"] } +indoc = "2.0.3" env_logger = "0.10" tracing = { version = "0.1", features = ["log"] } rand = "0.8" diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index 7ab378add13ff..305eea32de86c 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json = "1" tempfile = "3" -tokio = { version = "1.29.0", features = ["full"] } +tokio = { version = "1.29.1", features = ["full"] } log = "0.4" diff --git a/lib/loki-logproto/Cargo.toml b/lib/loki-logproto/Cargo.toml index f09e431152d02..c140e168546a9 100644 --- a/lib/loki-logproto/Cargo.toml +++ b/lib/loki-logproto/Cargo.toml @@ -2,6 +2,8 @@ name = "loki-logproto" version = "0.1.0" edition = "2021" +license = "MPL-2.0" +publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/loki-logproto/LICENSE b/lib/loki-logproto/LICENSE new file mode 100644 index 0000000000000..e87a115e462e1 --- /dev/null +++ b/lib/loki-logproto/LICENSE @@ -0,0 +1,363 @@ +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + diff --git a/lib/opentelemetry-proto/src/convert.rs b/lib/opentelemetry-proto/src/convert.rs index 4ccfda0b726ba..057561df755bb 100644 --- a/lib/opentelemetry-proto/src/convert.rs +++ b/lib/opentelemetry-proto/src/convert.rs @@ -208,7 +208,7 @@ impl ResourceLog { log_namespace.insert_vector_metadata( &mut log, - Some(log_schema().source_type_key()), + log_schema().source_type_key(), path!("source_type"), Bytes::from_static(SOURCE_NAME.as_bytes()), ); diff --git a/lib/tracing-limit/Cargo.toml b/lib/tracing-limit/Cargo.toml index 48c0bf0762a98..5eb8fe73f0840 100644 --- a/lib/tracing-limit/Cargo.toml +++ b/lib/tracing-limit/Cargo.toml @@ -9,7 +9,7 @@ license = "MPL-2.0" [dependencies] tracing-core = { version = "0.1", default-features = false } tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] } -dashmap = { version = "5.2.0", default-features = false } +dashmap = { version = "5.5.0", default-features = false } [dev-dependencies] criterion = "0.5" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index ed243d3e368ec..cdfb7504ba35f 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -9,16 +9,16 @@ license = "MPL-2.0" [dependencies] # Serde -serde = { version = "1.0.164", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.99", default-features = false, features = ["raw_value"] } +serde = { version = "1.0.173", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.103", default-features = false, features = ["raw_value"] } # Error handling -anyhow = { version = "1.0.71", default-features = false, features = ["std"] } +anyhow = { version = "1.0.72", default-features = false, features = ["std"] } # Tokio / Futures async-trait = { version = "0.1", default-features = false } futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } -tokio = { version = "1.29.0", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.29.1", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.14", default-features = false, features = ["sync"] } # GraphQL @@ -33,4 +33,4 @@ chrono = { version = "0.4.6", default-features = false, features = ["serde"] } clap = { version = "4.1.14", default-features = false, features = ["derive"] } url = { version = "2.4.0", default-features = false } uuid = { version = "1", default-features = false, features = ["serde", "v4"] } -indoc = { version = "2.0.1", default-features = false } +indoc = { version = "2.0.3", default-features = false } diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 09de0d0ac6153..23a5ebc1e4f5f 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -17,14 +17,14 @@ crossbeam-utils = { version = "0.8.16", default-features = false } fslock = { version = "0.2.1", default-features = false, features = ["std"] } futures = { version = "0.3.28", default-features = false, features = ["std"] } memmap2 = { version = "0.7.1", default-features = false } -metrics = "0.21.0" +metrics = "0.21.1" num-traits = { version = "0.2.15", default-features = false } -pin-project = { version = "1.1.1", default-features = false } +pin-project = { version = "1.1.2", default-features = false } rkyv = { version = "0.7.40", default-features = false, features = ["size_32", "std", "strict", "validation"] } -serde = { version = "1.0.164", default-features = false, features = ["derive"] } -snafu = { version = "0.7.4", default-features = false, features = ["std"] } +serde = { version = "1.0.173", default-features = false, features = ["derive"] } +snafu = { version = "0.7.5", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.29.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.29.1", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } @@ -37,7 +37,7 @@ criterion = { version = "0.5", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.8" hdrhistogram = "7.5.2" metrics-tracing-context = { version = "0.14.0", default-features = false } -metrics-util = { version = "0.15.0", default-features = false, features = ["debugging"] } +metrics-util = { version = "0.15.1", default-features = false, features = ["debugging"] } once_cell = "1.18" proptest = "1.2" quickcheck = "1.0" diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index 24023729e5104..7b07afb822d50 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -49,18 +49,18 @@ crossbeam-utils = { version = "0.8.16", default-features = false } derivative = { version = "2.2.0", default-features = false } futures = { version = "0.3.28", default-features = false, features = ["std"] } indexmap = { version = "~2.0.0", default-features = false, features = ["std"] } -metrics = "0.21.0" +metrics = "0.21.1" nom = { version = "7", optional = true } ordered-float = { version = "3.7.0", default-features = false } -paste = "1.0.12" -pin-project = { version = "1.1.1", default-features = false } +paste = "1.0.14" +pin-project = { version = "1.1.2", default-features = false } ryu = { version = "1", default-features = false } -serde_json = { version = "1.0.99", default-features = false, features = ["std", "raw_value"] } -serde = { version = "1.0.164", optional = true, features = ["derive"] } +serde_json = { version = "1.0.103", default-features = false, features = ["std", "raw_value"] } +serde = { version = "1.0.173", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } -tokio = { version = "1.29.0", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.29.1", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } vrl.workspace = true vector-config = { path = "../vector-config" } @@ -69,6 +69,6 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] futures = { version = "0.3.28", default-features = false, features = ["async-await", "std"] } -tokio = { version = "1.29.0", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.29.1", default-features = false, features = ["rt", "time"] } quickcheck = "1" quickcheck_macros = "1" diff --git a/lib/vector-common/src/internal_event/cached_event.rs b/lib/vector-common/src/internal_event/cached_event.rs index e672848c93584..daa3f8ea803af 100644 --- a/lib/vector-common/src/internal_event/cached_event.rs +++ b/lib/vector-common/src/internal_event/cached_event.rs @@ -17,8 +17,9 @@ use super::{InternalEventHandle, RegisterInternalEvent}; /// new event is emitted for a previously unseen set of tags an event is registered /// and stored in the cache. #[derive(Derivative)] -#[derivative(Clone(bound = ""), Default(bound = ""))] -pub struct RegisteredEventCache { +#[derivative(Clone(bound = "T: Clone"))] +pub struct RegisteredEventCache { + fixed_tags: T, cache: Arc< RwLock< BTreeMap< @@ -36,16 +37,31 @@ pub trait RegisterTaggedInternalEvent: RegisterInternalEvent { /// that will be used when registering the event. type Tags; - fn register(tags: Self::Tags) -> ::Handle; + /// The type that contains data necessary to extract the tags that will + /// be fixed and only need setting up front when the cache is first created. + type Fixed; + + fn register(fixed: Self::Fixed, tags: Self::Tags) -> ::Handle; } -impl RegisteredEventCache +impl RegisteredEventCache where Data: Sized, EventHandle: InternalEventHandle, Tags: Ord + Clone, - Event: RegisterInternalEvent + RegisterTaggedInternalEvent, + FixedTags: Clone, + Event: RegisterInternalEvent + + RegisterTaggedInternalEvent, { + /// Create a new event cache with a set of fixed tags. These tags are passed to + /// all registered events. + pub fn new(fixed_tags: FixedTags) -> Self { + Self { + fixed_tags, + cache: Arc::default(), + } + } + /// Emits the event with the given tags. /// It will register the event and store in the cache if this has not already /// been done. @@ -58,7 +74,10 @@ where if let Some(event) = read.get(tags) { event.emit(value); } else { - let event = ::register(tags.clone()); + let event = ::register( + self.fixed_tags.clone(), + tags.clone(), + ); event.emit(value); // Ensure the read lock is dropped so we can write. @@ -67,3 +86,43 @@ where } } } + +#[cfg(test)] +mod tests { + #![allow(unreachable_pub)] + use metrics::{register_counter, Counter}; + + use super::*; + + crate::registered_event!( + TestEvent { + fixed: String, + dynamic: String, + } => { + event: Counter = { + register_counter!("test_event_total", "fixed" => self.fixed, "dynamic" => self.dynamic) + }, + } + + fn emit(&self, count: u64) { + self.event.increment(count); + } + + fn register(fixed: String, dynamic: String) { + crate::internal_event::register(TestEvent { + fixed, + dynamic, + }) + } + ); + + #[test] + fn test_fixed_tag() { + let event: RegisteredEventCache = + RegisteredEventCache::new("fixed".to_string()); + + for tag in 1..=5 { + event.emit(&format!("dynamic{tag}"), tag); + } + } +} diff --git a/lib/vector-common/src/internal_event/events_sent.rs b/lib/vector-common/src/internal_event/events_sent.rs index d12a22bf17e8a..22d614b0016eb 100644 --- a/lib/vector-common/src/internal_event/events_sent.rs +++ b/lib/vector-common/src/internal_event/events_sent.rs @@ -91,7 +91,7 @@ crate::registered_event!( self.event_bytes.increment(byte_size.get() as u64); } - fn register(tags: EventCountTags) { + fn register(_fixed: (), tags: EventCountTags) { super::register(TaggedEventsSent::new( tags, )) diff --git a/lib/vector-common/src/internal_event/mod.rs b/lib/vector-common/src/internal_event/mod.rs index 4ce2f5335ac49..8785fa45757a8 100644 --- a/lib/vector-common/src/internal_event/mod.rs +++ b/lib/vector-common/src/internal_event/mod.rs @@ -1,6 +1,6 @@ mod bytes_received; mod bytes_sent; -mod cached_event; +pub mod cached_event; pub mod component_events_dropped; mod events_received; mod events_sent; @@ -219,7 +219,7 @@ macro_rules! registered_event { fn emit(&$slf:ident, $data_name:ident: $data:ident) $emit_body:block - $(fn register($tags_name:ident: $tags:ty) + $(fn register($fixed_name:ident: $fixed_tags:ty, $tags_name:ident: $tags:ty) $register_body:block)? ) => { paste::paste!{ @@ -251,10 +251,12 @@ macro_rules! registered_event { $(impl $crate::internal_event::cached_event::RegisterTaggedInternalEvent for $event { type Tags = $tags; + type Fixed = $fixed_tags; fn register( + $fixed_name: $fixed_tags, $tags_name: $tags, - ) -> ::Handle { + ) -> ::Handle { $register_body } })? diff --git a/lib/vector-common/src/request_metadata.rs b/lib/vector-common/src/request_metadata.rs index 87ff25a1cf5dc..d11077a206c0d 100644 --- a/lib/vector-common/src/request_metadata.rs +++ b/lib/vector-common/src/request_metadata.rs @@ -129,9 +129,9 @@ impl GroupedCountByteSize { } /// Emits our counts to a `RegisteredEvent` cached event. - pub fn emit_event(&self, event_cache: &RegisteredEventCache) + pub fn emit_event(&self, event_cache: &RegisteredEventCache<(), T>) where - T: RegisterTaggedInternalEvent, + T: RegisterTaggedInternalEvent, H: InternalEventHandle, { match self { diff --git a/lib/vector-config-common/Cargo.toml b/lib/vector-config-common/Cargo.toml index 0d4ddaed219a0..9de0b5159865a 100644 --- a/lib/vector-config-common/Cargo.toml +++ b/lib/vector-config-common/Cargo.toml @@ -6,11 +6,11 @@ license = "MPL-2.0" [dependencies] convert_case = { version = "0.6", default-features = false } -darling = { version = "0.13", default-features = false, features = ["suggestions"] } +darling = { version = "0.20", default-features = false, features = ["suggestions"] } once_cell = { version = "1", default-features = false, features = ["std"] } proc-macro2 = { version = "1.0", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false, features = ["std"] } -syn = { version = "1.0", features = ["full", "extra-traits", "visit-mut", "visit"] } +syn = { version = "2.0", features = ["full", "extra-traits", "visit-mut", "visit"] } tracing = { version = "0.1.34", default-features = false } quote = { version = "1.0", default-features = false } diff --git a/lib/vector-config-common/src/validation.rs b/lib/vector-config-common/src/validation.rs index f264b707fe461..569f80cf33007 100644 --- a/lib/vector-config-common/src/validation.rs +++ b/lib/vector-config-common/src/validation.rs @@ -1,7 +1,7 @@ use darling::FromMeta; use proc_macro2::TokenStream; use quote::{quote, ToTokens}; -use syn::{Lit, Meta}; +use syn::{Expr, Lit, Meta}; use crate::{ num::{ERR_NUMERIC_OUT_OF_RANGE, NUMERIC_ENFORCED_LOWER_BOUND, NUMERIC_ENFORCED_UPPER_BOUND}, @@ -336,17 +336,19 @@ fn maybe_float_or_int(meta: &Meta) -> darling::Result> { // First make sure we can even get a valid f64 from this meta item. let result = match meta { Meta::Path(_) => Err(darling::Error::unexpected_type("path")), - Meta::List(_) => Err(darling::Error::unexpected_type("path")), - Meta::NameValue(nv) => match &nv.lit { - Lit::Str(s) => { - let s = s.value(); - s.as_str() - .parse() - .map_err(|_| darling::Error::unknown_value(s.as_str())) - } - Lit::Int(i) => i.base10_parse::().map_err(Into::into), - Lit::Float(f) => f.base10_parse::().map_err(Into::into), - lit => Err(darling::Error::unexpected_lit_type(lit)), + Meta::List(_) => Err(darling::Error::unexpected_type("list")), + Meta::NameValue(nv) => match &nv.value { + Expr::Lit(expr) => match &expr.lit { + Lit::Str(s) => { + let s = s.value(); + s.parse() + .map_err(|_| darling::Error::unknown_value(s.as_str())) + } + Lit::Int(i) => i.base10_parse::().map_err(Into::into), + Lit::Float(f) => f.base10_parse::().map_err(Into::into), + lit => Err(darling::Error::unexpected_lit_type(lit)), + }, + expr => Err(darling::Error::unexpected_expr_type(expr)), }, }; diff --git a/lib/vector-config-macros/Cargo.toml b/lib/vector-config-macros/Cargo.toml index 2a133adaa4fcd..d15cd12c6ed76 100644 --- a/lib/vector-config-macros/Cargo.toml +++ b/lib/vector-config-macros/Cargo.toml @@ -8,13 +8,13 @@ license = "MPL-2.0" proc-macro = true [dependencies] -darling = { version = "0.13", default-features = false, features = ["suggestions"] } +darling = { version = "0.20", default-features = false, features = ["suggestions"] } proc-macro2 = { version = "1.0", default-features = false } quote = { version = "1.0", default-features = false } -serde_derive_internals = "0.26" -syn = { version = "1.0", default-features = false, features = ["full", "extra-traits", "visit-mut", "visit"] } +serde_derive_internals = "0.28" +syn = { version = "2.0", default-features = false, features = ["full", "extra-traits", "visit-mut", "visit"] } vector-config-common = { path = "../vector-config-common" } [dev-dependencies] -serde = { version = "1.0.164", default-features = false } +serde = { version = "1.0.173", default-features = false } vector-config = { path = "../vector-config" } diff --git a/lib/vector-config-macros/src/ast/container.rs b/lib/vector-config-macros/src/ast/container.rs index f4a47e71d18e7..28cc12f69a35a 100644 --- a/lib/vector-config-macros/src/ast/container.rs +++ b/lib/vector-config-macros/src/ast/container.rs @@ -368,7 +368,7 @@ impl<'a> Container<'a> { /// standard `#[deprecated]` attribute, neither automatically applying it nor deriving the /// deprecation status of a field when it is present. pub fn deprecated(&self) -> bool { - self.attrs.deprecated.is_some() + self.attrs.deprecated.is_present() } /// Metadata (custom attributes) for the container, if any. @@ -584,7 +584,7 @@ mod tests { assert_eq!(literals_to_idents(&["T"]), idents); // We don't support parenthesized type parameters, like when using a function pointer type. - let parenthesized_type: Type = parse_quote! { Something String> }; + let parenthesized_type: Type = parse_quote! { Something String> }; let idents = get_generic_type_param_idents(&parenthesized_type); assert_eq!(None, idents); } diff --git a/lib/vector-config-macros/src/ast/field.rs b/lib/vector-config-macros/src/ast/field.rs index 790bf07b7dadd..d38ce6c95417c 100644 --- a/lib/vector-config-macros/src/ast/field.rs +++ b/lib/vector-config-macros/src/ast/field.rs @@ -2,9 +2,10 @@ use darling::{ util::{Flag, Override, SpannedValue}, FromAttributes, }; -use proc_macro2::Span; +use proc_macro2::{Span, TokenStream}; +use quote::ToTokens; use serde_derive_internals::ast as serde_ast; -use syn::{parse_quote, spanned::Spanned, ExprPath, Ident}; +use syn::{parse_quote, ExprPath, Ident}; use vector_config_common::validation::Validation; use super::{ @@ -174,7 +175,7 @@ impl<'a> Field<'a> { /// variants, to simply document themselves at the container/variant level and avoid needing to /// document that inner field which itself needs no further title/description. pub fn transparent(&self) -> bool { - self.attrs.transparent.is_some() + self.attrs.transparent.is_present() } /// Whether or not the field is deprecated. @@ -239,12 +240,9 @@ impl<'a> Field<'a> { } } -impl<'a> Spanned for Field<'a> { - fn span(&self) -> proc_macro2::Span { - match self.original.ident.as_ref() { - Some(ident) => ident.span(), - None => self.original.ty.span(), - } +impl<'a> ToTokens for Field<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.original.to_tokens(tokens) } } @@ -300,12 +298,12 @@ impl Attributes { // to throw an error if they are. As we're going to forcefully mark the field as // transparent, there's no reason to allow setting derived/transparent manually, as it // only leads to boilerplate and potential confusion. - if self.transparent.is_some() { - return Err(err_field_implicit_transparent(&self.transparent)); + if self.transparent.is_present() { + return Err(err_field_implicit_transparent(&self.transparent.span())); } - if self.derived.is_some() { - return Err(err_field_implicit_transparent(&self.derived)); + if self.derived.is_present() { + return Err(err_field_implicit_transparent(&self.derived.span())); } self.transparent = SpannedValue::new(Flag::present(), Span::call_site()); @@ -339,8 +337,8 @@ impl Attributes { // like a field that is flattened or not visible, it makes no sense to require a description or title for fields // in a virtual newtype. if self.description.is_none() - && !self.derived.is_some() - && !self.transparent.is_some() + && !self.derived.is_present() + && !self.transparent.is_present() && self.visible && !self.flatten && !is_virtual_newtype diff --git a/lib/vector-config-macros/src/ast/mod.rs b/lib/vector-config-macros/src/ast/mod.rs index 7688d6f2df511..1825f9e6fd4e9 100644 --- a/lib/vector-config-macros/src/ast/mod.rs +++ b/lib/vector-config-macros/src/ast/mod.rs @@ -1,4 +1,4 @@ -use darling::{error::Accumulator, util::path_to_string, FromMeta}; +use darling::{ast::NestedMeta, error::Accumulator, util::path_to_string, FromMeta}; use quote::ToTokens; use serde_derive_internals::{ast as serde_ast, attr as serde_attr}; @@ -9,7 +9,7 @@ mod variant; pub use container::Container; pub use field::Field; -use syn::{Expr, NestedMeta}; +use syn::Expr; pub use variant::Variant; use vector_config_common::constants; @@ -196,36 +196,46 @@ impl FromMeta for Metadata { errors.push(darling::Error::unexpected_type("list").with_span(nmeta)); None } - syn::Meta::NameValue(nv) => match &nv.lit { - // When dealing with a string literal, we check if it ends in `()`. If so, - // we emit that as-is, leading to doing a function call and using the return - // value of that function as the value for this key/value pair. - // - // Otherwise, we just treat the string literal normally. - syn::Lit::Str(s) => { - if s.value().ends_with("()") { - if let Ok(expr) = s.parse::() { - Some(LazyCustomAttribute::KeyValue { - key: path_to_string(&nv.path), - value: expr.to_token_stream(), - }) - } else { - errors.push( - darling::Error::custom(INVALID_VALUE_EXPR).with_span(nmeta), - ); - None + syn::Meta::NameValue(nv) => match &nv.value { + Expr::Lit(expr) => { + match &expr.lit { + // When dealing with a string literal, we check if it ends in `()`. If so, + // we emit that as-is, leading to doing a function call and using the return + // value of that function as the value for this key/value pair. + // + // Otherwise, we just treat the string literal normally. + syn::Lit::Str(s) => { + if s.value().ends_with("()") { + if let Ok(expr) = s.parse::() { + Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: expr.to_token_stream(), + }) + } else { + errors.push( + darling::Error::custom(INVALID_VALUE_EXPR) + .with_span(nmeta), + ); + None + } + } else { + Some(LazyCustomAttribute::KeyValue { + key: path_to_string(&nv.path), + value: s.value().to_token_stream(), + }) + } } - } else { - Some(LazyCustomAttribute::KeyValue { + lit => Some(LazyCustomAttribute::KeyValue { key: path_to_string(&nv.path), - value: s.value().to_token_stream(), - }) + value: lit.to_token_stream(), + }), } } - lit => Some(LazyCustomAttribute::KeyValue { - key: path_to_string(&nv.path), - value: lit.to_token_stream(), - }), + expr => { + errors + .push(darling::Error::unexpected_expr_type(expr).with_span(nmeta)); + None + } }, }, NestedMeta::Lit(_) => { diff --git a/lib/vector-config-macros/src/ast/util.rs b/lib/vector-config-macros/src/ast/util.rs index 30f66bd6a406a..ddd36c5739448 100644 --- a/lib/vector-config-macros/src/ast/util.rs +++ b/lib/vector-config-macros/src/ast/util.rs @@ -1,7 +1,10 @@ -use darling::error::Accumulator; +use darling::{ast::NestedMeta, error::Accumulator}; use quote::{quote, ToTokens}; use serde_derive_internals::{attr as serde_attr, Ctxt}; -use syn::{spanned::Spanned, Attribute, ExprPath, Lit, Meta, MetaNameValue, NestedMeta}; +use syn::{ + punctuated::Punctuated, spanned::Spanned, token::Comma, Attribute, Expr, ExprLit, ExprPath, + Lit, Meta, MetaNameValue, +}; const ERR_FIELD_MISSING_DESCRIPTION: &str = "field must have a description -- i.e. `/// This is a widget...` or `#[configurable(description = \"...\")] -- or derive it from the underlying type of the field by specifying `#[configurable(derived)]`"; const ERR_FIELD_IMPLICIT_TRANSPARENT: &str = @@ -15,12 +18,16 @@ pub fn try_extract_doc_title_description( let doc_comments = attributes .iter() // We only care about `doc` attributes. - .filter(|attribute| attribute.path.is_ident("doc")) + .filter(|attribute| attribute.path().is_ident("doc")) // Extract the value of the attribute if it's in the form of `doc = "..."`. - .filter_map(|attribute| match attribute.parse_meta() { - Ok(Meta::NameValue(MetaNameValue { - lit: Lit::Str(s), .. - })) => Some(s.value()), + .filter_map(|attribute| match &attribute.meta { + Meta::NameValue(MetaNameValue { + value: + Expr::Lit(ExprLit { + lit: Lit::Str(s), .. + }), + .. + }) => Some(s.value()), _ => None, }) .collect::>(); @@ -209,12 +216,13 @@ fn find_name_value_attribute( attributes .iter() // Only take attributes whose name matches `attr_name`. - .filter(|attr| path_matches(&attr.path, attr_name)) - // Make sure the contents actually parse as a normal structured attribute. - .filter_map(|attr| attr.parse_meta().ok()) + .filter(|attr| path_matches(attr.path(), attr_name)) // Derive macro helper attributes will always be in the list form. - .filter_map(|meta| match meta { - Meta::List(ml) => Some(ml.nested.into_iter()), + .filter_map(|attr| match &attr.meta { + Meta::List(ml) => ml + .parse_args_with(Punctuated::::parse_terminated) + .map(|nested| nested.into_iter()) + .ok(), _ => None, }) .flatten() @@ -222,7 +230,10 @@ fn find_name_value_attribute( // name matches `name_key`, and return their value. .find_map(|nm| match nm { NestedMeta::Meta(meta) => match meta { - Meta::NameValue(nv) if path_matches(&nv.path, name_key) => Some(nv.lit), + Meta::NameValue(nv) if path_matches(&nv.path, name_key) => match nv.value { + Expr::Lit(ExprLit { lit, .. }) => Some(lit), + _ => None, + }, _ => None, }, _ => None, @@ -259,7 +270,7 @@ fn find_name_value_attribute( /// returned. pub fn find_delegated_serde_deser_ty(attributes: &[syn::Attribute]) -> Option { // Make sure `#[serde_as(as = "...")]` is present. - find_name_value_attribute(attributes, "serde_as", "as") + find_name_value_attribute(attributes, "serde_as", "r#as") // Make sure `#[serde(with = "...")]` is present, and grab its value. .and_then(|_| find_name_value_attribute(attributes, "serde", "with")) // Try and parse the value as a type path. diff --git a/lib/vector-config-macros/src/ast/variant.rs b/lib/vector-config-macros/src/ast/variant.rs index ea25903a9ec00..5b54ebc30d18f 100644 --- a/lib/vector-config-macros/src/ast/variant.rs +++ b/lib/vector-config-macros/src/ast/variant.rs @@ -1,7 +1,7 @@ use darling::{error::Accumulator, util::Flag, FromAttributes}; -use proc_macro2::Ident; +use proc_macro2::{Ident, TokenStream}; +use quote::ToTokens; use serde_derive_internals::ast as serde_ast; -use syn::spanned::Spanned; use super::{ util::{try_extract_doc_title_description, DarlingResultIterator}, @@ -140,7 +140,7 @@ impl<'a> Variant<'a> { /// standard `#[deprecated]` attribute, neither automatically applying it nor deriving the /// deprecation status of a variant when it is present. pub fn deprecated(&self) -> bool { - self.attrs.deprecated.is_some() + self.attrs.deprecated.is_present() } /// Whether or not this variant is visible during either serialization or deserialization. @@ -166,9 +166,9 @@ impl<'a> Variant<'a> { } } -impl<'a> Spanned for Variant<'a> { - fn span(&self) -> proc_macro2::Span { - self.original.span() +impl<'a> ToTokens for Variant<'a> { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.original.to_tokens(tokens) } } diff --git a/lib/vector-config-macros/src/component_name.rs b/lib/vector-config-macros/src/component_name.rs index 1dd974d604a7b..8c7e5efd2e65e 100644 --- a/lib/vector-config-macros/src/component_name.rs +++ b/lib/vector-config-macros/src/component_name.rs @@ -106,7 +106,7 @@ pub fn derive_component_name_impl(input: TokenStream) -> TokenStream { fn attr_to_component_name(attr: &Attribute) -> Result, Error> { // First, filter out anything that isn't ours. if !path_matches( - &attr.path, + attr.path(), &[ attrs::ENRICHMENT_TABLE_COMPONENT, attrs::PROVIDER_COMPONENT, @@ -121,14 +121,14 @@ fn attr_to_component_name(attr: &Attribute) -> Result, Error> { // Reconstruct the original attribute path (i.e. `source`) from our marker version of it (i.e. // `source_component`), so that any error message we emit is contextually relevant. - let path_str = path_to_string(&attr.path); + let path_str = path_to_string(attr.path()); let component_type_attr = path_str.replace("_component", ""); let component_type = component_type_attr.replace('_', " "); // Make sure the attribute actually has inner tokens. If it doesn't, this means they forgot // entirely to specify a component name, and we want to give back a meaningful error that looks // correct when applied in the context of `#[configurable_component(...)]`. - if attr.tokens.is_empty() { + if attr.meta.require_list().is_err() { return Err(Error::new( attr.span(), format!( @@ -162,6 +162,10 @@ fn check_component_name_validity(component_name: &str) -> Result<(), String> { // In a nutshell, component names must contain only lowercase ASCII alphabetic characters, or // numbers, or underscores. + if component_name.is_empty() { + return Err("component name must be non-empty".to_string()); + } + // We only support ASCII names, so get that out of the way. if !component_name.is_ascii() { return Err("component names may only contain ASCII characters".to_string()); diff --git a/lib/vector-config-macros/src/configurable.rs b/lib/vector-config-macros/src/configurable.rs index 21c87804e34bf..b26e317b998f5 100644 --- a/lib/vector-config-macros/src/configurable.rs +++ b/lib/vector-config-macros/src/configurable.rs @@ -2,7 +2,7 @@ use proc_macro::TokenStream; use proc_macro2::Span; use quote::{quote, quote_spanned}; use syn::{ - parse_macro_input, parse_quote, spanned::Spanned, token::Colon2, DeriveInput, ExprPath, Ident, + parse_macro_input, parse_quote, spanned::Spanned, token::PathSep, DeriveInput, ExprPath, Ident, PathArguments, Type, }; use vector_config_common::validation::Validation; @@ -1058,7 +1058,7 @@ fn get_ty_for_expr_pos(ty: &syn::Type) -> syn::Type { let mut new_tp = tp.clone(); for segment in new_tp.path.segments.iter_mut() { if let PathArguments::AngleBracketed(ab) = &mut segment.arguments { - ab.colon2_token = Some(Colon2::default()); + ab.colon2_token = Some(PathSep::default()); } } diff --git a/lib/vector-config-macros/src/configurable_component.rs b/lib/vector-config-macros/src/configurable_component.rs index 37acc7703f24d..42d7813768aa6 100644 --- a/lib/vector-config-macros/src/configurable_component.rs +++ b/lib/vector-config-macros/src/configurable_component.rs @@ -1,10 +1,10 @@ -use darling::{Error, FromMeta}; +use darling::{ast::NestedMeta, Error, FromMeta}; use proc_macro::TokenStream; use proc_macro2::{Ident, Span}; use quote::{quote, quote_spanned}; use syn::{ parse_macro_input, parse_quote, parse_quote_spanned, punctuated::Punctuated, spanned::Spanned, - token::Comma, AttributeArgs, DeriveInput, Lit, LitStr, Meta, MetaList, NestedMeta, Path, + token::Comma, DeriveInput, Lit, LitStr, Meta, MetaList, Path, }; use vector_config_common::{ constants::ComponentType, human_friendly::generate_human_friendly_string, @@ -41,16 +41,19 @@ impl TypedComponent { /// If the meta list does not have a path that matches a known component type, `None` is /// returned. Otherwise, `Some(...)` is returned with a valid `TypedComponent`. fn from_meta_list(ml: &MetaList) -> Option { - let mut items = ml.nested.iter(); + let mut items = ml + .parse_args_with(Punctuated::::parse_terminated) + .unwrap_or_default() + .into_iter(); ComponentType::try_from(&ml.path) .ok() .map(|component_type| { let component_name = match items.next() { - Some(NestedMeta::Lit(Lit::Str(component_name))) => Some(component_name.clone()), + Some(NestedMeta::Lit(Lit::Str(component_name))) => Some(component_name), _ => None, }; let description = match items.next() { - Some(NestedMeta::Lit(Lit::Str(description))) => Some(description.clone()), + Some(NestedMeta::Lit(Lit::Str(description))) => Some(description), _ => None, }; Self { @@ -157,7 +160,7 @@ struct Options { } impl FromMeta for Options { - fn from_list(items: &[syn::NestedMeta]) -> darling::Result { + fn from_list(items: &[NestedMeta]) -> darling::Result { let mut typed_component = None; let mut no_ser = false; let mut no_deser = false; @@ -255,7 +258,10 @@ impl Options { } pub fn configurable_component_impl(args: TokenStream, item: TokenStream) -> TokenStream { - let args = parse_macro_input!(args as AttributeArgs); + let args: Vec = + parse_macro_input!(args with Punctuated::::parse_terminated) + .into_iter() + .collect(); let input = parse_macro_input!(item as DeriveInput); let options = match Options::from_list(&args) { diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index e1acd6593233c..d8258a0dcb7bd 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -16,14 +16,14 @@ chrono-tz = { version = "0.8.3", default-features = false } encoding_rs = { version = "0.8", default-features = false, features = ["alloc", "serde"] } indexmap = { version = "2.0", default-features = false, features = ["std"] } inventory = { version = "0.3" } -no-proxy = { version = "0.3.1", default-features = false, features = ["serialize"] } +no-proxy = { version = "0.3.1", default-features = false, features = ["serialize"] } num-traits = { version = "0.2.15", default-features = false } once_cell = { version = "1", default-features = false } serde = { version = "1.0", default-features = false } serde_json = { version = "1.0", default-features = false, features = ["std"] } -serde_with = { version = "2.3.2", default-features = false, features = ["std"] } -snafu = { version = "0.7.4", default-features = false } -toml = { version = "0.7.5", default-features = false } +serde_with = { version = "3.1.0", default-features = false, features = ["std"] } +snafu = { version = "0.7.5", default-features = false } +toml = { version = "0.7.6", default-features = false } tracing = { version = "0.1.34", default-features = false } url = { version = "2.4.0", default-features = false, features = ["serde"] } vrl.workspace = true @@ -32,4 +32,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] assert-json-diff = { version = "2", default-features = false } -serde_with = { version = "2.3.2", default-features = false, features = ["std", "macros"] } +serde_with = { version = "3.1.0", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-config/src/external/serde_with.rs b/lib/vector-config/src/external/serde_with.rs index 05de17e3bf605..30267e2253943 100644 --- a/lib/vector-config/src/external/serde_with.rs +++ b/lib/vector-config/src/external/serde_with.rs @@ -50,7 +50,7 @@ where impl Configurable for serde_with::DurationSeconds { fn referenceable_name() -> Option<&'static str> { // We're masking the type parameters here because we only deal with whole seconds via this - // version, and handle fractional seconds with `DurationSeconds`, which we + // version, and handle fractional seconds with `DurationSecondsWithFrac`, which we // expose as `serde_with::DurationFractionalSeconds`. Some("serde_with::DurationSeconds") } @@ -76,7 +76,7 @@ impl Configurable for serde_with::DurationSeconds { +impl Configurable for serde_with::DurationSecondsWithFrac { fn referenceable_name() -> Option<&'static str> { // We're masking the type parameters here because we only deal with fractional seconds via this // version, and handle whole seconds with `DurationSeconds`, which we diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 1395883640469..7500aaf4c5ae2 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -8,12 +8,12 @@ publish = false [dependencies] async-graphql = { version = "5.0.10", default-features = false, features = ["playground" ], optional = true } async-trait = { version = "0.1", default-features = false } -bitmask-enum = { version = "2.1.0", default-features = false } +bitmask-enum = { version = "2.2.1", default-features = false } bytes = { version = "1.4.0", default-features = false, features = ["serde"] } chrono = { version = "0.4.19", default-features = false, features = ["serde"] } crossbeam-utils = { version = "0.8.16", default-features = false } db-key = { version = "0.0.5", default-features = false, optional = true } -dyn-clone = { version = "1.0.11", default-features = false } +dyn-clone = { version = "1.0.12", default-features = false } enrichment = { path = "../enrichment", optional = true } enumflags2 = { version = "0.7.7", default-features = false } float_eq = { version = "1.0", default-features = false } @@ -24,40 +24,40 @@ http = { version = "0.2.9", default-features = false } hyper-proxy = { version = "0.9.1", default-features = false, features = ["openssl-tls"] } indexmap = { version = "~2.0.0", default-features = false, features = ["serde", "std"] } lookup = { package = "vector-lookup", path = "../vector-lookup" } -metrics = "0.21.0" +metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } -metrics-util = { version = "0.15.0", default-features = false, features = ["registry"] } +metrics-util = { version = "0.15.1", default-features = false, features = ["registry"] } mlua = { version = "0.8.9", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.2", default-features = false, features = ["serialize"] } once_cell = { version = "1.18", default-features = false } ordered-float = { version = "3.7.0", default-features = false } openssl = { version = "0.10.55", default-features = false, features = ["vendored"] } parking_lot = { version = "0.12.1", default-features = false } -pin-project = { version = "1.1.1", default-features = false } +pin-project = { version = "1.1.2", default-features = false } proptest = { version = "1.2", optional = true } prost-types = { version = "0.11", default-features = false } prost = { version = "0.11", default-features = false, features = ["std"] } quanta = { version = "0.11.1", default-features = false } -regex = { version = "1.8.4", default-features = false, features = ["std", "perf"] } +regex = { version = "1.9.1", default-features = false, features = ["std", "perf"] } ryu = { version = "1", default-features = false } -serde = { version = "1.0.164", default-features = false, features = ["derive", "rc"] } -serde_json = { version = "1.0.99", default-features = false } -serde_with = { version = "2.3.2", default-features = false, features = ["std", "macros"] } +serde = { version = "1.0.173", default-features = false, features = ["derive", "rc"] } +serde_json = { version = "1.0.103", default-features = false } +serde_with = { version = "3.1.0", default-features = false, features = ["std", "macros"] } smallvec = { version = "1", default-features = false, features = ["serde", "const_generics"] } -snafu = { version = "0.7.4", default-features = false } +snafu = { version = "0.7.5", default-features = false } socket2 = { version = "0.5.3", default-features = false } -tokio = { version = "1.29.0", default-features = false, features = ["net"] } +tokio = { version = "1.29.1", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } -toml = { version = "0.7.5", default-features = false } +toml = { version = "0.7.6", default-features = false } tonic = { version = "0.9", default-features = false, features = ["transport"] } tower = { version = "0.4", default-features = false, features = ["util"] } tracing = { version = "0.1.34", default-features = false } tracing-core = { version = "0.1.26", default-features = false } tracing-log = { version = "0.1.3", default-features = false } tracing-subscriber = { version = "0.3.17", default-features = false, features = ["std"] } -typetag = { version = "0.2.8", default-features = false } +typetag = { version = "0.2.10", default-features = false } twox-hash = { version = "1.6.3", default-features = false } url = { version = "2", default-features = false } vector-buffers = { path = "../vector-buffers", default-features = false } @@ -71,7 +71,7 @@ vrl.workspace = true security-framework = "2.9.1" [target.'cfg(windows)'.dependencies] -schannel = "0.1.21" +schannel = "0.1.22" [build-dependencies] prost-build = "0.11" @@ -86,7 +86,7 @@ quickcheck_macros = "1" proptest = "1.2" similar-asserts = "1.4.2" tokio-test = "0.4.2" -toml = { version = "0.7.5", default-features = false, features = ["parse"] } +toml = { version = "0.7.6", default-features = false, features = ["parse"] } ndarray = "0.15.6" ndarray-stats = "0.5.1" noisy_float = "0.2.0" diff --git a/lib/vector-core/src/config/log_schema.rs b/lib/vector-core/src/config/log_schema.rs index d54575f89f252..8b23b1be7b1c6 100644 --- a/lib/vector-core/src/config/log_schema.rs +++ b/lib/vector-core/src/config/log_schema.rs @@ -1,7 +1,8 @@ -use lookup::lookup_v2::{parse_target_path, OptionalValuePath}; -use lookup::{owned_value_path, OwnedTargetPath, OwnedValuePath}; +use lookup::lookup_v2::OptionalValuePath; +use lookup::{OwnedTargetPath, OwnedValuePath}; use once_cell::sync::{Lazy, OnceCell}; use vector_config::configurable_component; +use vrl::path::parse_target_path; static LOG_SCHEMA: OnceCell = OnceCell::new(); static LOG_SCHEMA_DEFAULT: Lazy = Lazy::new(LogSchema::default); @@ -54,13 +55,13 @@ pub struct LogSchema { /// This field will generally represent a real host, or container, that generated the message, /// but is somewhat source-dependent. #[serde(default = "LogSchema::default_host_key")] - host_key: String, + host_key: OptionalValuePath, /// The name of the event field to set the source identifier in. /// /// This field will be set by the Vector source that the event was created in. #[serde(default = "LogSchema::default_source_type_key")] - source_type_key: String, + source_type_key: OptionalValuePath, /// The name of the event field to set the event metadata in. /// @@ -88,17 +89,15 @@ impl LogSchema { } fn default_timestamp_key() -> OptionalValuePath { - OptionalValuePath { - path: Some(owned_value_path!("timestamp")), - } + OptionalValuePath::new("timestamp") } - fn default_host_key() -> String { - String::from("host") + fn default_host_key() -> OptionalValuePath { + OptionalValuePath::new("host") } - fn default_source_type_key() -> String { - String::from("source_type") + fn default_source_type_key() -> OptionalValuePath { + OptionalValuePath::new("source_type") } fn default_metadata_key() -> String { @@ -122,12 +121,12 @@ impl LogSchema { self.timestamp_key.path.as_ref() } - pub fn host_key(&self) -> &str { - &self.host_key + pub fn host_key(&self) -> Option<&OwnedValuePath> { + self.host_key.path.as_ref() } - pub fn source_type_key(&self) -> &str { - &self.source_type_key + pub fn source_type_key(&self) -> Option<&OwnedValuePath> { + self.source_type_key.path.as_ref() } pub fn metadata_key(&self) -> &str { @@ -142,12 +141,12 @@ impl LogSchema { self.timestamp_key = OptionalValuePath { path: v }; } - pub fn set_host_key(&mut self, v: String) { - self.host_key = v; + pub fn set_host_key(&mut self, path: Option) { + self.host_key = OptionalValuePath { path }; } - pub fn set_source_type_key(&mut self, v: String) { - self.source_type_key = v; + pub fn set_source_type_key(&mut self, path: Option) { + self.source_type_key = OptionalValuePath { path }; } pub fn set_metadata_key(&mut self, v: String) { @@ -170,7 +169,7 @@ impl LogSchema { { errors.push("conflicting values for 'log_schema.host_key' found".to_owned()); } else { - self.set_host_key(other.host_key().to_string()); + self.set_host_key(other.host_key().cloned()); } if self.message_key() != LOG_SCHEMA_DEFAULT.message_key() && self.message_key() != other.message_key() @@ -191,7 +190,7 @@ impl LogSchema { { errors.push("conflicting values for 'log_schema.source_type_key' found".to_owned()); } else { - self.set_source_type_key(other.source_type_key().to_string()); + self.set_source_type_key(other.source_type_key().cloned()); } if self.metadata_key() != LOG_SCHEMA_DEFAULT.metadata_key() && self.metadata_key() != other.metadata_key() diff --git a/lib/vector-core/src/config/mod.rs b/lib/vector-core/src/config/mod.rs index 71786155d1d8f..ed3dde2bae112 100644 --- a/lib/vector-core/src/config/mod.rs +++ b/lib/vector-core/src/config/mod.rs @@ -476,7 +476,7 @@ impl LogNamespace { ) { self.insert_vector_metadata( log, - Some(log_schema().source_type_key()), + log_schema().source_type_key(), path!("source_type"), Bytes::from_static(source_name.as_bytes()), ); @@ -551,14 +551,15 @@ mod test { use chrono::Utc; use lookup::{event_path, owned_value_path, OwnedTargetPath}; use vector_common::btreemap; + use vrl::path::OwnedValuePath; use vrl::value::Kind; #[test] fn test_insert_standard_vector_source_metadata() { - let nested_path = "a.b.c.d"; + let nested_path = "a.b.c.d".to_string(); let mut schema = LogSchema::default(); - schema.set_source_type_key(nested_path.to_owned()); + schema.set_source_type_key(Some(OwnedValuePath::try_from(nested_path).unwrap())); init_log_schema(schema, false); let namespace = LogNamespace::Legacy; diff --git a/lib/vector-core/src/event/log_event.rs b/lib/vector-core/src/event/log_event.rs index e5755f12d7e66..87da44d6040c6 100644 --- a/lib/vector-core/src/event/log_event.rs +++ b/lib/vector-core/src/event/log_event.rs @@ -292,11 +292,19 @@ impl LogEvent { } } + /// Retrieves the value of a field based on it's meaning. + /// This will first check if the value has previously been dropped. It is worth being + /// aware that if the field has been dropped and then some how readded, we still fetch + /// the dropped value here. pub fn get_by_meaning(&self, meaning: impl AsRef) -> Option<&Value> { - self.metadata() - .schema_definition() - .meaning_path(meaning.as_ref()) - .and_then(|path| self.get(path)) + if let Some(dropped) = self.metadata().dropped_field(&meaning) { + Some(dropped) + } else { + self.metadata() + .schema_definition() + .meaning_path(meaning.as_ref()) + .and_then(|path| self.get(path)) + } } // TODO(Jean): Once the event API uses `Lookup`, the allocation here can be removed. @@ -458,7 +466,7 @@ impl LogEvent { pub fn host_path(&self) -> Option { match self.namespace() { LogNamespace::Vector => self.find_key_by_meaning("host"), - LogNamespace::Legacy => Some(log_schema().host_key().to_owned()), + LogNamespace::Legacy => log_schema().host_key().map(ToString::to_string), } } @@ -466,10 +474,10 @@ impl LogEvent { /// or from the `source_type` key set on the "Global Log Schema" (Legacy namespace). // TODO: This can eventually return a `&TargetOwnedPath` once Semantic meaning and the // "Global Log Schema" are updated to the new path lookup code - pub fn source_type_path(&self) -> &'static str { + pub fn source_type_path(&self) -> Option { match self.namespace() { - LogNamespace::Vector => "%vector.source_type", - LogNamespace::Legacy => log_schema().source_type_key(), + LogNamespace::Vector => Some("%vector.source_type".to_string()), + LogNamespace::Legacy => log_schema().source_type_key().map(ToString::to_string), } } @@ -505,7 +513,9 @@ impl LogEvent { pub fn get_host(&self) -> Option<&Value> { match self.namespace() { LogNamespace::Vector => self.get_by_meaning("host"), - LogNamespace::Legacy => self.get((PathPrefix::Event, log_schema().host_key())), + LogNamespace::Legacy => log_schema() + .host_key() + .and_then(|key| self.get((PathPrefix::Event, key))), } } @@ -514,7 +524,9 @@ impl LogEvent { pub fn get_source_type(&self) -> Option<&Value> { match self.namespace() { LogNamespace::Vector => self.get(metadata_path!("vector", "source_type")), - LogNamespace::Legacy => self.get((PathPrefix::Event, log_schema().source_type_key())), + LogNamespace::Legacy => log_schema() + .source_type_key() + .and_then(|key| self.get((PathPrefix::Event, key))), } } } diff --git a/lib/vector-core/src/event/metadata.rs b/lib/vector-core/src/event/metadata.rs index d86884be7582c..f577147e11ded 100644 --- a/lib/vector-core/src/event/metadata.rs +++ b/lib/vector-core/src/event/metadata.rs @@ -46,6 +46,14 @@ pub struct EventMetadata { /// TODO(Jean): must not skip serialization to track schemas across restarts. #[serde(default = "default_schema_definition", skip)] schema_definition: Arc, + + /// A store of values that may be dropped during the encoding process but may be needed + /// later on. The map is indexed by meaning. + /// Currently this is just used for the `service`. If the service field is dropped by `only_fields` + /// we need to ensure it is still available later on for emitting metrics tagged by the service. + /// This field could almost be keyed by `&'static str`, but because it needs to be deserializable + /// we have to use `String`. + dropped_fields: BTreeMap, } fn default_metadata_value() -> Value { @@ -123,6 +131,19 @@ impl EventMetadata { pub fn set_splunk_hec_token(&mut self, secret: Arc) { self.secrets.insert(SPLUNK_HEC_TOKEN, secret); } + + /// Adds the value to the dropped fields list. + /// There is currently no way to remove a field from this list, so if a field is dropped + /// and then the field is re-added with a new value - the dropped value will still be + /// retrieved. + pub fn add_dropped_field(&mut self, meaning: String, value: Value) { + self.dropped_fields.insert(meaning, value); + } + + /// Fetches the dropped field by meaning. + pub fn dropped_field(&self, meaning: impl AsRef) -> Option<&Value> { + self.dropped_fields.get(meaning.as_ref()) + } } impl Default for EventMetadata { @@ -134,6 +155,7 @@ impl Default for EventMetadata { schema_definition: default_schema_definition(), source_id: None, upstream_id: None, + dropped_fields: BTreeMap::new(), } } } diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index a3c5afc034cb4..421b0b91a043a 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; use crate::config::{log_schema, LegacyKey, LogNamespace}; -use lookup::lookup_v2::{parse_value_path, TargetPath}; +use lookup::lookup_v2::TargetPath; use lookup::{owned_value_path, OwnedTargetPath, OwnedValuePath, PathPrefix}; use vrl::value::{kind::Collection, Kind}; @@ -144,9 +144,7 @@ impl Definition { #[must_use] pub fn with_standard_vector_source_metadata(self) -> Self { self.with_vector_metadata( - parse_value_path(log_schema().source_type_key()) - .ok() - .as_ref(), + log_schema().source_type_key(), &owned_value_path!("source_type"), Kind::bytes(), None, diff --git a/lib/vector-core/src/schema/meaning.rs b/lib/vector-core/src/schema/meaning.rs new file mode 100644 index 0000000000000..ab766b0986924 --- /dev/null +++ b/lib/vector-core/src/schema/meaning.rs @@ -0,0 +1,17 @@ +//! Constants for commonly used semantic meanings. + +/// The service typically represents the application that generated the event. +pub const SERVICE: &str = "service"; + +/// The main text message of the event. +pub const MESSAGE: &str = "message"; + +/// The main timestamp of the event. +pub const TIMESTAMP: &str = "timestamp"; + +/// The hostname of the machine where the event was generated. +pub const HOST: &str = "host"; + +pub const SOURCE: &str = "source"; +pub const SEVERITY: &str = "severity"; +pub const TRACE_ID: &str = "trace_id"; diff --git a/lib/vector-core/src/schema/mod.rs b/lib/vector-core/src/schema/mod.rs index 96f6d99442fa8..2d1c01b8d281f 100644 --- a/lib/vector-core/src/schema/mod.rs +++ b/lib/vector-core/src/schema/mod.rs @@ -1,4 +1,5 @@ mod definition; +pub mod meaning; mod requirement; pub use definition::Definition; diff --git a/lib/vector-core/src/stream/driver.rs b/lib/vector-core/src/stream/driver.rs index 6ff23014c96d5..1a1ce79e1b27a 100644 --- a/lib/vector-core/src/stream/driver.rs +++ b/lib/vector-core/src/stream/driver.rs @@ -99,7 +99,7 @@ where pin!(batched_input); let bytes_sent = protocol.map(|protocol| register(BytesSent { protocol })); - let events_sent = RegisteredEventCache::default(); + let events_sent = RegisteredEventCache::new(()); loop { // Core behavior of the loop: @@ -203,7 +203,7 @@ where finalizers: EventFinalizers, event_count: usize, bytes_sent: &Option>, - events_sent: &RegisteredEventCache, + events_sent: &RegisteredEventCache<(), TaggedEventsSent>, ) { match result { Err(error) => { diff --git a/lib/vector-lookup/Cargo.toml b/lib/vector-lookup/Cargo.toml index a159f31561d0f..0edc385a6f165 100644 --- a/lib/vector-lookup/Cargo.toml +++ b/lib/vector-lookup/Cargo.toml @@ -7,7 +7,7 @@ publish = false license = "MPL-2.0" [dependencies] -serde = { version = "1.0.164", default-features = false, features = ["derive", "alloc"] } +serde = { version = "1.0.173", default-features = false, features = ["derive", "alloc"] } vector-config = { path = "../vector-config" } vector-config-macros = { path = "../vector-config-macros" } vrl.workspace = true diff --git a/lib/vector-lookup/src/lookup_v2/optional_path.rs b/lib/vector-lookup/src/lookup_v2/optional_path.rs index 9328aa8a2f138..5bbfe4ad082d4 100644 --- a/lib/vector-lookup/src/lookup_v2/optional_path.rs +++ b/lib/vector-lookup/src/lookup_v2/optional_path.rs @@ -1,4 +1,5 @@ use vector_config::configurable_component; +use vrl::owned_value_path; use crate::lookup_v2::PathParseError; use crate::{OwnedTargetPath, OwnedValuePath}; @@ -56,6 +57,12 @@ impl OptionalValuePath { pub fn none() -> Self { Self { path: None } } + + pub fn new(path: &str) -> Self { + Self { + path: Some(owned_value_path!(path)), + } + } } impl TryFrom for OptionalValuePath { @@ -84,3 +91,11 @@ impl From for OptionalValuePath { Self { path: Some(path) } } } + +impl From> for OptionalValuePath { + fn from(value: Option) -> Self { + value.map_or(OptionalValuePath::none(), |path| { + OptionalValuePath::from(path) + }) + } +} diff --git a/lib/vector-vrl/web-playground/README.md b/lib/vector-vrl/web-playground/README.md index 9354b77321205..11faa7a1c172a 100644 --- a/lib/vector-vrl/web-playground/README.md +++ b/lib/vector-vrl/web-playground/README.md @@ -13,13 +13,13 @@ To build the project we need to use `wasm-pack`. This compiles our Rust code to WebAssembly which can then be used within the browser. Install it by running: ```shell -cargo install --version 0.10.3 wasm-pack +cargo install --locked --version 0.10.3 wasm-pack ``` After installing `wasm-pack` we must compile our project by running: ```shell -wasm-pack build --target web --out-dir public +wasm-pack build --target web --out-dir public/pkg ``` Notice the `public` directory was populated with `.wasm`, and `.js`, @@ -30,6 +30,8 @@ For more information on Rust and WebAssembly please visit [the mozilla docs][mozilla-wasm-rust-docs] or [the Rust book wasm chapter][rust-book-wasm] +## Run locally + The `src/lib.rs` file is the entry point of the `web-playground` crate. This file is necessary so we can use the `run_vrl()` function in the browser. Notice our `index.html` imports the VRL wasm module from `./vrl_web_playground.js` @@ -39,6 +41,7 @@ the web browser console. To see this in action we host `index.html` locally, for example by running: ```shell +cd public python3 -m http.server ``` @@ -65,7 +68,42 @@ but they will either error (enrichment functions) or abort (all the others) at r - `get_enrichment_table_record` Functions from VRL stdlib that are currently not supported can be found -with this [issue filter][vrl-wasm-unsupported-filter] +with this [issue filter][vrl-wasm-unsupported-filter]. + +### macOS Troubleshooting + +If you are getting compilation errors on macOS here are some things to check: + +```shell +xcode-select -p +# Example: '/Library/Developer/CommandLineTools +# To change this use: xcode-select -s +``` + +You can clean and reinstall with: + +```shell +rm -rf /Library/Developer/CommandLineTools # might require sudo elevation +xcode-select --install +``` + +Check your `llvm` installation and ensure that there are no conflicting installations. Check that the following command returns the expected version: + +```shell +clang --version +# Example: +# Homebrew clang version 16.0.6 +# Target: arm64-apple-darwin22.5.0 +# Thread model: posix +# InstalledDir: /opt/homebrew/opt/llvm/bin +``` + +The output of the following command should contain `WebAssembly`: + +```shell +llvm-config --targets-built # WebAssembly should be in the results +# Example: AArch64 WebAssembly +``` ## Examples diff --git a/lib/vector-vrl/web-playground/netlify.toml b/lib/vector-vrl/web-playground/netlify.toml index 3c9447d330e58..824b50e04972f 100644 --- a/lib/vector-vrl/web-playground/netlify.toml +++ b/lib/vector-vrl/web-playground/netlify.toml @@ -11,7 +11,7 @@ publish = "public/" # Default build command. - command = "bash ../../../scripts/ensure-wasm-pack-installed.sh && wasm-pack build --target web --out-dir public" + command = "bash ../../../scripts/ensure-wasm-pack-installed.sh && wasm-pack build --target web --out-dir public/pkg" # Ignore everything except the base directory and changes to vector/lib/ ignore = "git diff --quiet $CACHED_COMMIT_REF $COMMIT_REF . ../../../lib" diff --git a/lib/vector-vrl/web-playground/public/.gitignore b/lib/vector-vrl/web-playground/public/.gitignore deleted file mode 100644 index de695ce7d35fc..0000000000000 --- a/lib/vector-vrl/web-playground/public/.gitignore +++ /dev/null @@ -1 +0,0 @@ -vector_vrl_web_playground** diff --git a/lib/vector-vrl/web-playground/public/README.md b/lib/vector-vrl/web-playground/public/README.md deleted file mode 100644 index b042c0f4a26db..0000000000000 --- a/lib/vector-vrl/web-playground/public/README.md +++ /dev/null @@ -1,146 +0,0 @@ -# VRL WASM Web Playground - -This directory houses the exposed VRL function to WASM `run_vrl()` used to -power [Vector Remap Language Playground][vrl-playground], or **VRL Playground** -for short. Although there is already a [local REPL][vrl-repl] supported for -use within the terminal, this playground will support running VRL in the web -browser and test input via uploading event files, or specifying an event via -a text input field. - -## Setup - -To build the project we need to use `wasm-pack`. This compiles our Rust code -to WebAssembly which can then be used within the browser. Install it by running: - -```shell -cargo install --version 0.10.3 wasm-pack -``` - -After installing `wasm-pack` we must compile our project by running: - -```shell -wasm-pack build --target web --out-dir public -``` - -Notice the `public` directory was populated with `.wasm`, and `.js`, -files these will be used by our `index.html` to run the `run_vrl()` -function originally written in Rust. - -For more information on Rust and WebAssembly please visit -[the mozilla docs][mozilla-wasm-rust-docs] or -[the Rust book wasm chapter][rust-book-wasm] - -The `src/lib.rs` file is the entry point of the `web-playground` crate. -This file is necessary so we can use the `run_vrl()` function in the browser. -Notice our `index.html` imports the VRL wasm module from `./vrl_web_playground.js` -and sets the `window.run_vrl` function so that we can test VRL within -the web browser console. - -To see this in action we host `index.html` locally, for example by running: - -```shell -python3 -m http.server -``` - -Remember to be in the `public` directory where index.html is located for the -relative paths specified in `index.html` to work. -We should also be able to open the `index.html` file in chrome, or use Live Server -in VSCode to see `index.html` working. - -## Support - -Some functions of VRL are not supported or don't work as expected at the -moment due to WASM limitations with some Rust crates, in -the future we will modify the functions so that they are supported. - -List of functions that aren't supported at the moment: - -- `log()` -- `decrypt()` -- `encrypt()` -- `get_hostname()` -- `parse_groks()` -- `random_bytes()` -- `reverse_dns()` - -Functions from VRL stdlib that are currently not supported can be found -with this [issue filter][vrl-wasm-unsupported-filter] - -## Examples - -### React - -For now, you can use an old npm-published version of vrl-web-playground, -please note that this was done for testing purposes and in the future we -will likely release this automatically upon each version release of Vector, -probably under a different package name. - -Use this dependency in `package.json` - -```json -"dependencies": { - "vrl-web-playground": "0.1.0" -} -``` - -Example import and usage in a React component - -```javascript -import init, { run_vrl } from 'vrl-web-playground'; - -export function VectorExecuteButton() { - let vrlInput = {}; - try { - vrlInput = { - program: '.something = "added by vrl!"\n.message = "modified by vrl!"', - event: JSON.parse('{message: "log message here"}'), - }; - } catch (error) { - console.log('error parsing the event contents as JSON object'); - } - - return ( -