From 78a16d7c205c9665bf9a3bd5e7d0955357b63c4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:04:42 +0000 Subject: [PATCH 01/22] chore(deps): bump serde_with from 2.0.0 to 2.0.1 (#14367) Bumps [serde_with](https://github.com/jonasbb/serde_with) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/jonasbb/serde_with/releases) - [Commits](https://github.com/jonasbb/serde_with/compare/v2.0.0...v2.0.1) --- updated-dependencies: - dependency-name: serde_with dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- lib/vector-config/Cargo.toml | 4 ++-- lib/vector-core/Cargo.toml | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 405fbc8c4287f..a2276e2276d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7054,9 +7054,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89df7a26519371a3cce44fbb914c2819c84d9b897890987fa3ab096491cc0ea8" +checksum = "368f2d60d049ea019a84dcd6687b0d1e0030fe663ae105039bdf967ed5e6a9a7" dependencies = [ "base64", "chrono", @@ -7064,7 +7064,7 @@ dependencies = [ "indexmap", "serde", "serde_json", - "serde_with_macros 2.0.0", + "serde_with_macros 2.0.1", "time", ] @@ -7082,9 +7082,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de337f322382fcdfbb21a014f7c224ee041a23785651db67b9827403178f698f" +checksum = "1ccadfacf6cf10faad22bbadf55986bdd0856edfb5d9210aa1dcf1f516e84e93" dependencies = [ "darling 0.14.1", "proc-macro2", @@ -7709,9 +7709,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" dependencies = [ "itoa 1.0.1", "libc", @@ -8842,7 +8842,7 @@ dependencies = [ "serde-toml-merge", "serde_bytes", "serde_json", - "serde_with 2.0.0", + "serde_with 2.0.1", "serde_yaml 0.9.13", "sha2 0.10.6", "smallvec", @@ -9005,7 +9005,7 @@ dependencies = [ "schemars", "serde", "serde_json", - "serde_with 2.0.0", + "serde_with 2.0.1", "snafu", "toml", "url", @@ -9090,7 +9090,7 @@ dependencies = [ "security-framework", "serde", "serde_json", - "serde_with 2.0.0", + "serde_with 2.0.1", "snafu", "socket2", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 88049ca8b84f7..86923a34e14dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -183,7 +183,7 @@ serde = { version = "1.0.145", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } serde_bytes = { version = "0.11.7", default-features = false, features = ["std"], optional = true } serde_json = { version = "1.0.85", default-features = false, features = ["raw_value"] } -serde_with = { version = "2.0.0", default-features = false, features = ["macros", "std"], optional = true } +serde_with = { version = "2.0.1", default-features = false, features = ["macros", "std"], optional = true } serde_yaml = { version = "0.9.13", default-features = false } # Messagepack diff --git a/lib/vector-config/Cargo.toml b/lib/vector-config/Cargo.toml index 83069cfe4ec42..a22bb77e8b562 100644 --- a/lib/vector-config/Cargo.toml +++ b/lib/vector-config/Cargo.toml @@ -16,7 +16,7 @@ num-traits = { version = "0.2.15", default-features = false } schemars = { version = "0.8.10", default-features = true, features = ["preserve_order"] } serde = { version = "1.0", default-features = false } serde_json = { version = "1.0.85", default-features = false } -serde_with = { version = "2.0.0", default-features = false, features = ["std"] } +serde_with = { version = "2.0.1", default-features = false, features = ["std"] } snafu = { version = "0.7.1", default-features = false } toml = { version = "0.5.9", default-features = false } url = { version = "2.3.1", default-features = false, features = ["serde"] } @@ -24,4 +24,4 @@ vector-config-common = { path = "../vector-config-common" } vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] -serde_with = { version = "2.0.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "2.0.1", default-features = false, features = ["std", "macros"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index f7c4a3faef763..fec4f8e7f0712 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -41,7 +41,7 @@ quanta = { version = "0.10.1", default-features = false } regex = { version = "1.6.0", default-features = false, features = ["std", "perf"] } serde = { version = "1.0.145", default-features = false, features = ["derive", "rc"] } serde_json = { version = "1.0.85", default-features = false } -serde_with = { version = "2.0.0", default-features = false, features = ["std", "macros"] } +serde_with = { version = "2.0.1", default-features = false, features = ["std", "macros"] } snafu = { version = "0.7.1", default-features = false } socket2 = { version = "0.4.7", default-features = false } tokio = { version = "1.20.1", default-features = false, features = ["net"] } From 1b24559f0abaf9879a2c03f98276a7ad75e1aff3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:04:48 +0000 Subject: [PATCH 02/22] chore(deps): bump itertools from 0.10.4 to 0.10.5 (#14566) Bumps [itertools](https://github.com/rust-itertools/itertools) from 0.10.4 to 0.10.5. - [Release notes](https://github.com/rust-itertools/itertools/releases) - [Changelog](https://github.com/rust-itertools/itertools/blob/master/CHANGELOG.md) - [Commits](https://github.com/rust-itertools/itertools/commits) --- updated-dependencies: - dependency-name: itertools dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- lib/datadog/search-syntax/Cargo.toml | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2276e2276d8e..0467e8184b7d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1920,7 +1920,7 @@ dependencies = [ "clap 3.2.22", "criterion-plot", "futures 0.3.24", - "itertools 0.10.4", + "itertools 0.10.5", "lazy_static", "num-traits", "oorandom", @@ -1942,7 +1942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" dependencies = [ "cast", - "itertools 0.10.4", + "itertools 0.10.5", ] [[package]] @@ -2246,7 +2246,7 @@ dependencies = [ name = "datadog-search-syntax" version = "0.1.0" dependencies = [ - "itertools 0.10.4", + "itertools 0.10.5", "once_cell", "pest", "pest_derive", @@ -3793,9 +3793,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8bf247779e67a9082a4790b45e71ac7cfd1321331a5c856a74a9faebdab78d0" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -4085,7 +4085,7 @@ dependencies = [ "bit-set", "diff", "ena", - "itertools 0.10.4", + "itertools 0.10.5", "lalrpop-util", "petgraph", "pico-args", @@ -4713,7 +4713,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af5a8477ac96877b5bd1fd67e0c28736c12943aba24eda92b127e036b0c8f400" dependencies = [ "indexmap", - "itertools 0.10.4", + "itertools 0.10.5", "ndarray", "noisy_float", "num-integer", @@ -5709,7 +5709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5aab5be6e4732b473071984b3164dbbfb7a3674d30ea5ff44410b6bcd960c3c" dependencies = [ "difflib", - "itertools 0.10.4", + "itertools 0.10.5", "predicates-core", ] @@ -5909,7 +5909,7 @@ dependencies = [ "cfg-if", "cmake", "heck 0.4.0", - "itertools 0.10.4", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -5929,7 +5929,7 @@ checksum = "7f835c582e6bd972ba8347313300219fed5bfa52caf175298d860b61ff6069bb" dependencies = [ "bytes 1.2.1", "heck 0.4.0", - "itertools 0.10.4", + "itertools 0.10.5", "lazy_static", "log", "multimap", @@ -5948,7 +5948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" dependencies = [ "anyhow", - "itertools 0.10.4", + "itertools 0.10.5", "proc-macro2", "quote", "syn", @@ -5961,7 +5961,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7345d5f0e08c0536d7ac7229952590239e77abf0a0100a1b1d890add6ea96364" dependencies = [ "anyhow", - "itertools 0.10.4", + "itertools 0.10.5", "proc-macro2", "quote", "syn", @@ -8786,7 +8786,7 @@ dependencies = [ "indoc", "infer 0.9.0", "inventory", - "itertools 0.10.4", + "itertools 0.10.5", "jsonschema", "k8s-openapi", "kube", diff --git a/Cargo.toml b/Cargo.toml index 86923a34e14dd..2b4874bd3e385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -204,7 +204,7 @@ lapin = { version = "2.1.1", default-features = false, optional = true } # API async-graphql = { version = "4.0.13", default-features = false, optional = true, features = ["chrono"] } async-graphql-warp = { version = "4.0.13", default-features = false, optional = true } -itertools = { version = "0.10.4", default-features = false, optional = true } +itertools = { version = "0.10.5", default-features = false, optional = true } # API client crossterm = { version = "0.25.0", default-features = false, features = ["event-stream"], optional = true } diff --git a/lib/datadog/search-syntax/Cargo.toml b/lib/datadog/search-syntax/Cargo.toml index 5fed3d39006bb..d268f3d7fa882 100644 --- a/lib/datadog/search-syntax/Cargo.toml +++ b/lib/datadog/search-syntax/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" publish = false [dependencies] -itertools = { version = "0.10.4", default-features = false } +itertools = { version = "0.10.5", default-features = false } once_cell = { version = "1.15", default-features = false, features = ["std"] } pest = { version = "2.3.1", default-features = false, features = ["std"] } pest_derive = { version = "2.3.1", default-features = false, features = ["std"] } From 24c5739887c2cc8718f62fce21297ee8c19c9918 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:12:46 +0000 Subject: [PATCH 03/22] chore(deps): bump hdrhistogram from 7.5.1 to 7.5.2 (#14571) Bumps [hdrhistogram](https://github.com/HdrHistogram/HdrHistogram_rust) from 7.5.1 to 7.5.2. - [Release notes](https://github.com/HdrHistogram/HdrHistogram_rust/releases) - [Changelog](https://github.com/HdrHistogram/HdrHistogram_rust/blob/main/CHANGELOG.md) - [Commits](https://github.com/HdrHistogram/HdrHistogram_rust/compare/v7.5.1...v7.5.2) --- updated-dependencies: - dependency-name: hdrhistogram dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lib/vector-buffers/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0467e8184b7d2..2095a516d630f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3195,9 +3195,9 @@ dependencies = [ [[package]] name = "hdrhistogram" -version = "7.5.1" +version = "7.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea9fe3952d32674a14e0975009a3547af9ea364995b5ec1add2e23c2ae523ab" +checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" dependencies = [ "base64", "byteorder", diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index 0816c9d5b06a0..ef8afdd1a03c8 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -38,7 +38,7 @@ vector-common = { path = "../vector-common", default-features = false, features clap = "3.2.22" criterion = { version = "0.4", features = ["html_reports", "async_tokio"] } crossbeam-queue = "0.3.6" -hdrhistogram = "7.5.1" +hdrhistogram = "7.5.2" metrics-tracing-context = { version = "0.12.0", default-features = false } metrics-util = { version = "0.14.0", default-features = false, features = ["debugging"] } once_cell = "1.15" From 641ad2b11a1fc4094f85f9bad42be28cf3da3f71 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:19:15 +0000 Subject: [PATCH 04/22] chore(deps): bump env_logger from 0.9.0 to 0.9.1 (#14574) Bumps [env_logger](https://github.com/env-logger-rs/env_logger) from 0.9.0 to 0.9.1. - [Release notes](https://github.com/env-logger-rs/env_logger/releases) - [Changelog](https://github.com/env-logger-rs/env_logger/blob/main/CHANGELOG.md) - [Commits](https://github.com/env-logger-rs/env_logger/compare/v0.9.0...v0.9.1) --- updated-dependencies: - dependency-name: env_logger dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2095a516d630f..b3b26445c7d64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2588,9 +2588,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2cf0344971ee6c64c31be0d530793fba457d322dfec2810c453d0ef228f9c3" +checksum = "c90bf5f19754d10198ccb95b70664fc925bd1fc090a0fd9a6ebc54acc8cd6272" dependencies = [ "atty", "humantime", @@ -3909,7 +3909,7 @@ dependencies = [ name = "k8s-e2e-tests" version = "0.1.0" dependencies = [ - "env_logger 0.9.0", + "env_logger 0.9.1", "futures 0.3.24", "indoc", "k8s-openapi", From 6b3fe88450919775b4b631d629841e6fbde09794 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 16:22:42 +0000 Subject: [PATCH 05/22] chore(deps): bump thiserror from 1.0.35 to 1.0.36 (#14564) Bumps [thiserror](https://github.com/dtolnay/thiserror) from 1.0.35 to 1.0.36. - [Release notes](https://github.com/dtolnay/thiserror/releases) - [Commits](https://github.com/dtolnay/thiserror/compare/1.0.35...1.0.36) --- updated-dependencies: - dependency-name: thiserror dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3b26445c7d64..cb7be65f0d1f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7659,18 +7659,18 @@ checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" [[package]] name = "thiserror" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" +checksum = "0a99cb8c4b9a8ef0e7907cd3b617cc8dc04d571c4e73c8ae403d80ac160bb122" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" +checksum = "3a891860d3c8d66fec8e73ddb3765f90082374dbaaa833407b904a94f1a7eb43" dependencies = [ "proc-macro2", "quote", From f5b91036c835ae78e43f53af08ff737004d89fb6 Mon Sep 17 00:00:00 2001 From: Alexander Zaitsev Date: Mon, 26 Sep 2022 19:33:44 +0300 Subject: [PATCH 06/22] fix(clickhouse): fix URI configuration (#14557) - fix URI configuration for Clickhouse sink. Now it fills missing parts with some predefined defaults Tested: - Local tests --- src/sinks/clickhouse.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sinks/clickhouse.rs b/src/sinks/clickhouse.rs index 1f73c3a6fa03c..c8b38976ead97 100644 --- a/src/sinks/clickhouse.rs +++ b/src/sinks/clickhouse.rs @@ -150,7 +150,7 @@ impl HttpSink for ClickhouseConfig { }; let uri = set_uri_query( - &self.endpoint.uri, + &self.endpoint.with_default_parts().uri, database, &self.table, self.skip_unknown_fields, @@ -175,7 +175,7 @@ impl HttpSink for ClickhouseConfig { async fn healthcheck(client: HttpClient, config: ClickhouseConfig) -> crate::Result<()> { // TODO: check if table exists? - let uri = format!("{}/?query=SELECT%201", config.endpoint); + let uri = format!("{}/?query=SELECT%201", config.endpoint.with_default_parts()); let mut request = Request::get(uri).body(Body::empty()).unwrap(); if let Some(auth) = &config.auth { From a4ed79736d12f89b36f039741a2cafa25e8b0a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=A9mie=20Drouet?= Date: Mon, 26 Sep 2022 18:59:04 +0200 Subject: [PATCH 07/22] feat(enterprise): use vector version when computing configuration hash (#14518) * rename config version field to hash Signed-off-by: Jeremie Drouet * add vector version to configuration hash Signed-off-by: Jeremie Drouet * change the way we get the version of vector Signed-off-by: Jeremie Drouet * fix tests to match hash Signed-off-by: Jeremie Drouet Signed-off-by: Jeremie Drouet --- src/config/builder.rs | 20 ++++++++++++-------- src/config/compiler.rs | 6 +++--- src/config/enterprise.rs | 2 +- src/config/mod.rs | 2 +- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/config/builder.rs b/src/config/builder.rs index 6c3acfac4f27d..eff5cdef35421 100644 --- a/src/config/builder.rs +++ b/src/config/builder.rs @@ -82,6 +82,7 @@ pub struct ConfigBuilder { #[cfg(feature = "enterprise")] #[derive(::serde::Serialize)] struct ConfigBuilderHash<'a> { + version: String, #[cfg(feature = "api")] api: &'a api::Options, schema: &'a schema::Options, @@ -160,6 +161,7 @@ fn sort_json_value(value: &mut Value) { impl<'a> From<&'a ConfigBuilder> for ConfigBuilderHash<'a> { fn from(value: &'a ConfigBuilder) -> Self { ConfigBuilderHash { + version: crate::get_version(), #[cfg(feature = "api")] api: &value.api, schema: &value.schema, @@ -192,7 +194,7 @@ impl From for ConfigBuilder { transforms, tests, secret, - version: _, + hash: _, expansions: _, } = config; @@ -430,6 +432,7 @@ mod tests { "sources", "tests", "transforms", + "version", ]; let builder = ConfigBuilder::default(); @@ -448,14 +451,15 @@ mod tests { } #[test] - /// If this hash changes, it means either the `ConfigBuilder` has changed what it - /// serializes, or the implementation of `serde_json` has changed. If this test fails, we - /// should ideally be able to fix so that the original hash passes! + /// If this hash changes, it means either the version of Vector has changed (here it's fixed), + /// the `ConfigBuilder` has changed what it serializes, or the implementation of `serde_json` has changed. + /// If this test fails, we should ideally be able to fix so that the original hash passes! fn version_hash_match() { - assert_eq!( - "bc0825487e137ee1d1fc76c616795d041c4825b4ca5a7236455ea4515238885c", - ConfigBuilder::default().sha256_hash() - ); + let expected_hash = "6c98bea9d9e2f3133e2d39ba04592d17f96340a9bc4c8d697b09f5af388a76bd"; + let builder = ConfigBuilder::default(); + let mut hash_builder = ConfigBuilderHash::from(&builder); + hash_builder.version = "1.2.3".into(); + assert_eq!(expected_hash, hash_builder.into_hash()); } #[test] diff --git a/src/config/compiler.rs b/src/config/compiler.rs index 380f2951ae8b4..bce4e2fe228a3 100644 --- a/src/config/compiler.rs +++ b/src/config/compiler.rs @@ -55,10 +55,10 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< } #[cfg(feature = "enterprise")] - let version = Some(builder.sha256_hash()); + let hash = Some(builder.sha256_hash()); #[cfg(not(feature = "enterprise"))] - let version = None; + let hash = None; let ConfigBuilder { global, @@ -123,7 +123,7 @@ pub fn compile(mut builder: ConfigBuilder) -> Result<(Config, Vec), Vec< schema, #[cfg(feature = "enterprise")] enterprise, - version, + hash, healthchecks, enrichment_tables, sources, diff --git a/src/config/enterprise.rs b/src/config/enterprise.rs index b521918ad6ba7..8c51dda45c73f 100644 --- a/src/config/enterprise.rs +++ b/src/config/enterprise.rs @@ -342,7 +342,7 @@ impl TryFrom<&Config> for EnterpriseMetadata { ); // Get the configuration version hash. In DD Pipelines, this is referred to as the 'config hash'. - let configuration_version_hash = value.version.clone().expect("Config should be versioned"); + let configuration_version_hash = value.hash.clone().expect("Config should be versioned"); Ok(Self { opts, diff --git a/src/config/mod.rs b/src/config/mod.rs index f8d934d07cfaa..24327fbb82192 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -101,7 +101,7 @@ pub struct Config { #[cfg(feature = "api")] pub api: api::Options, pub schema: schema::Options, - pub version: Option, + pub hash: Option, #[cfg(feature = "enterprise")] pub enterprise: Option, pub global: GlobalOptions, From 3bfb49a5dab3b1bf1096d8e2653a979abc778c13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:14:00 +0000 Subject: [PATCH 08/22] chore(deps): bump async-graphql from 4.0.13 to 4.0.14 (#14569) Bumps [async-graphql](https://github.com/async-graphql/async-graphql) from 4.0.13 to 4.0.14. - [Release notes](https://github.com/async-graphql/async-graphql/releases) - [Changelog](https://github.com/async-graphql/async-graphql/blob/master/CHANGELOG.md) - [Commits](https://github.com/async-graphql/async-graphql/commits) --- updated-dependencies: - dependency-name: async-graphql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 17 +++++++++-------- Cargo.toml | 2 +- lib/value/Cargo.toml | 4 ++-- lib/vector-core/Cargo.toml | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb7be65f0d1f5..9257b9f503f68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "async-graphql" -version = "4.0.13" +version = "4.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b4acd72d35f568664599c2cde503228b29910ca36656b653984c31c5a28cd6" +checksum = "af82ca20bc5a205231a504a0ca6dccd9d71665edcf73eaf14f2232da851cfb8b" dependencies = [ "async-graphql-derive", "async-graphql-parser", @@ -352,6 +352,7 @@ dependencies = [ "regex", "serde", "serde_json", + "serde_urlencoded", "static_assertions", "tempfile", "thiserror", @@ -359,9 +360,9 @@ dependencies = [ [[package]] name = "async-graphql-derive" -version = "4.0.13" +version = "4.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cead3c5f127c89b0abb2434c250409b9fe75763ee72583d4a90a412a927c8a8" +checksum = "66d47689c206c6234d3987b0ec406945ce832f45647d29f91054490ecc98a804" dependencies = [ "Inflector", "async-graphql-parser", @@ -375,9 +376,9 @@ dependencies = [ [[package]] name = "async-graphql-parser" -version = "4.0.13" +version = "4.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2e30051a98bcecc8baab3f7a8b12e8e49b365afa81b598ad5dffa49d343f51" +checksum = "7e1c7ef6924b3a7b056df47c44c000d18f53be84805354aed3682ae7cca5271a" dependencies = [ "async-graphql-value", "pest", @@ -387,9 +388,9 @@ dependencies = [ [[package]] name = "async-graphql-value" -version = "4.0.13" +version = "4.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c8c0d67fe10c7c49c6dcc9072b7a5c6c07af3b72b6e67cfb30ee070e2e940c" +checksum = "89f16b1bfce66e9551ddc1d0c97f060b870b51a3aa5b7a7510f773c34f38b504" dependencies = [ "bytes 1.2.1", "indexmap", diff --git a/Cargo.toml b/Cargo.toml index 2b4874bd3e385..2719baf94f206 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,7 @@ smpl_jwt = { version = "0.7.1", default-features = false, optional = true } lapin = { version = "2.1.1", default-features = false, optional = true } # API -async-graphql = { version = "4.0.13", default-features = false, optional = true, features = ["chrono"] } +async-graphql = { version = "4.0.14", default-features = false, optional = true, features = ["chrono"] } async-graphql-warp = { version = "4.0.13", default-features = false, optional = true } itertools = { version = "0.10.5", default-features = false, optional = true } diff --git a/lib/value/Cargo.toml b/lib/value/Cargo.toml index f271b4ac7e78d..bb2a73e767214 100644 --- a/lib/value/Cargo.toml +++ b/lib/value/Cargo.toml @@ -17,7 +17,7 @@ snafu = { version = "0.7.1", default-features = false } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } # Optional -async-graphql = { version = "4.0.13", default-features = false, optional = true } +async-graphql = { version = "4.0.14", default-features = false, optional = true } mlua = { version = "0.8.3", default-features = false, features = ["lua54", "send", "vendored"], optional = true} serde = { version = "1.0.145", default-features = false, features = ["derive", "rc"], optional = true } serde_json = { version = "1.0.85", optional = true } @@ -32,7 +32,7 @@ test = [] arbitrary = ["dep:quickcheck"] [dev-dependencies] -async-graphql = { version = "4.0.13", default-features = false } +async-graphql = { version = "4.0.14", default-features = false } indoc = { version = "1.0.7", default-features = false } quickcheck = "1.0.3" lookup = { path = "../lookup", default-features = false, features = ["arbitrary"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index fec4f8e7f0712..826e67f7899fa 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" publish = false [dependencies] -async-graphql = { version = "4.0.13", default-features = false, optional = true } +async-graphql = { version = "4.0.14", default-features = false, optional = true } async-trait = { version = "0.1", default-features = false } bitmask-enum = { version = "2.0.0", default-features = false } bytes = { version = "1.2.1", default-features = false, features = ["serde"] } From 109af3bc2cb4a35c369a4ac47ef979fc146f6df2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 17:23:44 +0000 Subject: [PATCH 09/22] chore(deps): bump md-5 from 0.10.4 to 0.10.5 (#14578) Bumps [md-5](https://github.com/RustCrypto/hashes) from 0.10.4 to 0.10.5. - [Release notes](https://github.com/RustCrypto/hashes/releases) - [Commits](https://github.com/RustCrypto/hashes/compare/md-5-v0.10.4...md-5-v0.10.5) --- updated-dependencies: - dependency-name: md-5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9257b9f503f68..c19e224a5b4ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4399,9 +4399,9 @@ dependencies = [ [[package]] name = "md-5" -version = "0.10.4" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66b48670c893079d3c2ed79114e3644b7004df1c361a4e0ad52e2e6940d07c3d" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ "digest 0.10.5", ] From fa80ef764d7dbf400ba001dab1ea225f5fd8838b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 18:12:41 +0000 Subject: [PATCH 10/22] chore(deps): bump proc-macro2 from 1.0.43 to 1.0.44 (#14576) Bumps [proc-macro2](https://github.com/dtolnay/proc-macro2) from 1.0.43 to 1.0.44. - [Release notes](https://github.com/dtolnay/proc-macro2/releases) - [Commits](https://github.com/dtolnay/proc-macro2/compare/1.0.43...1.0.44) --- updated-dependencies: - dependency-name: proc-macro2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c19e224a5b4ff..4e2025c102d17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5826,9 +5826,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "7bd7356a8122b6c4a24a82b278680c73357984ca2fc79a0f9fa6dea7dced7c58" dependencies = [ "unicode-ident", ] From 7760e6abb35b25c27bb241160aed35b0807d3b23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 18:54:11 +0000 Subject: [PATCH 11/22] chore(deps): bump bitmask-enum from 2.0.0 to 2.0.1 (#14565) Bumps [bitmask-enum](https://github.com/Lukas3674/rust-bitmask-enum) from 2.0.0 to 2.0.1. - [Release notes](https://github.com/Lukas3674/rust-bitmask-enum/releases) - [Commits](https://github.com/Lukas3674/rust-bitmask-enum/commits) --- updated-dependencies: - dependency-name: bitmask-enum dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- lib/vector-core/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e2025c102d17..62056510a6495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1246,9 +1246,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitmask-enum" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76487de46597d345d040a1be49a6fb636b71d0abab4696b7f3492e0cd4639c73" +checksum = "e33f40d686ed1a85afb4fd08bab2af946848fa47991dc256cdc1f81eea8c6b69" dependencies = [ "quote", "syn", diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 826e67f7899fa..da50bf12cd79d 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] async-graphql = { version = "4.0.14", default-features = false, optional = true } async-trait = { version = "0.1", default-features = false } -bitmask-enum = { version = "2.0.0", default-features = false } +bitmask-enum = { version = "2.0.1", default-features = false } bytes = { version = "1.2.1", default-features = false, features = ["serde"] } chrono = { version = "0.4.19", default-features = false, features = ["serde"] } crossbeam-utils = { version = "0.8.11", default-features = false } From b6ac5e789bd3bed9ac88700487f89441cb14d31f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 15:16:18 -0400 Subject: [PATCH 12/22] chore(deps): bump async-graphql-warp from 4.0.13 to 4.0.14 (#14579) Bumps [async-graphql-warp](https://github.com/async-graphql/async-graphql) from 4.0.13 to 4.0.14. - [Release notes](https://github.com/async-graphql/async-graphql/releases) - [Changelog](https://github.com/async-graphql/async-graphql/blob/master/CHANGELOG.md) - [Commits](https://github.com/async-graphql/async-graphql/commits) --- updated-dependencies: - dependency-name: async-graphql-warp dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62056510a6495..1d147a91545b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -400,9 +400,9 @@ dependencies = [ [[package]] name = "async-graphql-warp" -version = "4.0.13" +version = "4.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81db0c5c9de2304aa6d2daad3760224c7120ef9bbb1fb299bafe4931eadd121c" +checksum = "6c70c3a982a8583ba63390d797385cfd9683390ad50348b8f868f112e7019018" dependencies = [ "async-graphql", "futures-util", diff --git a/Cargo.toml b/Cargo.toml index 2719baf94f206..e377ce6a66cc6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -203,7 +203,7 @@ lapin = { version = "2.1.1", default-features = false, optional = true } # API async-graphql = { version = "4.0.14", default-features = false, optional = true, features = ["chrono"] } -async-graphql-warp = { version = "4.0.13", default-features = false, optional = true } +async-graphql-warp = { version = "4.0.14", default-features = false, optional = true } itertools = { version = "0.10.5", default-features = false, optional = true } # API client From 2a0e620a0759d63469a72df71c6f02740b15bce2 Mon Sep 17 00:00:00 2001 From: Pablo Sichert Date: Mon, 26 Sep 2022 22:03:15 +0200 Subject: [PATCH 13/22] chore(ci): Limit search in `check-events` script to `src`/`lib` (#14529) Signed-off-by: Pablo Sichert --- scripts/check-events | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/check-events b/scripts/check-events index 8dcef00f55c02..50d5b122bd5e7 100755 --- a/scripts/check-events +++ b/scripts/check-events @@ -308,7 +308,7 @@ $all_events = Hash::new { |hash, key| hash[key] = Event::new(key) } error_count = 0 # Scan sources and build internal structures -Find.find('.') do |path| +Find.find('./src', './lib') do |path| if path.start_with? './' path = path[2..] end From bf1d834a8a04f8256b0c2ff488cca102d87afc6d Mon Sep 17 00:00:00 2001 From: Toby Lawrence Date: Mon, 26 Sep 2022 16:06:16 -0400 Subject: [PATCH 14/22] chore: establish a basic style guide (#12482) --- STYLE.md | 317 +++++++++++++++++++++++++++++ scripts/check-events | 79 ++++++- src/sinks/blackhole/sink.rs | 12 +- src/sources/opentelemetry/reply.rs | 2 +- src/sources/postgresql_metrics.rs | 6 +- src/topology/builder.rs | 2 +- 6 files changed, 397 insertions(+), 21 deletions(-) create mode 100644 STYLE.md diff --git a/STYLE.md b/STYLE.md new file mode 100644 index 0000000000000..b33c5d4def630 --- /dev/null +++ b/STYLE.md @@ -0,0 +1,317 @@ +# Vector Style Guide + +As a large, open-source project, it can be a struggle to ensure a consistent level of code quality. +This style guide is meant to be the canonical reference for all things style: code comments, +formatting, acceptable (or unacceptable) crates, data structures, or algorithms, and so on. + +In essence, we hope to turn pull request review comments like "why did you do it this way?" or "I +think you could try doing it this way" into "we always do X this way: ". + +## Formatting + +At a high-level, code formatting is straightforward: we use the native `rustfmt` exclusively, and +comprehensively. All Rust source code within the repository should be formatted using `rustfmt`. + +You can acquire `rustfmt` -- which is invoked as `cargo fmt` -- by following the directions listed +out [in the rustfmt repository](https://github.com/rust-lang/rustfmt#on-the-stable-toolchain). + +Vector has its own formatting rules (`.rustfmt.toml`) that will automatically be used when you run +`cargo fmt` within the repository. If you're ever in doubt, you can also run `make check-fmt` which +will invoke `cargo fmt` in a dry-run mode, checking to see if any changed files are not currently +formatted correctly. + +As an additional note, `rustfmt` sometimes can fail to format code within macros, so if you happen +to see such code that doesn't look like it's formatted correctly, you may need to manually tweak it +if `rustfmt` cannot be persuaded to format it correctly for you. :) + +## Code Organization + +Code is primarily split into two main directories: `lib/` and `src/`. + +### `lib/`: shared libraries, etc + +We use `lib` almost entirely for shared libraries and for isolating specific pieces of code. As +Vector itself involves a large number of dependencies, it can be beneficial to move code into +isolated crates under `lib/` in order to allow them not only to be shared, but also to reduce the +amount of code that must be processed by helper tools like `cargo check` and `rust-analyzer` during +normal development, which in turn speeds up the feedback loop between writing code and getting +informed about errors, warnings, and so on. + +### `src/`: main binary and all related functionality + +The bulk of functional code resides in the `src/` directory/crate. When we refer to functional +code, we're talking about code that powers user-visible aspects of Vector, such as the sources, +transforms, and sinks themselves. There is also, of course, the requisite glue code such as parsing +command-line arguments, reading the configuration, constructing and configuring components, and +wiring them together. + +## Internal telemetry: logging, metrics, traces + +As a tool for ingesting, transforming, enriching, and shipping observability data, Vector has a +significant amount of its own internal telemetry. This telemetry is primarily logging and metrics, +but also includes some amount of tracing. + +### Logging + +For logging, we use **[`tracing`](https://docs.rs/tracing/latest/tracing)**, which doubles as both a +way to emit logs but also a way to use distributed tracing techniques to add nested and contextual +metadata by utilizing [spans](https://docs.rs/tracing/latest/tracing/#spans). + +#### Basic Usage + +For logging, we use `tracing`'s event macros which should look very similar to almost all other +logging libraries, with names that emulate the logging level being used i.e. `info!("A wild log has +appeared.");`. + +All of tracing's event macros -- `trace!`, `debug!`, `info!`, `warn!`, and `error!` -- support the +same argument format, which allows logging in some common ways: + +```rust +// Basic string literal message, no formatting: +info!("Server has started."); + +// A formatted message, with the same formatting support as `println!`/`format!`: +debug!("User connected: {}", username); + +// Adding structured fields to the even, mixing and matching the message format: +trace!(bytes_sent = 22, "Sent heartbeat packet to client.")` +error!(client_addr = %conn.get_ref().peer_addr, "Client actor received malformed packet: {}", parse_err.to_string()) +``` + +While this does not cover all the permutations of what the macros in `tracing` support, these +examples represent the preferred style of using the macros. + +#### Passing in the event message + +The `tracing` event macros support passing the event message itself in a few ways, but we prefer the +**fields/message/message arguments** order: + +```rust +// Don't do this: +info!(message = "Something happened."); +// Do this instead: +info!("Something happened."); + +// Don't do this: +debug!(%client_id, message = "Client entered authentication phase."); +debug!(message = "Client entered authentication phase.", %client_id); +// Do this instead: +debug!(%client_id, "Client entered authentication phase."); +``` + +#### Writing a good log message + +In general, there are both a few rules and a few suggestions to follow when it comes to writing a +(good) log message: + +- Messages must be written in English. No preference on which specific English dialect is used e.g. + American English, British English, Canadian English, etc. +- Sentences must be capitalized, and end with a period. +- Proper spelling and grammar when possible. Not all of us are native English speakers, and so this + is simply an ask, but not a hard requirement. +- Identifiers, or passages of note, should be called out by some means i.e. wrapping them in + backticks or quotes. Wrapping with special characters can be helpful in drawing the users eye to + anything of importance. +- If it's longer than one or two sentences, it's probably better suited as a single sentence briefly + explaining the event, with a link to external documentation that explains further. + +#### Choosing the right log level + +Similarly, choosing the right level can be important, both from the perspective of making it easy +for users to grok what they should pay attention to, but also to avoid the performance overhead of +excess logging (even if we filter it out and it never makes it to the console). + +- **TRACE**: Typically contains a high level of detail for deep/rich debugging. + + As trace logging is typically reached for when instrumenting algorithms and core pieces of logic, + care should be taken to avoid trace logging being added to tight loops, or commonly used + codepaths, where possible. Even when disabled, there can still be a small overhead associated with + logging an event at all. +- **DEBUG**: Basic information that can be helpful for initially debugging issues. + + Should typically not be used for things that happen per-event, or scales with event throughput, + but in some cases -- i.e. if it happens every 1000th event, etc -- it can safely be used. +- **INFO**: Common information about normal processes. + + This includes logical/temporal events such as notifications when components are stopped and + started, or other high-level events that, crucially, do not represent an event that an operator + needs to worry about. + + Said another way, **INFO** is primarily there for information that lets them know that an action + they just took completed successfully, whether that's the server initially starting up + successfully, or reloading a configuration successfully, or exiting Vector after receiving + SIGTERM. +- **WARN**: Something unexpected happened, but no data has been lost, nothing has crashed, and we + can recover from it without an issue. An operator might be interested in something at the **WARN** + level, but it shouldn't be informing them of things serious enough to require immediate attention. +- **ERROR**: Data loss, unrecoverable errors, and anything else that will require an operator to + intervene and recover from. These should be rare so that they maintain a high signal-to-noise + ratio in the observability tooling that operators themselves are using. + +### Metrics + +For metrics, we use **[`metrics`](https://docs.rs/metrics/latest/metrics/)**, which like `tracing`, +provides macros for emitting counters, gauges, and histograms. + +#### Basic Usage + +There are three basic metric types: counters, gauges, and histograms. **Counters** are meant for +counting things, such as the total number of requests processed, where the count only grows over +time. This is also sometimes called a *monotonic counter*, or a *monotonically increasing* counter. +**Gauges** are for tracking a single value that changes over time, and can go up and down, such as +the number of current connections. **Histograms** are for tracking multiple observations of the same +logical event, such as the time it takes to serve a request. + +Emitting metrics always involves a metric name and a value being measured. They can optionally +accept descriptive labels: + +```rust +// Counters can be incremented by an arbitrary amount, or `increment_counter!` +// can be used to simply increment by one: +counter!("bytes_sent_total", 212); +increment_counter!("requests_processed_total", "service" => "admin_grpc"); + +// Gauges can be set to an absolute value, such as setting it to the latest value of an +// external measurement, or it can be incremented and decremented by an arbitrary amount: +gauge!("bytes_allocated", 42.0); +increment_gauge!("bytes_allocated", 2048.0, "table_name" => self.table_name.to_string()); +increment_gauge!("bytes_allocated", 512.0, "table_name" => self.table_name.to_string()); +decrement_gauge!("bytes_allocated", 2560.0, "table_name" => self.table_name.to_string()) + +// Histograms simply record a measurement, but there's a fun little trait that `metrics` +// uses called `IntoF64` that lets custom types provide a way to convert themselves to a +// `f64`, which there's a default implementation of for `Duration`: +let delta = Duration::from_micros(750); +histogram!("request_duration_ns", delta); +histogram!("request_duration_ns", 742_130, "endpoint" => "frontend"); +``` + +#### Avoiding pitfalls with gauges + +Many values can appear, at first, to be best tracked as a gauge: current connection count, in-flight +request count, and so on. However, in some cases, the value being measured may change too frequently +to be reliably tracked. Metrics are typically collected on an interval, which is fine for counters +and histograms: they're purely additive. However, since a gauge is simply the _latest_ value, you +cannot know _how_ it's changed since the last time you've observed it. + +This is a common problem where a gauge tracks something like a queue size. If there's an event where +the queue grows rapidly but drains back down quickly, you may not ever observe the gauge having +changed if your collection interval is greater than the duration of such events. + +A simple pattern to follow to handle these scenarios is to use two counters -- one for the +increments, one for the decrements -- so that you can graph the difference between them, giving you +the equivalent of the "current" value. In our example above, we might have `queue_items_pushed` and +`queued_items_popped`, and if `queue_items_pushed` equals 100, and `queued_items_popped` equals 80, +we know our queue size is 20. More importantly, if we queried both of them at the same time, and +they were both zero, and then queried them both a second later, and saw that both were 100,000, we +would know that the queue size was _currently_ zero but we'd also know that we just processed +100,000 items in the past second. + +#### Best Practices + +- **Do** attempt to limit the cardinality, or number of unique values, of label values. If the + number of unique values for a label grows over time, this can represent a large amount of consumed + memory. This is a problem we expect to be fixed in the medium-term + ([#11995](https://github.com/vectordotdev/vector/issues/11995)) but is a good rule to follow + unless there's a competing reason to do so, such as when following the guidelines in the + [Component + Specification](https://github.com/vectordotdev/vector/blob/master/docs/specs/component.md). +- **Don't** emit metrics in tight loops. Each metric emission carries an overhead, and emitting them + in tight loops can cause that overhead to become noticable in terms of CPU usage and throughput + reduction. Instead of incrementing a counter every time a loop iteration occurs, you might + consider incrementing a local variable instead, and then emitting that sum after the loop is over. +- **Don't** update a counter to measure the total number of operations/events/etc if you're already + tracking a histogram of those operations/events/etc. Histograms have a `count` property that + counts how many samples the histogram has recorded, as well as a `sum` property that is a sum of + the value of all samples the histogram has recorded. This means you can potentially get three + metrics for the cost of emitting one. + +## Dependencies + +### Error handling + +For error handling, there are two parts: _creating errors_ and _working with errors_. + +For **creating errors**, we prefer **[`snafu`](https://docs.rs/snafu)**. The `snafu` crate provides +a derive for generating the boilerplate `std::error::Error` implementation on your custom error +struct or enum. It additionally provides helpers for defining the `Display` output for your error +(potentially on a per-variant basis when working with enums). + +While there are popular alternatives such as [`failure`](https://docs.rs/failure) and +[`thiserror`](https://docs.rs/thiserror), they generally lack either the thoroughness in +documentation or the flexibility of `snafu`. + +For **working with errors**, we have a more lax approach. At the highest level, we use a boxed trait +approach -- `Box` -- for maximum flexibility. This +allows developers to avoid needing to _always_ derive custom error types in order to return errors +back up the call stack. This does not prevent, and indeed, should not discourage developers from +using `snafu` to create rich error types that provide additional context, whether through +descriptive error messages, source errors, or backtraces. + +### Concurrency and synchronization + +#### Atomics + +In general, we strive to use the atomic types in the [standard +library](https://doc.rust-lang.org/stable/std/sync/atomic/index.html) when possible, as they are the +most portable and well-tested. In cases where the standard library atomics cannot be used, such as +when using a 64-bit atomic but wanting to support a 32-bit platform, or support a platform without +atomic instructions at all, we prefer to use +**[`crossbeam-utils`](https://docs.rs/crossbeam-utils)** and its `AtomicCell` helper. This type will +automatically handle either using native atomic support or ensuring mutally exclusive access, and +handle it in a transparent way. It uses a fixed acquire/release ordering that generally provides the +expected behavior when using atomics, but may not be suitable for usages which require stronger +ordering. + +#### Global state + +When there is a need or desire to share global state, there are a few options depending on the +required constraints. + +If you're working with data that is _lazily initialized_ but _never changes after initialization_, +we prefer **[`once_cell`](https://docs.rs/once_cell)**. It is slightly faster than +[`lazy_static`](https://docs.rs/lazy-static), and additionally provides a richer API than both +`lazy_static` and the standard library variants, such as `std::sync::Once`. Additionally, there is +[active work happening](https://github.com/rust-lang/rust/issues/74465) to migrate the types in +`once_cell` into `std::sync` directly, which will be easier to switch to if we're already using +`once_cell`. + +If you're working with data that _changes over time_, but has a very high read-to-write ratio, such +as _many readers_, but _one writer_ and infrequent writes, we prefer +**[`arc-swap`](https://docs.rs/arc-swap)**. The main feature of this crate is allowing a piece of +data to be atomically updated while being shared concurrently. It does this by wrapping all data in +`Arc` to provide the safe, concurrent access, while adding the ability to atomically swap the +`Arc` itself. As it cannot be constructed in a const fashion, `arc-swap` pairs well with +`once_cell` for actually storing it in a global static variable. + +#### Concurrent data structures + +When there is a need for a concurrent and _indexable_ storage, we prefer +**[`sharded-slab`](https://docs.rs/sharded-slab)**. This crate provides a means to insert items +such that the caller gets back to the index by which it can access the item again in the future. +Additionally, when an item is removed, its storage can be reused by future inserts, making +`sharded-slab` a good choice for long-running processes where memory allocation reduction is +paramount. There is also a pool data structure based on the same underlying design of the slab +itself for use cases where pooling is desired. + +### Synchronization + +Synchronization can be a very common sight when writing multi-threaded code in any language, and +this document does not aim to familiarize you with all of the common synchronization primitives and +their intended usage. Instesd, however, there are some caveats that you must be aware of when using +synchronization primitives in synchronous versus asynchronous code. + +Generally speaking, developers will lean on the synchronization primitives in `std::sync`, such as +`Mutex`, `RwLock`, and so on. These are typically the right choice -- in terms of using the ones +provided by `std`, vs alternative implementations of the same primitives -- as they're well-tested, +and have been improving over time in terms of performance. However, developers must be careful when +using these primitives in asynchronous code, as their behavior can sometimes adversely affect the +performance and correctness of Vector. + +To wit, developers must exercise caution when using synchronous (i.e. `std::sync`, `parking_lot`, +etc) synchronization primitives in asynchronous code, as they can be used in a way that deadlocks +the asynchronous runtime even though the code compiles and appears to be correct. In some cases, +you'll need to use an asynchronous-specific synchronization primitives, namely the ones from `tokio` +itself. The documentation on `tokio`'s own +[`Mutex`](https://docs.rs/tokio/latest/tokio/sync/struct.Mutex.html), for example, calls out the +specifics of when and where you might need to use it vs the one from `std::sync`. diff --git a/scripts/check-events b/scripts/check-events index 50d5b122bd5e7..94f1d7c6ead78 100755 --- a/scripts/check-events +++ b/scripts/check-events @@ -36,6 +36,16 @@ def is_constant?(name) name.start_with? '"' and name.end_with? '"' or name.match? /^(.+::)[A-Z0-9_]$/ end +def find_line_number(haystack, needle) + idx = haystack.index(needle) + if !idx.nil? + prior = haystack[0,idx] + prior.lines.count + else + nil + end +end + # A class to hold error reports and common functionality class Event attr_accessor :path, :skip_dropped_events, :uses, :impl_internal_event, :impl_register_event, :impl_event_handle @@ -321,20 +331,67 @@ Find.find('./src', './lib') do |path| $all_events[event_name].uses += 1 end - # Check log message texts for correct formatting. See below for the - # full regex + # Check log message texts for correct formatting. if path.start_with? 'src/' - text.scan(/(trace|debug|info|warn|error)!\(\s*(message\s*=\s*)?"([^({)][^("]+)"/) do - |type, has_message_prefix, message| - reports = [] - reports.append('Message must start with a capital.') unless message.match(/^[[:upper:]]/) - reports.append('Message must end with a period.') unless message.match(/\.$/) - unless reports.empty? - puts "Error(s) in file #{path}" - reports.each { |report| puts " #{report}" } - error_count += 1 + reports = [] + + # Try to find all general usage of the various `tracing` macros. + text.scan(/( + (trace|debug|info|warn|error)!\( # Log type. + ([^;]*?) # All parameters to the macro. + \)(?:;|\n\s*}) # Handles usages that lack a trailing semicolon. + )/mx) \ + do |full, type, params| + # Extract each parameter to the macros, which involves handling structured fields and + # string literals. We parse them further below so that we can iterate through them to try + # and determine what the actual log message is, depending on if it's set by using the + # `message` field, or implicitly with a string literal. + # + # We also have some special handling in there for `tracing`-specific "target" and "parent" + # settings which influence how the event is handled when being processed by a subscriber, + # which we don't care about _here_ but need to account for in our pattern to parse things. + params = params.scan(/("(?:[^"\\]++|\\.)*+"|(?:target|parent):\s*[^,]+|(\w+\s*=\s*(?:"(?:[^"\\]++|\\.)*+"|[%?]?[^,]+))|[%?][^,]+)/) \ + .map do |param| + if /^\".*\"$/.match?(param[0].strip) + { "type" => "litstr", "value" => param[0] } + elsif param[0].include? "=" + parts = param[0].split('=', 2).map { |part| part.strip } + { "type" => "named_field", "field" => parts[0], "value" => parts[1] } + else + { "type" => "field", "field" => param[0] } + end + end + + # See if we found a message field. + message_param = params.find { |param| + # Use the first string literal parameter. + param["type"] == "litstr" || + # Or the first named field called `message` that has a value that is a string literal. + (param["type"] == "named_field" && param["field"] == "message" && /^\".*\"$/.match?(param["value"])) + } + + # We further scrutinize the message field, if we believe we found one. This lets us avoid + # scenarios where variable interpolation is being used, since we can't reasonably detect if + # an interpolated variable at the beginning or end of the message is capitalized or has a + # trailing period, respectively. + has_message = !message_param.nil? + message = if has_message then message_param["value"].gsub(/^"|"$/, '') else nil end + is_capitalized = !has_message || (message[0] == "{" || message.match(/^[[:upper:]]/)) + has_trailing_period = !has_message || (message[-1, 1] == "}" || message.match(/\.$/)) + + match_reports = [] + match_reports.append('Message must start with a capital.') unless is_capitalized + match_reports.append('Message must end with a period.') unless has_trailing_period + unless match_reports.empty? + line_no = find_line_number(text, full) + match_reports.each { |report| reports.push(" #{report} (`#{type}` call on #{path}:#{line_no})") } end end + + unless reports.empty? + reports.each { |report| puts report } + error_count += reports.length + end end # TODO remove @skip_dropped_events check logic after DroppedEvents audit is complete diff --git a/src/sinks/blackhole/sink.rs b/src/sinks/blackhole/sink.rs index 55dce44d2ad74..900f1da8d9a27 100644 --- a/src/sinks/blackhole/sink.rs +++ b/src/sinks/blackhole/sink.rs @@ -55,19 +55,21 @@ impl StreamSink for BlackholeSink { loop { select! { _ = print_interval.tick() => { - info!({ + info!( events = total_events.load(Ordering::Relaxed), raw_bytes_collected = total_raw_bytes.load(Ordering::Relaxed), - }, "Total events collected"); + "Collected events." + ); }, _ = tripwire.changed() => break, } } - info!({ + info!( events = total_events.load(Ordering::Relaxed), - raw_bytes_collected = total_raw_bytes.load(Ordering::Relaxed) - }, "Total events collected"); + raw_bytes_collected = total_raw_bytes.load(Ordering::Relaxed), + "Collected events." + ); }); } diff --git a/src/sources/opentelemetry/reply.rs b/src/sources/opentelemetry/reply.rs index 2a7f1ab6a399a..d61078b662bc1 100644 --- a/src/sources/opentelemetry/reply.rs +++ b/src/sources/opentelemetry/reply.rs @@ -17,7 +17,7 @@ where let mut buf = BytesMut::with_capacity(1024); Protobuf { inner: val.encode(&mut buf).map(|_| buf.to_vec()).map_err(|err| { - tracing::error!("http::reply::protobuf() error: {}", err); + error!("Failed to encode value: {}", err); }), } } diff --git a/src/sources/postgresql_metrics.rs b/src/sources/postgresql_metrics.rs index 7a55cd82c9d4e..49fa31e021770 100644 --- a/src/sources/postgresql_metrics.rs +++ b/src/sources/postgresql_metrics.rs @@ -872,7 +872,7 @@ fn config_to_endpoint(config: &Config) -> String { SslMode::Require => params.push(("sslmode", "require".to_string())), // non_exhaustive enum _ => { - warn!(r#"Unknown variant of "SslMode.""#); + warn!("Unknown variant of \"SslMode\"."); } }; @@ -914,7 +914,7 @@ fn config_to_endpoint(config: &Config) -> String { } // non_exhaustive enum _ => { - warn!(r#"Unknown variant of "TargetSessionAttrs.""#); + warn!("Unknown variant of \"TargetSessionAttr\"."); } } @@ -925,7 +925,7 @@ fn config_to_endpoint(config: &Config) -> String { ChannelBinding::Require => params.push(("channel_binding", "require".to_owned())), // non_exhaustive enum _ => { - warn!(r#"Unknown variant of "ChannelBinding"."#); + warn!("Unknown variant of \"ChannelBinding\"."); } } diff --git a/src/topology/builder.rs b/src/topology/builder.rs index 58e9794f9d0d9..ac40a47fb2126 100644 --- a/src/topology/builder.rs +++ b/src/topology/builder.rs @@ -332,7 +332,7 @@ pub async fn build_pieces( .sinks() .filter(|(key, _)| diff.sinks.contains_new(key)) { - debug!(component = %key, "Building new sink"); + debug!(component = %key, "Building new sink."); let sink_inputs = &sink.inputs; let healthcheck = sink.healthcheck(); From 9215ec5fdfe76e82e9df028abe70f2330455b9b9 Mon Sep 17 00:00:00 2001 From: Spencer Gilbert Date: Mon, 26 Sep 2022 16:07:55 -0400 Subject: [PATCH 15/22] chore: Handle installing protoc on arm and x86 (#14581) * chore: Handle installing protoc on arm and x86 Signed-off-by: Spencer Gilbert * +shellcheck Signed-off-by: Spencer Gilbert * +shellcheck ignore for readonly Signed-off-by: Spencer Gilbert Signed-off-by: Spencer Gilbert --- scripts/environment/install-protoc.sh | 62 ++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 10 deletions(-) diff --git a/scripts/environment/install-protoc.sh b/scripts/environment/install-protoc.sh index 040ff981f286d..6d85a608cfaf6 100644 --- a/scripts/environment/install-protoc.sh +++ b/scripts/environment/install-protoc.sh @@ -1,15 +1,57 @@ #! /usr/bin/env bash set -o errexit -o verbose -TEMP=$(mktemp -d) - # Protoc. No guard because we want to override Ubuntu's old version in # case it is already installed by a dependency. -PROTOC_VERSION=3.19.5 -PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip -curl -fsSL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/$PROTOC_ZIP \ - --output "$TEMP/$PROTOC_ZIP" -unzip "$TEMP/$PROTOC_ZIP" bin/protoc -d "$TEMP" -chmod +x "$TEMP"/bin/protoc -mv --force --verbose "$TEMP"/bin/protoc /usr/bin/protoc -rm -fr "$TEMP" +# +# Basis of script copied from: +# https://github.com/paxosglobal/asdf-protoc/blob/46c2f9349b8420144b197cfd064a9677d21cfb0c/bin/install + +# shellcheck disable=SC2155 +readonly TMP_DIR="$(mktemp -d -t "protoc_XXXX")" +trap 'rm -rf "${TMP_DIR?}"' EXIT + +get_platform() { + local os + os=$(uname) + if [[ "${os}" == "Darwin" ]]; then + echo "osx" + elif [[ "${os}" == "Linux" ]]; then + echo "linux" + else + >&2 echo "unsupported os: ${os}" && exit 1 + fi +} + +get_arch() { + local os + local arch + os=$(uname) + arch=$(uname -m) + # On ARM Macs, uname -m returns "arm64", but in protoc releases this architecture is called "aarch_64" + if [[ "${os}" == "Darwin" && "${arch}" == "arm64" ]]; then + echo "aarch_64" + elif [[ "${os}" == "Linux" && "${arch}" == "aarch64" ]]; then + echo "aarch_64" + else + echo "${arch}" + fi +} + +install_protoc() { + local version=$1 + local install_path=$2 + + local base_url="https://github.com/protocolbuffers/protobuf/releases/download" + local url + url="${base_url}/v${version}/protoc-${version}-$(get_platform)-$(get_arch).zip" + local download_path="${TMP_DIR}/protoc.zip" + + echo "Downloading ${url}" + curl -fsSL "${url}" -o "${download_path}" + + unzip -qq "${download_path}" -d "${TMP_DIR}" + mv --force --verbose "${TMP_DIR}/bin/protoc" "${install_path}" +} + +install_protoc "3.19.5" "/usr/bin/protoc" From 6b8c9da0ed73fa6c05b8a9ea74ce68c25a05251c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 20:09:58 +0000 Subject: [PATCH 16/22] chore(deps): bump tokio from 1.20.1 to 1.21.0 (#14274) * chore(deps): bump tokio from 1.20.1 to 1.21.0 Bumps [tokio](https://github.com/tokio-rs/tokio) from 1.20.1 to 1.21.0. - [Release notes](https://github.com/tokio-rs/tokio/releases) - [Commits](https://github.com/tokio-rs/tokio/compare/tokio-1.20.1...tokio-1.21.0) --- updated-dependencies: - dependency-name: tokio dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] * Fix unused lock guard in test Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Bruce Guenter --- Cargo.lock | 21 +++++---------------- Cargo.toml | 4 ++-- lib/file-source/Cargo.toml | 2 +- lib/k8s-e2e-tests/Cargo.toml | 2 +- lib/k8s-test-framework/Cargo.toml | 2 +- lib/vector-api-client/Cargo.toml | 2 +- lib/vector-buffers/Cargo.toml | 2 +- lib/vector-common/Cargo.toml | 4 ++-- lib/vector-core/Cargo.toml | 2 +- src/sources/syslog.rs | 4 ++-- src/topology/test/end_to_end.rs | 2 +- 11 files changed, 18 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1d147a91545b8..7cb2e66a1d6d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4528,25 +4528,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52da4364ffb0e4fe33a9841a98a3f3014fb964045ce4f7a45a398243c8d6b0c9" +checksum = "57ee1c23c7c63b0c9250c339ffdc69255f110b298b901b9f6c82547b7b87caaf" dependencies = [ "libc", "log", - "miow", - "ntapi", "wasi 0.11.0+wasi-snapshot-preview1", - "winapi", -] - -[[package]] -name = "miow" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21" -dependencies = [ - "winapi", + "windows-sys 0.36.1", ] [[package]] @@ -7763,9 +7752,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.20.1" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a8325f63a7d4774dd041e363b2409ed1c5cbbd0f867795e661df066b2b0a581" +checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" dependencies = [ "autocfg", "bytes 1.2.1", diff --git a/Cargo.toml b/Cargo.toml index e377ce6a66cc6..d372544ecef48 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -136,7 +136,7 @@ vrl-cli = { path = "lib/vrl/cli", optional = true } async-stream = { version = "0.3.3", default-features = false } async-trait = { version = "0.1.57", default-features = false } futures = { version = "0.3.24", default-features = false, features = ["compat", "io-compat"], package = "futures" } -tokio = { version = "1.20.1", default-features = false, features = ["full"] } +tokio = { version = "1.21.1", default-features = false, features = ["full"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1.10", default-features = false, features = ["net", "sync", "time"] } tokio-util = { version = "0.7", default-features = false, features = ["io", "time"] } @@ -344,7 +344,7 @@ proptest = "1.0" quickcheck = "1.0.3" reqwest = { version = "0.11", features = ["json"] } tempfile = "3.3.0" -tokio = { version = "1.20.1", features = ["test-util"] } +tokio = { version = "1.21.0", features = ["test-util"] } tokio-test = "0.4.2" tower-test = "0.4.0" value = { path = "lib/value", features = ["test"] } diff --git a/lib/file-source/Cargo.toml b/lib/file-source/Cargo.toml index d02f57cad3391..c488113780b62 100644 --- a/lib/file-source/Cargo.toml +++ b/lib/file-source/Cargo.toml @@ -66,7 +66,7 @@ default-features = false features = [] [dependencies.tokio] -version = "1.20.1" +version = "1.21.0" default-features = false features = ["full"] diff --git a/lib/k8s-e2e-tests/Cargo.toml b/lib/k8s-e2e-tests/Cargo.toml index 97564027b8214..ae7aa7137c376 100644 --- a/lib/k8s-e2e-tests/Cargo.toml +++ b/lib/k8s-e2e-tests/Cargo.toml @@ -14,7 +14,7 @@ k8s-test-framework = { version = "0.1", path = "../k8s-test-framework" } regex = "1" reqwest = { version = "0.11.12", features = ["json"] } serde_json = "1" -tokio = { version = "1.20.1", features = ["full"] } +tokio = { version = "1.21.0", features = ["full"] } indoc = "1.0.7" env_logger = "0.9" tracing = { version = "0.1", features = ["log"] } diff --git a/lib/k8s-test-framework/Cargo.toml b/lib/k8s-test-framework/Cargo.toml index d3efa49a45335..9f9c375837dc2 100644 --- a/lib/k8s-test-framework/Cargo.toml +++ b/lib/k8s-test-framework/Cargo.toml @@ -11,5 +11,5 @@ license = "MPL-2.0" k8s-openapi = { version = "0.16.0", default-features = false, features = ["v1_19"] } serde_json = "1" tempfile = "3" -tokio = { version = "1.20.1", features = ["full"] } +tokio = { version = "1.21.0", features = ["full"] } log = "0.4" diff --git a/lib/vector-api-client/Cargo.toml b/lib/vector-api-client/Cargo.toml index 333c2d0b63411..cde23af6d470f 100644 --- a/lib/vector-api-client/Cargo.toml +++ b/lib/vector-api-client/Cargo.toml @@ -18,7 +18,7 @@ anyhow = { version = "1.0.65", default-features = false, features = ["std"] } # Tokio / Futures async-trait = { version = "0.1", default-features = false } futures = { version = "0.3", default-features = false, features = ["compat", "io-compat"] } -tokio = { version = "1.20.1", default-features = false, features = ["macros", "rt", "sync"] } +tokio = { version = "1.21.0", default-features = false, features = ["macros", "rt", "sync"] } tokio-stream = { version = "0.1.10", default-features = false, features = ["sync"] } # GraphQL diff --git a/lib/vector-buffers/Cargo.toml b/lib/vector-buffers/Cargo.toml index ef8afdd1a03c8..2815a7e00ef9d 100644 --- a/lib/vector-buffers/Cargo.toml +++ b/lib/vector-buffers/Cargo.toml @@ -27,7 +27,7 @@ rkyv = { version = "0.7.39", default-features = false, features = ["size_32", "s serde = { version = "1.0.145", default-features = false, features = ["derive"] } snafu = { version = "0.7.1", default-features = false, features = ["std"] } tokio-util = { version = "0.7.0", default-features = false } -tokio = { version = "1.20.1", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } +tokio = { version = "1.21.0", default-features = false, features = ["rt", "macros", "rt-multi-thread", "sync", "fs", "io-util", "time"] } tracing = { version = "0.1.34", default-features = false, features = ["attributes"] } vector-config = { path = "../vector-config", default-features = false } vector-config-common = { path = "../vector-config-common", default-features = false } diff --git a/lib/vector-common/Cargo.toml b/lib/vector-common/Cargo.toml index e53e70dcbdbfa..a220179e50abb 100644 --- a/lib/vector-common/Cargo.toml +++ b/lib/vector-common/Cargo.toml @@ -62,7 +62,7 @@ serde = { version = "1.0.145", optional = true, features = ["derive"] } smallvec = { version = "1", default-features = false } snafu = { version = "0.7", optional = true } stream-cancel = { version = "0.8.1", default-features = false } -tokio = { version = "1.20.1", default-features = false, features = ["macros", "time"] } +tokio = { version = "1.21.0", default-features = false, features = ["macros", "time"] } tracing = { version = "0.1.34", default-features = false } value = { path = "../value", features = ["json"] } vector-config = { path = "../vector-config" } @@ -71,4 +71,4 @@ vector-config-macros = { path = "../vector-config-macros" } [dev-dependencies] futures = { version = "0.3.24", default-features = false, features = ["async-await", "std"] } -tokio = { version = "1.20.1", default-features = false, features = ["rt", "time"] } +tokio = { version = "1.21.0", default-features = false, features = ["rt", "time"] } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index da50bf12cd79d..c0e21327b7877 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -44,7 +44,7 @@ serde_json = { version = "1.0.85", default-features = false } serde_with = { version = "2.0.1", default-features = false, features = ["std", "macros"] } snafu = { version = "0.7.1", default-features = false } socket2 = { version = "0.4.7", default-features = false } -tokio = { version = "1.20.1", default-features = false, features = ["net"] } +tokio = { version = "1.21.0", default-features = false, features = ["net"] } tokio-openssl = { version = "0.6.3", default-features = false } tokio-stream = { version = "0.1", default-features = false, features = ["time"], optional = true } tokio-util = { version = "0.7.0", default-features = false, features = ["time"] } diff --git a/src/sources/syslog.rs b/src/sources/syslog.rs index e92a9bfaa0ab6..322617c925c1f 100644 --- a/src/sources/syslog.rs +++ b/src/sources/syslog.rs @@ -822,7 +822,7 @@ mod test { send_lines(in_addr, input_lines).await.unwrap(); // Wait a short period of time to ensure the messages get sent. - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; // Shutdown the source, and make sure we've got all the messages we sent in. shutdown @@ -976,7 +976,7 @@ mod test { send_encodable(in_addr, codec, input_lines).await.unwrap(); // Wait a short period of time to ensure the messages get sent. - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; // Shutdown the source, and make sure we've got all the messages we sent in. shutdown diff --git a/src/topology/test/end_to_end.rs b/src/topology/test/end_to_end.rs index d1f87fd80a4b5..89e9871a8325e 100644 --- a/src/topology/test/end_to_end.rs +++ b/src/topology/test/end_to_end.rs @@ -24,7 +24,7 @@ pub async fn respond( tx.send(()) .await .expect("Error sending 'before' status from test server"); - waiter.lock().await; + let _ = waiter.lock().await; Ok(Response::builder() .status(status) .body(Body::empty()) From 63c2ba8248d1ea906eafeca25eb13d4591e6365a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 20:53:44 +0000 Subject: [PATCH 17/22] chore(deps): bump pulsar from 4.1.2 to 4.1.3 (#14577) Bumps [pulsar](https://github.com/streamnative/pulsar-rs) from 4.1.2 to 4.1.3. - [Release notes](https://github.com/streamnative/pulsar-rs/releases) - [Commits](https://github.com/streamnative/pulsar-rs/compare/v4.1.2...v4.1.3) --- updated-dependencies: - dependency-name: pulsar dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 6 ++++-- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7cb2e66a1d6d8..6cf2179103d45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5999,9 +5999,9 @@ dependencies = [ [[package]] name = "pulsar" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31a5a4fbe9363c113b7e25ae76dd2d6455411c815d688977be6f0b68ae5e73b8" +checksum = "5b55e5d7c8ffe92d9d788624ade45200d19ffd371bbc6dde99b57f8843aaaf92" dependencies = [ "async-trait", "bit-vec 0.6.3", @@ -6029,6 +6029,7 @@ dependencies = [ "tokio-native-tls", "tokio-util 0.7.1", "url", + "uuid 1.1.2", ] [[package]] @@ -8653,6 +8654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f" dependencies = [ "getrandom 0.2.6", + "rand 0.8.5", "serde", ] diff --git a/Cargo.toml b/Cargo.toml index d372544ecef48..d371b711b2ebb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -281,7 +281,7 @@ ordered-float = { version = "3.1.0", default-features = false } percent-encoding = { version = "2.2.0", default-features = false } pin-project = { version = "1.0.12", default-features = false } postgres-openssl = { version = "0.5.0", default-features = false, features = ["runtime"], optional = true } -pulsar = { version = "4.1.2", default-features = false, features = ["tokio-runtime", "auth-oauth2"], optional = true } +pulsar = { version = "4.1.3", default-features = false, features = ["tokio-runtime", "auth-oauth2"], optional = true } rand = { version = "0.8.5", default-features = false, features = ["small_rng"] } rand_distr = { version = "0.4.3", default-features = false } rdkafka = { version = "0.28.0", default-features = false, features = ["tokio", "libz", "ssl", "zstd"], optional = true } From ff503c200408807f434c0e5d5da2c597db9ab01a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 22:31:02 +0000 Subject: [PATCH 18/22] chore(deps): bump syn from 1.0.99 to 1.0.101 (#14584) Bumps [syn](https://github.com/dtolnay/syn) from 1.0.99 to 1.0.101. - [Release notes](https://github.com/dtolnay/syn/releases) - [Commits](https://github.com/dtolnay/syn/compare/1.0.99...1.0.101) --- updated-dependencies: - dependency-name: syn dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6cf2179103d45..06a3c6531fb8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7489,9 +7489,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.99" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58dbef6ec655055e20b86b15a8cc6d439cca19b667537ac6a1369572d151ab13" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" dependencies = [ "proc-macro2", "quote", From e463399a532413bc240d3db4eac17da35ae12713 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 23:22:24 +0000 Subject: [PATCH 19/22] chore(deps): bump governor from 0.4.2 to 0.5.0 (#14562) Bumps [governor](https://github.com/antifuchs/governor) from 0.4.2 to 0.5.0. - [Release notes](https://github.com/antifuchs/governor/releases) - [Changelog](https://github.com/antifuchs/governor/blob/master/release.toml) - [Commits](https://github.com/antifuchs/governor/compare/v0.4.2...v0.5.0) --- updated-dependencies: - dependency-name: governor dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06a3c6531fb8a..667c27a1a6f3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3073,10 +3073,11 @@ dependencies = [ [[package]] name = "governor" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87" +checksum = "de1b4626e87b9eb1d603ed23067ba1e29ec1d0b35325a2b96c3fe1cf20871f56" dependencies = [ + "cfg-if", "dashmap", "futures 0.3.24", "futures-timer", diff --git a/Cargo.toml b/Cargo.toml index d371b711b2ebb..fd3626eff14d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -246,7 +246,7 @@ exitcode = { version = "1.1.2", default-features = false } flate2 = { version = "1.0.24", default-features = false, features = ["default"] } futures-util = { version = "0.3.21", default-features = false } glob = { version = "0.3.0", default-features = false } -governor = { version = "0.4.1", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } +governor = { version = "0.5.0", default-features = false, features = ["dashmap", "jitter", "std"], optional = true } grok = { version = "2.0.0", default-features = false, optional = true } h2 = { version = "0.3.13", default-features = false, optional = true } hash_hasher = { version = "2.0.0", default-features = false } From 88506e819b596a3f69a275fafa2c5b31e3a336d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Sep 2022 21:15:01 -0400 Subject: [PATCH 20/22] chore(deps): bump openssl from 0.10.41 to 0.10.42 (#14588) Bumps [openssl](https://github.com/sfackler/rust-openssl) from 0.10.41 to 0.10.42. - [Release notes](https://github.com/sfackler/rust-openssl/releases) - [Commits](https://github.com/sfackler/rust-openssl/compare/openssl-v0.10.41...openssl-v0.10.42) --- updated-dependencies: - dependency-name: openssl dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- lib/vector-core/Cargo.toml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 667c27a1a6f3c..1dc4b80dc133c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5195,9 +5195,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.41" +version = "0.10.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618febf65336490dfcf20b73f885f5651a0c89c64c2d4a8c3662585a70bf5bd0" +checksum = "12fc0523e3bd51a692c8850d075d74dc062ccf251c0110668cbd921917118a13" dependencies = [ "bitflags", "cfg-if", @@ -5235,9 +5235,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.75" +version = "0.9.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5f9bd0c2710541a3cda73d6f9ac4f1b240de4ae261065d309dbe73d9dceb42f" +checksum = "5230151e44c0f05157effb743e8d517472843121cf9243e8b81393edb5acd9ce" dependencies = [ "autocfg", "cc", diff --git a/Cargo.toml b/Cargo.toml index fd3626eff14d3..191c0e232edc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -274,7 +274,7 @@ nkeys = { version = "0.2.0", default-features = false, optional = true } nom = { version = "7.1.1", default-features = false, optional = true } notify = { version = "5.0.0", default-features = false, features = ["macos_fsevent"] } once_cell = { version = "1.15", default-features = false } -openssl = { version = "0.10.41", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.42", default-features = false, features = ["vendored"] } openssl-probe = { version = "0.1.5", default-features = false } openssl-src = { version = "111", default-features = false } ordered-float = { version = "3.1.0", default-features = false } diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index c0e21327b7877..9194429c294d1 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -30,7 +30,7 @@ metrics-util = { version = "0.14.0", default-features = false, features = ["regi mlua = { version = "0.8.3", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.2", default-features = false, features = ["serialize"] } once_cell = { version = "1.15", default-features = false } -openssl = { version = "0.10.41", default-features = false, features = ["vendored"] } +openssl = { version = "0.10.42", default-features = false, features = ["vendored"] } ordered-float = { version = "3.1.0", default-features = false } parking_lot = { version = "0.12.1", default-features = false } pin-project = { version = "1.0.12", default-features = false } From f38e3c39e6984e61ec000278d37e74dce7960c5b Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 27 Sep 2022 11:05:48 +0100 Subject: [PATCH 21/22] fix(observability): emit `RequestBuildError` when `HttpSink::request_builder` errors (#14575) Emit RequestBuildError when request_builder errors. Signed-off-by: Stephen Wakely Signed-off-by: Stephen Wakely --- src/sinks/util/http.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sinks/util/http.rs b/src/sinks/util/http.rs index 61349bc911a0a..b25c1b7b001e7 100644 --- a/src/sinks/util/http.rs +++ b/src/sinks/util/http.rs @@ -27,7 +27,7 @@ use super::{ use crate::{ event::Event, http::{HttpClient, HttpError}, - internal_events::EndpointBytesSent, + internal_events::{EndpointBytesSent, SinkRequestBuildError}, }; pub trait HttpEventEncoder { @@ -382,11 +382,16 @@ where let mut http_client = self.inner.clone(); Box::pin(async move { - let request = request_builder(body).await?; + let request = request_builder(body).await.map_err(|error| { + emit!(SinkRequestBuildError { error: &error }); + error + })?; let byte_size = request.body().len(); let request = request.map(Body::from); let (protocol, endpoint) = uri::protocol_endpoint(request.uri().clone()); + // Any errors raised in `http_client.call` results in a `GotHttpWarning` event being emmited + // in `HttpClient::send`. let response = http_client.call(request).await?; if response.status().is_success() { From fd5ba448c73261601b722705e0a788a20492d3d8 Mon Sep 17 00:00:00 2001 From: Stephen Wakely Date: Tue, 27 Sep 2022 14:53:02 +0100 Subject: [PATCH 22/22] fix(gcp_chronicle sink): add run_and_assert_sink_error to unhappy path test (#14596) Add run_and_assert_sink_error to unhappy path test Signed-off-by: Stephen Wakely Signed-off-by: Stephen Wakely --- Makefile | 3 +- .../integration/docker-compose.chronicle.yml | 2 +- src/sinks/gcp/chronicle_unstructured.rs | 43 ++++++++++++++----- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 0c54f5bd03b7c..8351194ba2eac 100644 --- a/Makefile +++ b/Makefile @@ -334,7 +334,8 @@ test-enterprise: ## Runs enterprise related behavioral tests .PHONY: test-integration test-integration: ## Runs all integration tests -test-integration: test-integration-amqp test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-clickhouse test-integration-docker-logs test-integration-elasticsearch +test-integration: test-integration-amqp test-integration-apex test-integration-aws test-integration-axiom test-integration-azure test-integration-chronicle test-integration-clickhouse +test-integration: test-integration-docker-logs test-integration-elasticsearch test-integration: test-integration-eventstoredb test-integration-fluent test-integration-gcp test-integration-humio test-integration-http-scrape test-integration-influxdb test-integration: test-integration-kafka test-integration-logstash test-integration-loki test-integration-mongodb test-integration-nats test-integration: test-integration-nginx test-integration-opentelemetry test-integration-postgres test-integration-prometheus test-integration-pulsar diff --git a/scripts/integration/docker-compose.chronicle.yml b/scripts/integration/docker-compose.chronicle.yml index 6a1ea6bf77af9..dbdc289e9e58c 100644 --- a/scripts/integration/docker-compose.chronicle.yml +++ b/scripts/integration/docker-compose.chronicle.yml @@ -1,4 +1,4 @@ -version: "1" +version: "3" services: chronicle-emulator: diff --git a/src/sinks/gcp/chronicle_unstructured.rs b/src/sinks/gcp/chronicle_unstructured.rs index 047f174c08f85..61b58a9827603 100644 --- a/src/sinks/gcp/chronicle_unstructured.rs +++ b/src/sinks/gcp/chronicle_unstructured.rs @@ -4,7 +4,7 @@ use bytes::{Bytes, BytesMut}; use futures_util::{future::BoxFuture, task::Poll}; use goauth::scopes::Scope; -use http::{header::HeaderValue, Request, Uri}; +use http::{header::HeaderValue, Request, StatusCode, Uri}; use hyper::Body; use indoc::indoc; use serde_json::json; @@ -23,7 +23,7 @@ use crate::{ codecs::{self, EncodingConfig}, config::{log_schema, GenerateConfig, SinkConfig, SinkContext}, gcp::{GcpAuthConfig, GcpAuthenticator}, - http::{HttpClient, HttpError}, + http::HttpClient, sinks::{ gcs_common::{ config::{healthcheck_response, GcsRetryLogic}, @@ -423,9 +423,17 @@ impl ChronicleService { } } +#[derive(Debug, Snafu)] +pub enum ChronicleResponseError { + #[snafu(display("Server responded with an error: {}", code))] + ServerError { code: StatusCode }, + #[snafu(display("Failed to make HTTP(S) request: {}", error))] + HttpError { error: crate::http::HttpError }, +} + impl Service for ChronicleService { type Response = GcsResponse; - type Error = HttpError; + type Error = ChronicleResponseError; type Future = BoxFuture<'static, Result>; fn poll_ready(&mut self, _: &mut std::task::Context<'_>) -> Poll> { @@ -449,12 +457,22 @@ impl Service for ChronicleService { let mut client = self.client.clone(); Box::pin(async move { - let result = client.call(http_request).await; - result.map(|inner| GcsResponse { - inner, - protocol: "http", - metadata: request.metadata, - }) + match client.call(http_request).await { + Ok(response) => { + let status = response.status(); + + if status.is_success() { + Ok(GcsResponse { + inner: response, + protocol: "http", + metadata: request.metadata, + }) + } else { + Err(ChronicleResponseError::ServerError { code: status }) + } + } + Err(error) => Err(ChronicleResponseError::HttpError { error }), + } }) } } @@ -467,7 +485,10 @@ mod integration_tests { use super::*; use crate::test_util::{ - components::{run_and_assert_sink_compliance, SINK_TAGS}, + components::{ + run_and_assert_sink_compliance, run_and_assert_sink_error, COMPONENT_ERROR_TAGS, + SINK_TAGS, + }, random_events_with_stream, random_string, trace_init, }; @@ -553,7 +574,7 @@ mod integration_tests { let (batch, mut receiver) = BatchNotifier::new_with_receiver(); let (_input, events) = random_events_with_stream(100, 100, Some(batch)); - let _ = sink.run(events).await; + run_and_assert_sink_error(sink, events, &COMPONENT_ERROR_TAGS).await; assert_eq!(receiver.try_recv(), Ok(BatchStatus::Rejected)); }