From 352eb1652c195174f75e5075fb2a36e383c5371d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 25 Sep 2023 18:38:52 -0400 Subject: [PATCH] chore(deps): Update VRL to 0.7.0 --- Cargo.lock | 248 +++++++++--------- Cargo.toml | 6 +- lib/codecs/Cargo.toml | 2 +- .../vector_generate_fixtures.patch | 4 +- lib/vector-core/Cargo.toml | 4 +- lib/vrl/value/Cargo.toml | 4 +- .../remap/functions/community_id.cue | 80 ++++++ .../reference/remap/functions/encode_json.cue | 6 + .../functions/parse_aws_vpc_flow_log.cue | 40 ++- .../remap/functions/to_timestamp.cue | 90 ------- website/cue/reference/urls.cue | 3 +- 11 files changed, 268 insertions(+), 219 deletions(-) create mode 100644 website/cue/reference/remap/functions/community_id.cue delete mode 100644 website/cue/reference/remap/functions/to_timestamp.cue diff --git a/Cargo.lock b/Cargo.lock index a4a9e90d3c5a6..9eb8d5c51a624 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -307,12 +307,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "arrayvec" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" - [[package]] name = "arrayvec" version = "0.7.2" @@ -662,7 +656,7 @@ dependencies = [ "rand 0.8.5", "regex", "ring", - "rustls 0.21.6", + "rustls 0.21.7", "rustls-native-certs", "rustls-pemfile", "rustls-webpki", @@ -1551,9 +1545,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.2" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" [[package]] name = "bitmask-enum" @@ -2069,9 +2063,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.3" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84ed82781cea27b43c9b106a979fe450a13a31aab0500595fb3fc06616de08e6" +checksum = "824956d0dca8334758a5b7f7e50518d66ea319330cbceedcf76905c2f6ab30e3" dependencies = [ "clap_builder", "clap_derive", @@ -2083,21 +2077,21 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eef05769009513df2eb1c3b4613e7fad873a14c600ff025b08f250f59fee7de" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", "log", ] [[package]] name = "clap_builder" -version = "4.4.2" +version = "4.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" +checksum = "122ec64120a49b4563ccaedcbea7818d069ed8e9aa6d829b82d8a4128936b2ab" dependencies = [ "anstream", "anstyle 1.0.0", "clap_lex", "strsim 0.10.0", - "terminal_size 0.2.2", + "terminal_size 0.3.0", ] [[package]] @@ -2106,7 +2100,7 @@ version = "4.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4110a1e6af615a9e6d0a36f805d5c99099f8bab9b8042f5bc1fa220a4a89e36f" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", ] [[package]] @@ -2237,6 +2231,20 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "community-id" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a97e158411823bf87634e2e47552f712e51fa4119cdb2255da799e7cb5c90a9" +dependencies = [ + "anyhow", + "base64 0.21.4", + "hex", + "lazy_static", + "num_enum 0.6.1", + "sha1", +] + [[package]] name = "concurrent-queue" version = "1.2.4" @@ -2411,7 +2419,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.4.3", + "clap 4.4.5", "criterion-plot", "futures 0.3.28", "is-terminal", @@ -2499,7 +2507,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "crossterm_winapi", "futures-core", "libc", @@ -2980,9 +2988,9 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "2.0.2" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f332aa79f9e9de741ac013237294ef42ce2e9c6394dc7d766725812f1238812" +checksum = "8d0fa3cd8dc96ada974e126a940d37d4079bbbe6a24aca15b1113c2f362441c5" dependencies = [ "cfg-if", "libc", @@ -3218,17 +3226,6 @@ dependencies = [ "serde", ] -[[package]] -name = "errno" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" -dependencies = [ - "errno-dragonfly", - "libc", - "winapi", -] - [[package]] name = "errno" version = "0.3.1" @@ -4459,12 +4456,6 @@ version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1be380c410bf0595e94992a648ea89db4dd3f3354ba54af206fd2a68cf5ac8e" -[[package]] -name = "io-lifetimes" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" - [[package]] name = "io-lifetimes" version = "1.0.11" @@ -4519,7 +4510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi 0.3.1", - "io-lifetimes 1.0.11", + "io-lifetimes", "rustix 0.37.19", "windows-sys 0.48.0", ] @@ -4929,15 +4920,15 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.0.46" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" [[package]] name = "linux-raw-sys" -version = "0.3.8" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" [[package]] name = "listenfd" @@ -5016,11 +5007,12 @@ dependencies = [ [[package]] name = "luajit-src" -version = "210.4.3+resty8384278" +version = "210.4.8+resty107baaf" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ee5d5afddf1ec76ffa55ca7c3001f2f8a703834beba53c56a38ea6641cef44" +checksum = "e05167e8b2a2185758d83ed23541e5bd8bce37072e4204e0ef2c9b322bc87c4e" dependencies = [ "cc", + "which", ] [[package]] @@ -5278,18 +5270,28 @@ dependencies = [ [[package]] name = "mlua" -version = "0.8.10" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e" +checksum = "6c3a7a7ff4481ec91b951a733390211a8ace1caba57266ccb5f4d4966704e560" +dependencies = [ + "bstr 1.6.2", + "mlua-sys", + "num-traits", + "once_cell", + "rustc-hash", +] + +[[package]] +name = "mlua-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ec8b54eddb76093069cce9eeffb4c7b3a1a0fe66962d7bd44c4867928149ca3" dependencies = [ - "bstr 0.2.17", "cc", + "cfg-if", "lua-src", "luajit-src", - "num-traits", - "once_cell", "pkg-config", - "rustc-hash", ] [[package]] @@ -5524,7 +5526,7 @@ version = "6.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "filetime", "fsevent-sys", "inotify", @@ -5589,7 +5591,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.2", + "arrayvec", "itoa", "num-bigint", ] @@ -5644,6 +5646,15 @@ dependencies = [ "num_enum_derive 0.5.11", ] +[[package]] +name = "num_enum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a015b430d3c108a207fd776d2e2196aaf8b1cf8cf93253e3a097ff3085076a1" +dependencies = [ + "num_enum_derive 0.6.1", +] + [[package]] name = "num_enum" version = "0.7.0" @@ -5665,6 +5676,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num_enum_derive" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96667db765a921f7b295ffee8b60472b686a51d4f21c2ee4ffdb94c7013b65a6" +dependencies = [ + "proc-macro-crate 1.2.1", + "proc-macro2 1.0.67", + "quote 1.0.33", + "syn 2.0.37", +] + [[package]] name = "num_enum_derive" version = "0.7.0" @@ -5840,7 +5863,7 @@ version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cfg-if", "foreign-types", "libc", @@ -5929,6 +5952,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a540f3e3b3d7929c884e46d093d344e4e5bdeed54d08bf007df50c93cc85d5" +dependencies = [ + "num-traits", +] + [[package]] name = "os_info" version = "3.7.0" @@ -6073,19 +6105,20 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pest" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1acb4a4365a13f749a93f1a094a7805e5cfa0955373a9de860d962eaa3a5fe5a" +checksum = "d7a4d085fd991ac8d5b05a147b437791b4260b76326baf0fc60cf7c9c27ecd33" dependencies = [ + "memchr", "thiserror", "ucd-trie", ] [[package]] name = "pest_derive" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666d00490d4ac815001da55838c500eafb0320019bbaa44444137c48b443a853" +checksum = "a2bee7be22ce7918f641a33f08e3f43388c7656772244e2bbb2477f44cc9021a" dependencies = [ "pest", "pest_generator", @@ -6093,9 +6126,9 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ca01446f50dbda87c1786af8770d535423fa8a53aec03b8f4e3d7eb10e0929" +checksum = "d1511785c5e98d79a05e8a6bc34b4ac2168a0e3e92161862030ad84daa223141" dependencies = [ "pest", "pest_meta", @@ -6106,9 +6139,9 @@ dependencies = [ [[package]] name = "pest_meta" -version = "2.7.2" +version = "2.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56af0a30af74d0445c0bf6d9d051c979b516a1a5af790d251daee76005420a48" +checksum = "b42f0394d3123e33353ca5e1e89092e533d2cc490389f2bd6131c43c634ebc5f" dependencies = [ "once_cell", "pest", @@ -6946,7 +6979,7 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e2e4cd95294a85c3b4446e63ef054eea43e0205b1fd60120c16b74ff7ff96ad" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cassowary", "crossterm", "indoc", @@ -7365,7 +7398,7 @@ version = "1.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26bd36b60561ee1fb5ec2817f198b6fd09fa571c897a5e86d1487cfc2b096dfc" dependencies = [ - "arrayvec 0.7.2", + "arrayvec", "borsh", "bytecheck", "byteorder", @@ -7419,29 +7452,28 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.13" +version = "0.37.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" +checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" dependencies = [ "bitflags 1.3.2", - "errno 0.2.8", - "io-lifetimes 0.7.5", + "errno", + "io-lifetimes", "libc", - "linux-raw-sys 0.0.46", - "windows-sys 0.42.0", + "linux-raw-sys 0.3.8", + "windows-sys 0.48.0", ] [[package]] name = "rustix" -version = "0.37.19" +version = "0.38.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" dependencies = [ - "bitflags 1.3.2", - "errno 0.3.1", - "io-lifetimes 1.0.11", + "bitflags 2.4.0", + "errno", "libc", - "linux-raw-sys 0.3.8", + "linux-raw-sys 0.4.7", "windows-sys 0.48.0", ] @@ -7524,7 +7556,7 @@ version = "12.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "994eca4bca05c87e86e15d90fc7a91d1be64b4482b38cb2d27474568fe7c9db9" dependencies = [ - "bitflags 2.3.2", + "bitflags 2.4.0", "cfg-if", "clipboard-win", "libc", @@ -8236,22 +8268,13 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "strip-ansi-escapes" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "011cbb39cf7c1f62871aea3cc46e5817b0937b49e9447370c93cacbe93a766d8" -dependencies = [ - "vte 0.10.1", -] - [[package]] name = "strip-ansi-escapes" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55ff8ef943b384c414f54aefa961dd2bd853add74ec75e7ac74cf91dba62bcfa" dependencies = [ - "vte 0.11.1", + "vte", ] [[package]] @@ -8465,12 +8488,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" +checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" dependencies = [ - "rustix 0.35.13", - "windows-sys 0.42.0", + "rustix 0.38.14", + "windows-sys 0.48.0", ] [[package]] @@ -8971,7 +8994,7 @@ checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ "async-compression", "base64 0.21.4", - "bitflags 2.3.2", + "bitflags 2.4.0", "bytes 1.5.0", "futures-core", "futures-util", @@ -9536,7 +9559,7 @@ dependencies = [ "atty", "cached", "chrono", - "clap 4.4.3", + "clap 4.4.5", "clap-verbosity-flag", "clap_complete", "confy", @@ -9613,7 +9636,7 @@ dependencies = [ "bytesize", "chrono", "cidr-utils", - "clap 4.4.3", + "clap 4.4.5", "codecs", "colored", "console-subscriber", @@ -9723,7 +9746,7 @@ dependencies = [ "snap", "socket2 0.5.4", "stream-cancel", - "strip-ansi-escapes 0.2.0", + "strip-ansi-escapes", "syslog", "tempfile", "test-generator", @@ -9774,7 +9797,7 @@ dependencies = [ "anyhow", "async-trait", "chrono", - "clap 4.4.3", + "clap 4.4.5", "futures 0.3.28", "graphql_client", "indoc", @@ -9797,7 +9820,7 @@ dependencies = [ "async-trait", "bytecheck", "bytes 1.5.0", - "clap 4.4.3", + "clap 4.4.5", "crc32fast", "criterion", "crossbeam-queue", @@ -10017,7 +10040,7 @@ dependencies = [ name = "vector-vrl-cli" version = "0.1.0" dependencies = [ - "clap 4.4.3", + "clap 4.4.5", "vector-vrl-functions", "vrl", ] @@ -10036,7 +10059,7 @@ dependencies = [ "ansi_term", "chrono", "chrono-tz", - "clap 4.4.3", + "clap 4.4.5", "enrichment", "glob", "prettydiff", @@ -10078,9 +10101,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf1c43c0a6834959dd38eb762b462d156df043f3dfd86550ff62a96bd572466c" +checksum = "10d3c3ffe5322a8a7f63cf422aebb5000774e4152955fca1cd024c262893db5d" dependencies = [ "aes", "ansi_term", @@ -10097,8 +10120,9 @@ dependencies = [ "chrono", "chrono-tz", "cidr-utils", - "clap 4.4.3", + "clap 4.4.5", "codespan-reporting", + "community-id", "crypto_secretbox", "csv", "ctr", @@ -10122,7 +10146,7 @@ dependencies = [ "ofb", "once_cell", "onig", - "ordered-float 3.9.1", + "ordered-float 4.1.0", "paste", "peeking_take_while", "percent-encoding", @@ -10144,7 +10168,7 @@ dependencies = [ "sha2", "sha3", "snafu", - "strip-ansi-escapes 0.1.1", + "strip-ansi-escapes", "syslog_loose", "termcolor", "thiserror", @@ -10158,17 +10182,6 @@ dependencies = [ "zstd 0.12.4", ] -[[package]] -name = "vte" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cbce692ab4ca2f1f3047fcf732430249c0e971bfdd2b234cf2c47ad93af5983" -dependencies = [ - "arrayvec 0.5.2", - "utf8parse", - "vte_generate_state_changes", -] - [[package]] name = "vte" version = "0.11.1" @@ -10409,13 +10422,14 @@ dependencies = [ [[package]] name = "which" -version = "4.3.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix 0.38.14", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9526d4897b938..08c67e6b00457 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,7 +117,7 @@ members = [ ] [workspace.dependencies] -vrl = { version = "0.6.0", default-features = false, features = ["cli", "test", "test_framework", "arbitrary", "compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] } +vrl = { version = "0.7.0", default-features = false, features = ["cli", "test", "test_framework", "arbitrary", "compiler", "value", "diagnostic", "path", "parser", "stdlib", "datadog", "core"] } pin-project = { version = "1.1.3", default-features = false } @@ -336,7 +336,7 @@ arr_macro = { version = "0.2.1" } heim = { git = "https://github.com/vectordotdev/heim.git", branch = "update-nix", default-features = false, features = ["disk"] } # make sure to update the external docs when the Lua version changes -mlua = { version = "0.8.10", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"], optional = true } [target.'cfg(windows)'.dependencies] windows-service = "0.6.0" @@ -373,7 +373,7 @@ tokio = { version = "1.32.0", features = ["test-util"] } tokio-test = "0.4.3" tower-test = "0.4.0" vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] } -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } wiremock = "0.5.19" zstd = { version = "0.12.4", default-features = false } diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 1c86f744d01a4..49f227ddc357d 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -38,7 +38,7 @@ futures = { version = "0.3", default-features = false } indoc = { version = "2", default-features = false } tokio = { version = "1", features = ["test-util"] } similar-asserts = "1.5.0" -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } vector-core = { path = "../vector-core", default-features = false, features = ["test"] } [features] diff --git a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch index bc0821aae9c76..ac046ce0adf7d 100644 --- a/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch +++ b/lib/codecs/tests/data/native_encoding/vector_generate_fixtures.patch @@ -6,8 +6,8 @@ index 0f51fc830..aadf99841 100644 rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } --vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } -+vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"], path = "../../../vrl" } +-vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } ++vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"], path = "../../../vrl" } [features] api = ["dep:async-graphql"] diff --git a/lib/vector-core/Cargo.toml b/lib/vector-core/Cargo.toml index 849a6dca87c5d..04e65f0956e86 100644 --- a/lib/vector-core/Cargo.toml +++ b/lib/vector-core/Cargo.toml @@ -28,7 +28,7 @@ lookup = { package = "vector-lookup", path = "../vector-lookup" } metrics = "0.21.1" metrics-tracing-context = { version = "0.14.0", default-features = false } metrics-util = { version = "0.15.1", default-features = false, features = ["registry"] } -mlua = { version = "0.8.10", default-features = false, features = ["lua54", "send", "vendored"], optional = true } +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"], optional = true } no-proxy = { version = "0.3.4", default-features = false, features = ["serialize"] } once_cell = { version = "1.18", default-features = false } ordered-float = { version = "3.9.1", default-features = false } @@ -95,7 +95,7 @@ rand = "0.8.5" rand_distr = "0.4.3" tracing-subscriber = { version = "0.3.17", default-features = false, features = ["env-filter", "fmt", "ansi", "registry"] } vector-common = { path = "../vector-common", default-features = false, features = ["test"] } -vrl = { version = "0.6.0", features = ["cli", "test", "test_framework", "arbitrary"] } +vrl = { version = "0.7.0", features = ["cli", "test", "test_framework", "arbitrary"] } [features] api = ["dep:async-graphql"] diff --git a/lib/vrl/value/Cargo.toml b/lib/vrl/value/Cargo.toml index d9e51dbe25ba5..11854a880617f 100644 --- a/lib/vrl/value/Cargo.toml +++ b/lib/vrl/value/Cargo.toml @@ -18,7 +18,7 @@ tracing = { version = "0.1.34", default-features = false, features = ["attribute # Optional async-graphql = { version = "5.0.7", default-features = false, optional = true } -mlua = { version = "0.8.8", default-features = false, features = ["lua54", "send", "vendored"], optional = true} +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"], optional = true} serde = { version = "1.0.158", default-features = false, features = ["derive", "rc"], optional = true } serde_json = { version = "1.0.94", optional = true } toml = { version = "0.7.3", default-features = false, optional = true } @@ -39,4 +39,4 @@ lookup = { path = "../lookup", default-features = false, features = ["arbitrary" serde = { version = "1.0.158", default-features = false, features = ["derive", "rc"]} serde_json = { version = "1.0.94"} toml = { version = "0.7.3", default-features = false } -mlua = { version = "0.8.8", default-features = false, features = ["lua54", "send", "vendored"]} +mlua = { version = "0.9.1", default-features = false, features = ["lua54", "send", "vendored"]} diff --git a/website/cue/reference/remap/functions/community_id.cue b/website/cue/reference/remap/functions/community_id.cue new file mode 100644 index 0000000000000..41a53170e4dad --- /dev/null +++ b/website/cue/reference/remap/functions/community_id.cue @@ -0,0 +1,80 @@ +package metadata + +remap: functions: community_id: { + category: "String" + description: """ + Used to generate an id based on the [Community ID Spec](\(urls.community_id_spec)). + """ + + arguments: [ + { + name: "source_ip" + description: "The source IP address." + required: true + type: ["string"] + }, + { + name: "destination_ip" + description: "The destination IP address." + required: true + type: ["string"] + }, + { + name: "protocol" + description: "The protocol number." + required: true + type: ["integer"] + }, + { + name: "source_port" + description: "The source port." + required: false + type: ["integer"] + }, + { + name: "destination_port" + description: "The destination port." + required: false + type: ["integer"] + }, + { + name: "seed" + description: "The custom seed number." + required: false + type: ["integer"] + }, + ] + internal_failure_reasons: [] + return: types: ["string"] + + examples: [ + { + title: "TCP" + source: #""" + community_id!(source_ip: "1.2.3.4", destination_ip: "5.6.7.8", source_port: 1122, destination_port: 3344, protocol: 6) + """# + return: "1:wCb3OG7yAFWelaUydu0D+125CLM=" + }, + { + title: "UDP" + #""" + community_id!(source_ip: "1.2.3.4", destination_ip: "5.6.7.8", source_port: 1122, destination_port: 3344, protocol: 17) + """# + return: "1:0Mu9InQx6z4ZiCZM/7HXi2WMhOg=" + }, + { + title: "ICMP" + #""" + community_id!(source_ip: "1.2.3.4", destination_ip: "5.6.7.8", source_port: 8, destination_port: 0, protocol: 1) + """# + return: "1:crodRHL2FEsHjbv3UkRrfbs4bZ0=" + }, + { + title: "RSVP" + #""" + community_id!(source_ip: "1.2.3.4", destination_ip: "5.6.7.8", protocol: 46) + """# + return: "1:ikv3kmf89luf73WPz1jOs49S768=" + }, + ] +} diff --git a/website/cue/reference/remap/functions/encode_json.cue b/website/cue/reference/remap/functions/encode_json.cue index f2fa58df05130..1757905cf7ee6 100644 --- a/website/cue/reference/remap/functions/encode_json.cue +++ b/website/cue/reference/remap/functions/encode_json.cue @@ -13,6 +13,12 @@ remap: functions: encode_json: { required: true type: ["any"] }, + { + name: "pretty" + description: "Whether to pretty print the JSON string or not." + required: false + type: ["boolean"] + }, ] internal_failure_reasons: [] return: types: ["string"] diff --git a/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue b/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue index 0bbf752a562da..8fcd5a8e29abc 100644 --- a/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue +++ b/website/cue/reference/remap/functions/parse_aws_vpc_flow_log.cue @@ -33,7 +33,7 @@ remap: functions: parse_aws_vpc_flow_log: { """# return: { "version": 2 - "account_id": 123456789010 + "account_id": "123456789010" "interface_id": "eni-1235b8ca123456789" "srcaddr": null "dstaddr": null @@ -65,5 +65,43 @@ remap: functions: parse_aws_vpc_flow_log: { "pkt_dstaddr": "203.0.113.5" } }, + { + title: "Parse AWS VPC Flow including v5 fields" + source: #""" + parse_aws_vpc_flow_log!("5 52.95.128.179 10.0.0.71 80 34210 6 1616729292 1616729349 IPv4 14 15044 123456789012 vpc-abcdefab012345678 subnet-aaaaaaaa012345678 i-0c50d5961bcb2d47b eni-1235b8ca123456789 ap-southeast-2 apse2-az3 - - ACCEPT 19 52.95.128.179 10.0.0.71 S3 - - ingress OK", + format: "version srcaddr dstaddr srcport dstport protocol start end type packets bytes account_id vpc_id subnet_id instance_id interface_id region az_id sublocation_type sublocation_id action tcp_flags pkt_srcaddr pkt_dstaddr pkt_src_aws_service pkt_dst_aws_service traffic_path flow_direction log_status") + """# + return: { + "account_id": "123456789012" + "action": "ACCEPT" + "az_id": "apse2-az3" + "bytes": 15044 + "dstaddr": "10.0.0.71" + "dstport": 34210 + "end": 1616729349 + "flow_direction": "ingress" + "instance_id": "i-0c50d5961bcb2d47b" + "interface_id": "eni-1235b8ca123456789" + "log_status": "OK" + "packets": 14 + "pkt_dst_aws_service": null + "pkt_dstaddr": "10.0.0.71" + "pkt_src_aws_service": "S3" + "pkt_srcaddr": "52.95.128.179" + "protocol": 6 + "region": "ap-southeast-2" + "srcaddr": "52.95.128.179" + "srcport": 80 + "start": 1616729292 + "sublocation_id": null + "sublocation_type": null + "subnet_id": "subnet-aaaaaaaa012345678" + "tcp_flags": 19 + "traffic_path": null + "type": "IPv4" + "version": 5 + "vpc_id": "vpc-abcdefab012345678" + } + }, ] } diff --git a/website/cue/reference/remap/functions/to_timestamp.cue b/website/cue/reference/remap/functions/to_timestamp.cue deleted file mode 100644 index f46e3a37d23b1..0000000000000 --- a/website/cue/reference/remap/functions/to_timestamp.cue +++ /dev/null @@ -1,90 +0,0 @@ -package metadata - -remap: functions: to_timestamp: { - category: "Coerce" - description: """ - This function is deprecated. - For integer values, use `from_unix_timestamp`, otherwise use `parse_timestamp`. - Coerces the `value` into a timestamp. - """ - notices: ["There is the possibility of precision loss due to float arithmetic when coercing floats."] - - deprecated: true - - arguments: [ - { - name: "value" - description: "The value that is to be converted to a timestamp. If a string, must be a valid representation of a `timestamp` otherwise an `ArgumentError` will be raised." - required: true - type: ["string", "float", "integer", "timestamp"] - }, - { - name: "unit" - description: "The time unit." - type: ["string"] - required: false - enum: { - seconds: "Express Unix time in seconds" - milliseconds: "Express Unix time in milliseconds" - nanoseconds: "Express Unix time in nanoseconds" - } - default: "seconds" - }, - ] - internal_failure_reasons: [ - "When `value` is a `string`, it is not a valid timestamp format", - "When `value` is an `int`, it is not within the Unix timestamp range", - "When `value` is a `float`, it is not within the Unix timestamp range", - ] - return: { - types: ["timestamp"] - rules: [ - "If `value` is a `string`, the timestamp is parsed in these formats.", - "If `value` is an `integer`, it is assumed to be a Unix representation of the timestamp (the number of seconds after January 1st, 1970).", - "If `value` is a `float`, it s assumed to be a Unix representation of the timestamp (the number of seconds after January 1st, 1970) with fractional seconds.", - ] - } - - examples: [ - { - title: "Coerce a string to a timestamp" - source: """ - to_timestamp!("2020-10-21T16:00:00Z") - """ - return: "2020-10-21T16:00:00Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp (integer) to a timestamp" - source: """ - to_timestamp!(1675968923) - """ - return: "2023-02-09T18:55:23Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp (float) to a timestamp" - source: """ - to_timestamp!(1675968923.567) - """ - return: "2023-02-09T18:55:23.566999912Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp, in milliseconds, to a timestamp" - source: """ - to_timestamp!(1676478566639, unit: "milliseconds") - """ - return: "2023-02-15T16:29:26.639Z" - skip_test: true - }, - { - title: "Coerce a unix timestamp, in nanoseconds, to a timestamp" - source: """ - to_timestamp!(1675968923012312311, unit: "nanoseconds") - """ - return: "2023-02-09T18:55:23.012312311Z" - skip_test: true - }, - ] -} diff --git a/website/cue/reference/urls.cue b/website/cue/reference/urls.cue index 485352e891fd5..836b1050114ad 100644 --- a/website/cue/reference/urls.cue +++ b/website/cue/reference/urls.cue @@ -112,9 +112,10 @@ urls: { cloudsmith: "https://cloudsmith.io/~timber/repos/vector/packages/" cloudsmith_apt: "https://cloudsmith.io/~timber/repos/vector/setup/#formats-deb" cloudsmith_yum: "https://cloudsmith.io/~timber/repos/vector/setup/#formats-rpm" + community_id_spec: "https://github.com/corelight/community-id-spec" console: "\(wikipedia)/wiki/System_console" - conventional_commits: "https://www.conventionalcommits.org" contributing: "\(vector_repo)/blob/master/CONTRIBUTING.md#setup" + conventional_commits: "https://www.conventionalcommits.org" crc: "\(wikipedia)/wiki/Cyclic_redundancy_check" ctime: "https://www.cplusplus.com/reference/ctime" cue: "https://cuelang.org/"