Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: Upgrade VRL to 0.3.0 #17325

Merged
merged 24 commits into from
May 5, 2023
173 changes: 69 additions & 104 deletions Cargo.lock

Large diffs are not rendered by default.

39 changes: 17 additions & 22 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,6 @@ portpicker = { path = "lib/portpicker" }
prometheus-parser = { path = "lib/prometheus-parser", optional = true }
opentelemetry-proto = { path = "lib/opentelemetry-proto", optional = true }
tracing-limit = { path = "lib/tracing-limit" }
value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
vector-api-client = { path = "lib/vector-api-client", optional = true }
vector-buffers = { path = "lib/vector-buffers", default-features = false }
vector-common = { path = "lib/vector-common" }
Expand All @@ -135,7 +134,6 @@ vector-config-common = { path = "lib/vector-config-common" }
vector-config-macros = { path = "lib/vector-config-macros" }
vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl"] }
vector-vrl-functions = { path = "lib/vector-vrl/functions" }
vrl-cli = { package = "vrl-cli", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", optional = true }
loki-logproto = { path = "lib/loki-logproto", optional = true }

# Tokio / Futures
Expand Down Expand Up @@ -223,14 +221,12 @@ number_prefix = { version = "0.4.0", default-features = false, features = ["std"
tui = { version = "0.19.0", optional = true, default-features = false, features = ["crossterm"] }

# Datadog Pipelines
datadog-filter = { package = "datadog-filter", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
datadog-search-syntax = { package = "datadog-search-syntax", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }

hex = { version = "0.4.3", default-features = false, optional = true }
sha2 = { version = "0.10.6", default-features = false, optional = true }

# VRL Lang
vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
vrl-stdlib = { package = "vrl-stdlib", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", features = ["cli"] }

# External libs
arc-swap = { version = "1.6", default-features = false, optional = true }
Expand Down Expand Up @@ -287,7 +283,7 @@ notify = { version = "5.1.0", default-features = false, features = ["macos_fseve
once_cell = { version = "1.17", default-features = false }
openssl = { version = "0.10.52", default-features = false, features = ["vendored"] }
openssl-probe = { version = "0.1.5", default-features = false }
ordered-float = { version = "3.6.0", default-features = false }
ordered-float = { version = "3.7.0", default-features = false }
paste = "1.0.12"
percent-encoding = { version = "2.2.0", default-features = false }
pin-project = { version = "1.0.12", default-features = false }
Expand Down Expand Up @@ -360,7 +356,6 @@ test-generator = "0.3.1"
tokio-test = "0.4.2"
tokio = { version = "1.28.0", features = ["test-util"] }
tower-test = "0.4.0"
value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", features = ["test"] }
vector-core = { path = "lib/vector-core", default-features = false, features = ["vrl", "test"] }
wiremock = "0.5.18"
zstd = { version = "0.12.3", default-features = false }
Expand All @@ -378,16 +373,16 @@ ntapi = { git = "https://github.com/MSxDOS/ntapi.git", rev = "24fc1e47677fc9f6e3

[features]
# Default features for *-unknown-linux-gnu and *-apple-darwin
default = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise", "component-validation-runner"]
default = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise", "component-validation-runner"]
# Default features for `cargo docs`. The same as `default` but without `rdkafka?/gssapi-vendored` which would require installing libsasl in our doc build environment.
docs = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise", "component-validation-runner"]
docs = ["api", "api-client", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise", "component-validation-runner"]
# Default features for *-unknown-linux-* which make use of `cmake` for dependencies
default-cmake = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"]
default-cmake = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"]
# Default features for *-pc-windows-msvc
# TODO: Enable SASL https://github.com/vectordotdev/vector/pull/3081#issuecomment-659298042
default-msvc = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "transforms", "vrl-cli", "enterprise"]
default-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"]
default-no-api-client = ["api", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"]
default-msvc = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "transforms", "enterprise"]
default-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"]
default-no-api-client = ["api", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"]
default-no-vrl-cli = ["api", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"]
tokio-console = ["dep:console-subscriber", "tokio/tracing"]

Expand All @@ -397,16 +392,16 @@ all-metrics = ["sinks-metrics", "sources-metrics", "transforms-metrics", "enterp
# Target specific release features.
# The `make` tasks will select this according to the appropriate triple.
# Use this section to turn off or on specific features for specific triples.
target-aarch64-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-aarch64-unknown-linux-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-armv7-unknown-linux-gnueabihf = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-armv7-unknown-linux-musleabihf = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "vrl-cli", "enterprise"]
target-x86_64-unknown-linux-gnu = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "vrl-cli", "enterprise"]
target-x86_64-unknown-linux-musl = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-aarch64-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]
target-aarch64-unknown-linux-musl = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]
target-armv7-unknown-linux-gnueabihf = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]
target-armv7-unknown-linux-musleabihf = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "enterprise"]
target-x86_64-unknown-linux-gnu = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "rdkafka?/gssapi-vendored", "enterprise"]
target-x86_64-unknown-linux-musl = ["api", "api-client", "rdkafka?/cmake_build", "enrichment-tables", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]
# Does not currently build
target-powerpc64le-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-powerpc64le-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]
# Currently doesn't build due to lack of support for 64-bit atomics
target-powerpc-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "vrl-cli", "enterprise"]
target-powerpc-unknown-linux-gnu = ["api", "api-client", "enrichment-tables", "rdkafka?/cmake_build", "sinks", "sources", "sources-dnstap", "transforms", "unix", "enterprise"]

# Enables features that work only on systems providing `cfg(unix)`
unix = ["tikv-jemallocator", "allocation-tracing"]
Expand Down
2 changes: 1 addition & 1 deletion benches/enrichment_tables.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ use std::{collections::BTreeMap, time::SystemTime};
use chrono::prelude::*;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use enrichment::Case;
use value::Value;
use vector::enrichment_tables::{
file::File,
geoip::{Geoip, GeoipConfig},
Condition, Table,
};
use vrl::value::Value;

criterion_group!(
name = benches;
Expand Down
2 changes: 1 addition & 1 deletion benches/transform/route.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ use criterion::{
black_box, criterion_group, measurement::WallTime, BatchSize, BenchmarkGroup, BenchmarkId,
Criterion, SamplingMode, Throughput,
};
use value::Value;
use vector::transforms::{
route::{Route, RouteConfig},
TransformOutputsBuf,
Expand All @@ -16,6 +15,7 @@ use vector_core::{
event::{Event, EventContainer, EventMetadata, LogEvent},
transform::{SyncTransform, TransformContext},
};
use vrl::value::Value;

#[derive(Debug)]
struct Param {
Expand Down
4 changes: 2 additions & 2 deletions lib/codecs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ dyn-clone = { version = "1", default-features = false }
lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false }
memchr = { version = "2", default-features = false }
once_cell = { version = "1.17", default-features = false }
ordered-float = { version = "3.6.0", default-features = false }
ordered-float = { version = "3.7.0", default-features = false }
prost = { version = "0.11.8", default-features = false, features = ["std"] }
regex = { version = "1.8.1", default-features = false, features = ["std", "perf"] }
serde = { version = "1", default-features = false, features = ["derive"] }
Expand All @@ -25,7 +25,7 @@ snafu = { version = "0.7.4", default-features = false, features = ["futures"] }
syslog_loose = { version = "0.18", default-features = false, optional = true }
tokio-util = { version = "0.7", default-features = false, features = ["codec"] }
tracing = { version = "0.1", default-features = false }
value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false }
vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = ["value"] }
vector-common = { path = "../vector-common", default-features = false }
vector-config = { path = "../vector-config", default-features = false }
vector-config-common = { path = "../vector-config-common", default-features = false }
Expand Down
4 changes: 2 additions & 2 deletions lib/codecs/src/decoding/format/bytes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ use lookup::lookup_v2::parse_value_path;
use lookup::OwnedTargetPath;
use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use value::Kind;
use vector_core::config::LogNamespace;
use vector_core::{
config::{log_schema, DataType},
event::{Event, LogEvent},
schema,
};
use vrl::value::Kind;

use super::Deserializer;

Expand Down Expand Up @@ -99,8 +99,8 @@ impl Deserializer for BytesDeserializer {

#[cfg(test)]
mod tests {
use value::Value;
use vector_core::config::log_schema;
use vrl::value::Value;

use super::*;

Expand Down
12 changes: 5 additions & 7 deletions lib/codecs/src/decoding/format/gelf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@ use lookup::{event_path, owned_value_path, PathPrefix};
use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use std::collections::HashMap;
use value::kind::Collection;
use value::Kind;
use vector_core::config::LogNamespace;
use vector_core::{
config::{log_schema, DataType},
event::Event,
event::LogEvent,
schema,
};
use vrl::value::kind::Collection;
use vrl::value::{Kind, Value};

use super::Deserializer;
use crate::{gelf_fields::*, VALID_FIELD_REGEX};
Expand Down Expand Up @@ -124,9 +124,7 @@ impl GelfDeserializer {
if let Some(line) = parsed.line {
log.insert(
LINE,
value::Value::Float(
ordered_float::NotNan::new(line).expect("JSON doesn't allow NaNs"),
),
Value::Float(ordered_float::NotNan::new(line).expect("JSON doesn't allow NaNs")),
);
}
if let Some(file) = &parsed.file {
Expand Down Expand Up @@ -156,7 +154,7 @@ impl GelfDeserializer {

// per GELF spec, Additional field values must be either strings or numbers
if val.is_string() || val.is_number() {
let vector_val: value::Value = val.into();
let vector_val: Value = val.into();
log.insert(event_path!(key.as_str()), vector_val);
} else {
let type_ = match val {
Expand Down Expand Up @@ -216,8 +214,8 @@ mod tests {
use serde_json::json;
use similar_asserts::assert_eq;
use smallvec::SmallVec;
use value::Value;
use vector_core::{config::log_schema, event::Event};
use vrl::value::Value;

fn deserialize_gelf_input(
input: &serde_json::Value,
Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/decoding/format/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,12 @@ use chrono::Utc;
use lookup::PathPrefix;
use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use value::Kind;
use vector_core::{
config::{log_schema, DataType, LogNamespace},
event::Event,
schema,
};
use vrl::value::Kind;

use super::Deserializer;

Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/decoding/format/native.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,13 @@ use bytes::Bytes;
use prost::Message;
use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use value::Kind;
use vector_core::config::LogNamespace;
use vector_core::{
config::DataType,
event::{proto, Event, EventArray, EventContainer},
schema,
};
use vrl::value::Kind;

use super::Deserializer;

Expand Down
4 changes: 2 additions & 2 deletions lib/codecs/src/decoding/format/native_json.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use bytes::Bytes;
use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use value::kind::Collection;
use value::Kind;
use vector_core::{config::DataType, event::Event, schema};
use vrl::value::kind::Collection;
use vrl::value::Kind;

use super::Deserializer;
use vector_core::config::LogNamespace;
Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/decoding/format/syslog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,13 @@ use serde::{Deserialize, Serialize};
use smallvec::{smallvec, SmallVec};
use std::collections::BTreeMap;
use syslog_loose::{IncompleteDate, Message, ProcId, Protocol};
use value::{kind::Collection, Kind};
use vector_core::config::{LegacyKey, LogNamespace};
use vector_core::{
config::{log_schema, DataType},
event::{Event, LogEvent, Value},
schema,
};
use vrl::value::{kind::Collection, Kind};

use super::Deserializer;

Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/avro.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,8 @@ impl Encoder<Event> for AvroSerializer {
mod tests {
use bytes::BytesMut;
use indoc::indoc;
use value::btreemap;
use vector_core::event::{LogEvent, Value};
use vrl::value::btreemap;

use super::*;

Expand Down
4 changes: 2 additions & 2 deletions lib/codecs/src/encoding/format/gelf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -240,9 +240,9 @@ mod tests {

use super::*;
use chrono::{DateTime, NaiveDateTime, Utc};
use value::btreemap;
use value::Value;
use vector_core::event::{Event, EventMetadata};
use vrl::value::btreemap;
use vrl::value::Value;

fn do_serialize(
expect_success: bool,
Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ impl Encoder<Event> for JsonSerializer {
mod tests {
use bytes::{Bytes, BytesMut};
use chrono::{TimeZone, Timelike, Utc};
use value::btreemap;
use vector_core::event::{LogEvent, Metric, MetricKind, MetricValue, StatisticKind, Value};
use vector_core::metric_tags;
use vrl::value::btreemap;

use super::*;

Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/logfmt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ impl Encoder<Event> for LogfmtSerializer {
mod tests {
use super::*;
use bytes::BytesMut;
use value::btreemap;
use vector_core::event::{LogEvent, Value};
use vrl::value::btreemap;

#[test]
fn serialize_logfmt() {
Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/native_json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ impl Encoder<Event> for NativeJsonSerializer {
#[cfg(test)]
mod tests {
use bytes::BytesMut;
use value::btreemap;
use vector_core::event::{LogEvent, Value};
use vrl::value::btreemap;

use super::*;

Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/raw_message.rs
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
use bytes::{BufMut, BytesMut};
use serde::{Deserialize, Serialize};
use tokio_util::codec::Encoder;
use value::Kind;
use vector_core::{
config::{log_schema, DataType},
event::Event,
schema,
};
use vrl::value::Kind;

/// Config used to build a `RawMessageSerializer`.
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
Expand Down
2 changes: 1 addition & 1 deletion lib/codecs/src/encoding/format/text.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
use bytes::{BufMut, BytesMut};
use tokio_util::codec::Encoder;
use value::Kind;
use vector_core::{
config::{log_schema, DataType},
event::Event,
schema,
};
use vrl::value::Kind;

use crate::MetricTagValues;

Expand Down
3 changes: 1 addition & 2 deletions lib/enrichment/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,4 @@ arc-swap = { version = "1.6.0", default-features = false }
dyn-clone = { version = "1.0.11", default-features = false }
chrono = { version = "0.4.19", default-features = false }
vector-common = { path = "../vector-common", default-features = false, features = [ "btreemap", "conversion", "serde" ] }
vrl = { package = "vrl", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
value = { package = "value", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false }
vrl = { git = "https://github.com/vectordotdev/vrl", rev = "v0.3.0", default-features = false, features = ["diagnostic"] }
Loading