chore(dev): Add wrapper for lookup in vector-lib (vectordotdev#18995

)
aholmberg · Oct 31, 2023 · 164f1e9 · 164f1e9
1 parent 1eb418b
commit 164f1e9
Show file tree

Hide file tree

Showing 103 changed files with 188 additions and 273 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -97,7 +97,6 @@ members = [
   "lib/k8s-e2e-tests",
   "lib/k8s-test-framework",
   "lib/loki-logproto",
-  "lib/vector-lookup",
   "lib/portpicker",
   "lib/prometheus-parser",
   "lib/opentelemetry-proto",
@@ -131,7 +130,6 @@ pin-project.workspace = true
 # Internal libs
 dnsmsg-parser = { path = "lib/dnsmsg-parser", optional = true }
 fakedata = { path = "lib/fakedata", optional = true }
-lookup = { package = "vector-lookup", path = "lib/vector-lookup" }
 portpicker = { path = "lib/portpicker" }
 tracing-limit = { path = "lib/tracing-limit" }
 vector-lib = { path = "lib/vector-lib", default-features = false, features = ["vrl"] }
@@ -357,7 +355,6 @@ libc = "0.2.149"
 similar-asserts = "1.5.0"
 proptest = "1.3"
 quickcheck = "1.0.3"
-lookup = { package = "vector-lookup", path = "lib/vector-lookup", features = ["test"] }
 reqwest = { version = "0.11", features = ["json"] }
 rstest = {version = "0.18.2"}
 tempfile = "3.8.1"

diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml
@@ -16,6 +16,7 @@ vector-buffers = { path = "../vector-buffers", default-features = false }
 vector-common = { path = "../vector-common" }
 vector-config = { path = "../vector-config" }
 vector-core = { path = "../vector-core", default-features = false }
+vector-lookup = { path = "../vector-lookup", features = ["test"] }
 vector-stream = { path = "../vector-stream" }
 
 [features]

diff --git a/lib/vector-lib/src/lib.rs b/lib/vector-lib/src/lib.rs
@@ -20,6 +20,7 @@ pub use vector_core::{
     buckets, default_data_dir, event, fanout, metric_tags, metrics, partition, quantiles, samples,
     schema, serde, sink, source, tcp, tls, transform, update_counter, EstimatedJsonEncodedSizeOf,
 };
+pub use vector_lookup as lookup;
 pub use vector_stream as stream;
 
 pub mod config {

diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs
@@ -154,7 +154,7 @@ where
 
 #[cfg(test)]
 mod test {
-    use lookup::lookup_v2::{parse_value_path, ConfigValuePath};
+    use vector_lib::lookup::lookup_v2::{parse_value_path, ConfigValuePath};
 
     use super::*;
     use crate::codecs::encoding::TimestampFormat;

diff --git a/src/codecs/encoding/transformer.rs b/src/codecs/encoding/transformer.rs
@@ -4,12 +4,12 @@ use chrono::{DateTime, Utc};
 use core::fmt::Debug;
 use std::collections::BTreeMap;
 
-use lookup::lookup_v2::ConfigValuePath;
-use lookup::{event_path, PathPrefix};
 use ordered_float::NotNan;
 use serde::{Deserialize, Deserializer};
 use vector_lib::configurable::configurable_component;
 use vector_lib::event::{LogEvent, MaybeAsLogMut};
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
+use vector_lib::lookup::{event_path, PathPrefix};
 use vector_lib::schema::meaning;
 use vrl::path::OwnedValuePath;
 use vrl::value::Value;
@@ -266,9 +266,9 @@ pub enum TimestampFormat {
 #[cfg(test)]
 mod tests {
     use indoc::indoc;
-    use lookup::path::parse_target_path;
     use vector_lib::btreemap;
     use vector_lib::config::{log_schema, LogNamespace};
+    use vector_lib::lookup::path::parse_target_path;
     use vrl::value::Kind;
 
     use crate::config::schema;
@@ -375,10 +375,7 @@ mod tests {
         let mut base = Event::Log(LogEvent::from("Demo"));
         let timestamp = base
             .as_mut_log()
-            .get((
-                lookup::PathPrefix::Event,
-                log_schema().timestamp_key().unwrap(),
-            ))
+            .get((PathPrefix::Event, log_schema().timestamp_key().unwrap()))
             .unwrap()
             .clone();
         let timestamp = timestamp.as_timestamp().unwrap();
@@ -407,11 +404,8 @@ mod tests {
 
             for actual in [
                 // original key
-                log.get((
-                    lookup::PathPrefix::Event,
-                    log_schema().timestamp_key().unwrap(),
-                ))
-                .unwrap(),
+                log.get((PathPrefix::Event, log_schema().timestamp_key().unwrap()))
+                    .unwrap(),
                 // second key
                 log.get("another").unwrap(),
             ] {

diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs
@@ -6,6 +6,7 @@ use chrono::Duration;
 use futures::{stream, StreamExt};
 use similar_asserts::assert_eq;
 use vector_lib::codecs::TextSerializerConfig;
+use vector_lib::lookup;
 
 use super::*;
 use crate::aws::create_client;

diff --git a/src/sinks/aws_kinesis/config.rs b/src/sinks/aws_kinesis/config.rs
@@ -1,5 +1,5 @@
-use lookup::lookup_v2::ConfigValuePath;
 use std::marker::PhantomData;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 
 use vector_lib::stream::BatcherSettings;
 

diff --git a/src/sinks/aws_kinesis/sink.rs b/src/sinks/aws_kinesis/sink.rs
@@ -1,7 +1,7 @@
 use std::{borrow::Cow, fmt::Debug, marker::PhantomData};
 
-use lookup::lookup_v2::ConfigValuePath;
 use rand::random;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vrl::path::PathPrefix;
 
 use crate::{

diff --git a/src/sinks/aws_kinesis/streams/config.rs b/src/sinks/aws_kinesis/streams/config.rs
@@ -3,9 +3,9 @@ use aws_sdk_kinesis::{
     types::SdkError,
 };
 use futures::FutureExt;
-use lookup::lookup_v2::ConfigValuePath;
 use snafu::Snafu;
 use vector_lib::configurable::{component::GenerateConfig, configurable_component};
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 
 use crate::sinks::util::retries::RetryAction;
 use crate::{

diff --git a/src/sinks/azure_monitor_logs/config.rs b/src/sinks/azure_monitor_logs/config.rs
@@ -1,5 +1,5 @@
-use lookup::{lookup_v2::OptionalValuePath, OwnedValuePath};
 use openssl::{base64, pkey};
+use vector_lib::lookup::{lookup_v2::OptionalValuePath, OwnedValuePath};
 
 use vector_lib::configurable::configurable_component;
 use vector_lib::sensitive_string::SensitiveString;

diff --git a/src/sinks/azure_monitor_logs/service.rs b/src/sinks/azure_monitor_logs/service.rs
@@ -4,12 +4,12 @@ use http::{
     HeaderName, HeaderValue, Request, StatusCode, Uri,
 };
 use hyper::Body;
-use lookup::lookup_v2::OwnedValuePath;
 use once_cell::sync::Lazy;
 use openssl::{base64, hash, pkey, sign};
 use regex::Regex;
 use std::task::{Context, Poll};
 use tracing::Instrument;
+use vector_lib::lookup::lookup_v2::OwnedValuePath;
 
 use crate::{http::HttpClient, sinks::prelude::*};
 

diff --git a/src/sinks/azure_monitor_logs/sink.rs b/src/sinks/azure_monitor_logs/sink.rs
@@ -1,8 +1,8 @@
 use std::{fmt::Debug, io};
 
 use bytes::Bytes;
-use lookup::{OwnedValuePath, PathPrefix};
 use vector_lib::codecs::{encoding::Framer, CharacterDelimitedEncoder, JsonSerializerConfig};
+use vector_lib::lookup::{OwnedValuePath, PathPrefix};
 
 use crate::sinks::prelude::*;
 

diff --git a/src/sinks/clickhouse/integration_tests.rs b/src/sinks/clickhouse/integration_tests.rs
@@ -16,6 +16,7 @@ use serde::Deserialize;
 use serde_json::Value;
 use tokio::time::{timeout, Duration};
 use vector_lib::event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event, LogEvent};
+use vector_lib::lookup::PathPrefix;
 use warp::Filter;
 
 use super::*;
@@ -171,10 +172,7 @@ async fn insert_events_unix_timestamps() {
 
     let exp_event = input_event.as_mut_log();
     exp_event.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         format!(
             "{}",
             exp_event
@@ -232,10 +230,7 @@ timestamp_format = "unix""#,
 
     let exp_event = input_event.as_mut_log();
     exp_event.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         format!(
             "{}",
             exp_event

diff --git a/src/sinks/datadog/events/request_builder.rs b/src/sinks/datadog/events/request_builder.rs
@@ -1,8 +1,8 @@
 use std::{io, sync::Arc};
 
 use bytes::Bytes;
-use lookup::lookup_v2::ConfigValuePath;
 use vector_lib::codecs::JsonSerializerConfig;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vector_lib::request_metadata::{MetaDescriptive, RequestMetadata};
 use vector_lib::ByteSizeOf;
 

diff --git a/src/sinks/datadog/events/sink.rs b/src/sinks/datadog/events/sink.rs
@@ -1,6 +1,6 @@
 use std::fmt;
 
-use lookup::event_path;
+use vector_lib::lookup::event_path;
 
 use crate::{
     internal_events::{ParserMissingFieldError, DROP_EVENT},

diff --git a/src/sinks/datadog/logs/sink.rs b/src/sinks/datadog/logs/sink.rs
@@ -1,9 +1,9 @@
 use std::{fmt::Debug, io, sync::Arc};
 
 use bytes::Bytes;
-use lookup::event_path;
 use snafu::Snafu;
 use vector_lib::codecs::{encoding::Framer, CharacterDelimitedEncoder, JsonSerializerConfig};
+use vector_lib::lookup::event_path;
 
 use super::{config::MAX_PAYLOAD_BYTES, service::LogApiRequest};
 use crate::sinks::{

diff --git a/src/sinks/elasticsearch/config.rs b/src/sinks/elasticsearch/config.rs
@@ -31,8 +31,8 @@ use crate::{
     tls::TlsConfig,
     transforms::metric_to_log::MetricToLogConfig,
 };
-use lookup::event_path;
-use lookup::lookup_v2::ConfigValuePath;
+use vector_lib::lookup::event_path;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vector_lib::schema::Requirement;
 use vrl::value::Kind;
 

diff --git a/src/sinks/elasticsearch/sink.rs b/src/sinks/elasticsearch/sink.rs
@@ -1,6 +1,6 @@
 use std::fmt;
 
-use lookup::lookup_v2::ConfigValuePath;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vrl::path::PathPrefix;
 
 use crate::{

diff --git a/src/sinks/elasticsearch/tests.rs b/src/sinks/elasticsearch/tests.rs
@@ -1,5 +1,7 @@
 use std::{collections::BTreeMap, convert::TryFrom};
 
+use vector_lib::lookup::PathPrefix;
+
 use crate::{
     codecs::Transformer,
     event::{LogEvent, Metric, MetricKind, MetricValue, Value},
@@ -37,10 +39,7 @@ async fn sets_create_action_when_configured() {
 
     let mut log = LogEvent::from("hello there");
     log.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         Utc.with_ymd_and_hms(2020, 12, 1, 1, 2, 3)
             .single()
             .expect("invalid timestamp"),
@@ -106,10 +105,7 @@ async fn encode_datastream_mode() {
 
     let mut log = LogEvent::from("hello there");
     log.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         Utc.with_ymd_and_hms(2020, 12, 1, 1, 2, 3)
             .single()
             .expect("invalid timestamp"),
@@ -173,10 +169,7 @@ async fn encode_datastream_mode_no_routing() {
         ),
     );
     log.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         Utc.with_ymd_and_hms(2020, 12, 1, 1, 2, 3)
             .single()
             .expect("invalid timestamp"),
@@ -324,10 +317,7 @@ async fn encode_datastream_mode_no_sync() {
         ),
     );
     log.insert(
-        (
-            lookup::PathPrefix::Event,
-            log_schema().timestamp_key().unwrap(),
-        ),
+        (PathPrefix::Event, log_schema().timestamp_key().unwrap()),
         Utc.with_ymd_and_hms(2020, 12, 1, 1, 2, 3)
             .single()
             .expect("invalid timestamp"),

diff --git a/src/sinks/gcp/stackdriver/logs/config.rs b/src/sinks/gcp/stackdriver/logs/config.rs
@@ -15,9 +15,9 @@ use crate::{
 };
 use http::{Request, Uri};
 use hyper::Body;
-use lookup::lookup_v2::ConfigValuePath;
 use snafu::Snafu;
 use std::collections::HashMap;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vrl::value::Kind;
 
 use super::{

diff --git a/src/sinks/gcp/stackdriver/logs/encoder.rs b/src/sinks/gcp/stackdriver/logs/encoder.rs
@@ -3,8 +3,8 @@
 use std::{collections::HashMap, io};
 
 use bytes::BytesMut;
-use lookup::lookup_v2::ConfigValuePath;
 use serde_json::{json, to_vec, Map};
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 use vrl::path::PathPrefix;
 
 use crate::{

diff --git a/src/sinks/gcp/stackdriver/logs/tests.rs b/src/sinks/gcp/stackdriver/logs/tests.rs
@@ -5,9 +5,9 @@ use chrono::{TimeZone, Utc};
 use futures::{future::ready, stream};
 use http::Uri;
 use indoc::indoc;
-use lookup::lookup_v2::ConfigValuePath;
 use serde::Deserialize;
 use std::collections::HashMap;
+use vector_lib::lookup::lookup_v2::ConfigValuePath;
 
 use crate::{
     config::{GenerateConfig, SinkConfig, SinkContext},

diff --git a/src/sinks/humio/logs.rs b/src/sinks/humio/logs.rs
@@ -1,6 +1,6 @@
-use lookup::lookup_v2::{ConfigValuePath, OptionalValuePath};
 use vector_lib::codecs::JsonSerializerConfig;
 use vector_lib::configurable::configurable_component;
+use vector_lib::lookup::lookup_v2::{ConfigValuePath, OptionalValuePath};
 use vector_lib::sensitive_string::SensitiveString;
 
 use super::config_host_key;

diff --git a/src/sinks/humio/metrics.rs b/src/sinks/humio/metrics.rs
@@ -2,9 +2,10 @@ use async_trait::async_trait;
 use futures::StreamExt;
 use futures_util::stream::BoxStream;
 use indoc::indoc;
-use lookup::lookup_v2::{ConfigValuePath, OptionalValuePath};
 use vector_lib::codecs::JsonSerializerConfig;
 use vector_lib::configurable::configurable_component;
+use vector_lib::lookup;
+use vector_lib::lookup::lookup_v2::{ConfigValuePath, OptionalValuePath};
 use vector_lib::sensitive_string::SensitiveString;
 use vector_lib::sink::StreamSink;
 

diff --git a/src/sinks/humio/mod.rs b/src/sinks/humio/mod.rs
@@ -1,4 +1,4 @@
-use lookup::lookup_v2::OptionalValuePath;
+use vector_lib::lookup::lookup_v2::OptionalValuePath;
 
 pub mod logs;
 pub mod metrics;