bring back deny_unknown_fields + other small tweaks

Signed-off-by: Toby Lawrence <toby@nuclearfurnace.com>

lib/vector-config-macros/src/ast/container.rs

@@ -15,7 +15,6 @@ use super::{
 };
 
 const ERR_NO_ENUM_TUPLES: &str = "enum variants cannot be tuples (multiple unnamed fields)";
-const ERR_NO_ENUM_NEWTYPE_INTERNAL_TAG: &str = "newtype variants (i.e. `enum SomeEnum { SomeVariant(T) }`) cannot be used with tag-only mode as the type inside may or may not support embedding the tag field";
 const ERR_NO_ENUM_VARIANT_DESCRIPTION: &str = "enum variants must have a description i.e. `/// This is a description` or `#[configurable(description = \"This is a description...\")]`";
 const ERR_ENUM_UNTAGGED_DUPLICATES: &str = "enum variants must be unique in style/shape when in untagged mode i.e. there cannot be multiple unit variants, or tuple variants with the same fields, etc";
 const ERR_NO_UNIT_STRUCTS: &str = "unit structs are not supported by `Configurable`";

lib/vector-config-macros/src/lib.rs

@@ -1,13 +1,38 @@
-// TODO: Remove this once we add validation since that's the only piece of dead code in this crate at the moment.
-#![allow(dead_code)]
-
 use proc_macro::TokenStream;
 
 mod ast;
 mod configurable;
 mod configurable_component;
 
 /// Designates a type as being part of a Vector configuration.
+///
+/// This will automatically derive the [`Configurable`][vector_config::Configurable] trait for the given struct/enum, as
+/// well as ensuring that serialization/deserialization (via `serde`) is derived.
+///
+/// ### Examples
+///
+/// In its most basic form, this attribute macro can be used to simply derive the aforementioned traits, making it using
+/// in any other type also deriving `Configurable`:
+///
+/// ```norun
+/// #[configurable_component]
+/// pub struct Something {
+///   ...
+/// }
+/// ```
+///
+/// Additionally, callers can specify the component type, when being used directly on the top-level configuration object
+/// for a component by specifying the component type (`source`, `transform`, or `sink`) as the sole parameter:
+///
+/// ```norun
+/// #[configurable_component(source)]
+/// pub struct KafkaSourceConfig {
+///   ...
+/// }
+/// ```
+///
+/// This adds special metadata to the generated schema for that type indicating that it represents the configuration of
+/// a component of the specified type.
 #[proc_macro_attribute]
 pub fn configurable_component(args: TokenStream, item: TokenStream) -> TokenStream {
     configurable_component::configurable_component_impl(args, item)

lib/vector-config/src/num.rs

@@ -7,10 +7,6 @@ use serde_json::Number;
 use vector_config_common::num::{NUMERIC_ENFORCED_LOWER_BOUND, NUMERIC_ENFORCED_UPPER_BOUND};
 
 /// A numeric type that can be represented correctly in a JSON Schema document.
-///
-/// `N` must be an integral numeric type i.e. `f64`, `u8`, `i32`, and so on. The numeric type is parameterized in this
-/// way to allow generating the schema for wrapper types such as `NonZeroU64`, where the overall type must be
-/// represented as `NonZeroU64` but the integeral numeric type that we're constraining against is `u64`.
 pub trait ConfigurableNumber {
     /// The integral numeric type.
     ///

lib/vector-config/src/schema.rs

@@ -170,7 +170,7 @@ where
         ..Default::default()
     };
 
-    // If the actual numeric type we're generating the schema for is a nonzero variant, and its constrain can't be
+    // If the actual numeric type we're generating the schema for is a nonzero variant, and its constraint can't be
     // represently solely by the normal minimum/maximum bounds, we explicitly add an exclusion for the appropriate zero
     // value of the given numeric type.
     if N::requires_nonzero_exclusion() {

lib/vector-core/src/config/mod.rs

@@ -141,11 +141,11 @@ impl Output {
     }
 }
 
-/// Acknowledgement configuration.
+/// Configuration of acknowledgement behavior.
 #[configurable_component]
 #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
 pub struct AcknowledgementsConfig {
-    /// Whether or not acknowledgements should be enabled.
+    /// Enables end-to-end acknowledgements.
     enabled: Option<bool>,
 }
 

src/aws/auth.rs

@@ -24,26 +24,29 @@ pub enum AwsAuthentication {
         /// The AWS secret access key.
         secret_access_key: String,
     },
+
     /// Authenticate using credentials stored in a file.
     ///
-    /// Optionally, specifies a credentials profile to use.
+    /// Additionally, the specific credential profile to use can be set.
     File {
         /// Path to the credentials file.
         credentials_file: String,
         /// The credentials profile to use.
         profile: Option<String>,
     },
-    /// Assumes the given role ARN.
+
+    /// Assume the given role ARN.
     Role {
         /// The ARN of the role to assume.
         assume_role: String,
         /// Timeout for assuming the role, in seconds.
         load_timeout_secs: Option<u64>,
     },
-    /// Default authentication strategy which tries a variety of substrategies in a chained fallback fashion.
+
+    /// Default authentication strategy which tries a variety of substrategies in a one-after-the-other fashion.
     #[derivative(Default)]
     Default {
-        /// Timeout for successfully loading credentials, in seconds.
+        /// Timeout for successfully loading any credentials, in seconds.
         load_timeout_secs: Option<u64>,
     },
 }

src/aws/region.rs

@@ -3,15 +3,16 @@ use std::str::FromStr;
 use aws_smithy_http::endpoint::Endpoint;
 use aws_types::region::Region;
 use http::Uri;
-use serde::{Deserialize, Serialize};
-use vector_config::Configurable;
+use vector_config::configurable_component;
 
-/// The region/endpoint configuration for interacting with an AWS service.
-#[derive(Clone, Configurable, Debug, Default, Deserialize, PartialEq, Serialize)]
+/// Configuration of the region/endpoint to use when interacting with an AWS service.
+#[configurable_component]
+#[derive(Clone, Debug, Default, PartialEq)]
 #[serde(default)]
 pub struct RegionOrEndpoint {
     /// The AWS region to use.
     pub region: Option<String>,
+
     /// The API endpoint of the service.
     pub endpoint: Option<String>,
 }

src/docker.rs

@@ -21,21 +21,22 @@ pub enum Error {
     NoHost,
 }
 
-/// TLS options to connect to the Docker daemon.
+/// Configuration of TLS when connecting to the Docker daemon.
 ///
 /// Only relevant when connecting to Docker via an HTTPS URL.
 ///
 /// If not configured, Vector will try to use environment variable `DOCKER_CERT_PATH` and then` DOCKER_CONFIG`. If both environment variables are absent, Vector will try to read certificates in `~/.docker/`.
 #[configurable_component]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub struct DockerTlsConfig {
-    /// Path to CA certificate file.
+    /// Path to the CA certificate file.
     ca_file: PathBuf,
 
-    /// Path to TLS certificate file.
+    /// Path to the TLS certificate file.
     crt_file: PathBuf,
 
-    /// Path to TLS key file.
+    /// Path to the TLS key file.
     key_file: PathBuf,
 }
 

src/gcp.rs

@@ -52,7 +52,7 @@ pub enum GcpError {
     BuildHttpClient { source: HttpError },
 }
 
-/// Authentication configuration for GCP services.
+/// Configuration of the authentication strategy for interacting with GCP services.
 // TODO: We're duplicating the "either this or that" verbiage for each field because this struct gets flattened into the
 // component config types, which means all that's carried over are the fields, not the type itself.
 //

src/http.rs

@@ -209,13 +209,13 @@ impl<B> fmt::Debug for HttpClient<B> {
     }
 }
 
-/// Authentication strategy for requests.
+/// Configuration of the authentication strategy for HTTP requests.
 ///
 /// HTTP authentication should almost always be used with HTTPS only, as the authentication credentials are passed as an
 /// HTTP header without any additional encryption beyond what is provided by the transport itself.
 #[configurable_component]
 #[derive(Clone, Debug, Eq, PartialEq)]
-#[serde(rename_all = "snake_case", tag = "strategy")]
+#[serde(deny_unknown_fields, rename_all = "snake_case", tag = "strategy")]
 pub enum Auth {
     /// Basic authentication.
     ///

src/kafka.rs

@@ -36,20 +36,28 @@ pub(crate) struct KafkaAuthConfig {
     pub(crate) tls: Option<TlsEnableableConfig>,
 }
 
-/// /// Options for SASL/SCRAM authentication support.
+/// Configuration for SASL authentication when interacting with Kafka.
 #[configurable_component]
 #[derive(Clone, Debug, Default)]
 pub(crate) struct KafkaSaslConfig {
-    /// Enable SASL/SCRAM authentication to the remote (not supported on Windows at this time).
+    /// Enables SASL authentication.
+    ///
+    /// Only `PLAIN` and `SCRAM`-based mechanisms are supported when configuring SASL authentication via `sasl.*`. For
+    /// other mechanisms, `librdkafka_options.*` must be used directly to configure other `librdkafka`-specific values
+    /// i.e. `sasl.kerberos.*` and so on.
+    ///
+    /// See the [librdkafka documentation](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for details.
+    ///
+    /// SASL authentication is not supported on Windows.
     pub(crate) enabled: Option<bool>,
 
-    /// The Kafka SASL/SCRAM authentication username.
+    /// The SASL username.
     pub(crate) username: Option<String>,
 
-    /// The Kafka SASL/SCRAM authentication password.
+    /// The SASL password.
     pub(crate) password: Option<String>,
 
-    /// The Kafka SASL/SCRAM mechanisms.
+    /// The SASL mechanism to use.
     pub(crate) mechanism: Option<String>,
 }
 

src/line_agg.rs

@@ -14,12 +14,12 @@ use bytes::{Bytes, BytesMut};
 use futures::{Stream, StreamExt};
 use pin_project::pin_project;
 use regex::bytes::Regex;
-use serde::{Deserialize, Serialize};
 use tokio_util::time::delay_queue::{DelayQueue, Key};
-use vector_config::Configurable;
+use vector_config::configurable_component;
 
-/// The mode of operation of the line aggregator.
-#[derive(Clone, Configurable, Copy, Debug, Hash, Deserialize, PartialEq, Serialize)]
+/// Mode of operation of the line aggregator.
+#[configurable_component]
+#[derive(Clone, Copy, Debug, Hash, PartialEq)]
 #[serde(rename_all = "snake_case")]
 pub enum Mode {
     /// All consecutive lines matching this pattern are included in the group.
@@ -30,7 +30,7 @@ pub enum Mode {
     /// whitespace) indicates that it is an extension of the proceeding line.
     ContinueThrough,
 
-    /// All consecutive lines matching this pattern, plus one additional line,  are included in the group.
+    /// All consecutive lines matching this pattern, plus one additional line, are included in the group.
     ///
     /// This is useful in cases where a log message ends with a continuation marker, such as a backslash, indicating
     /// that the following line is part of the same message.
@@ -47,21 +47,23 @@ pub enum Mode {
     HaltWith,
 }
 
-/// Configuration parameters of the line aggregator.
-#[derive(Debug, Clone)]
+/// Configuration of multi-line aggregation.
+#[derive(Clone, Debug)]
 pub struct Config {
-    /// The regular expression pattern for detecting the beginning of the message.
+    /// Regular expression pattern that is used to match the start of a new message.
     pub start_pattern: Regex,
-    /// The regular expression pattern used for evaluating whether the current line should be aggregated or if
-    /// aggregation should stop.
+
+    /// Regular expression pattern that is used to determine whether or not more lines should be read.
     ///
-    /// Configured in tandem with `mode` to define the overall aggregation behavior.
+    /// This setting must be configured in conjunction with `mode`.
     pub condition_pattern: Regex,
-    /// The mode of aggregation.
+
+    /// Aggregation mode.
     ///
-    /// Configured in tandem with `condition_pattern` to define the overall aggregation behavior.
+    /// This setting must be configured in conjunction with `condition_pattern`.
     pub mode: Mode,
-    /// The maximum time to wait for subsequent lines to be received and evaluated for aggregation.
+
+    /// The maximum amount of time to wait for the next additional line, in milliseconds.
     ///
     /// Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete.
     pub timeout: Duration,

src/nats.rs

@@ -14,7 +14,7 @@ pub enum NatsConfigError {
     TlsMissingCert,
 }
 
-/// Configuration for how Vector should authenticate to NATS.
+/// Configuration of the authentication strategy when interacting with NATS.
 #[configurable_component]
 #[derive(Clone, Debug)]
 #[serde(rename_all = "snake_case", tag = "strategy")]
@@ -64,6 +64,7 @@ impl std::fmt::Display for NatsAuthConfig {
 /// Username and password configuration.
 #[configurable_component]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub(crate) struct NatsAuthUserPassword {
     /// Username.
     pub(crate) user: String,
@@ -75,6 +76,7 @@ pub(crate) struct NatsAuthUserPassword {
 /// Token configuration.
 #[configurable_component]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub(crate) struct NatsAuthToken {
     /// Token.
     pub(crate) value: String,
@@ -83,6 +85,7 @@ pub(crate) struct NatsAuthToken {
 /// Credentials file configuration.
 #[configurable_component]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub(crate) struct NatsAuthCredentialsFile {
     /// Path to credentials file.
     pub(crate) path: String,
@@ -91,15 +94,16 @@ pub(crate) struct NatsAuthCredentialsFile {
 /// NKeys configuration.
 #[configurable_component]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub(crate) struct NatsAuthNKey {
     /// User.
     ///
-    /// This is equivalent to a public key.
+    /// Conceptually, this is equivalent to a public key.
     pub(crate) nkey: String,
 
     /// Seed.
     ///
-    /// This is equivalent to a private key.
+    /// Conceptually, this is equivalent to a private key.
     pub(crate) seed: String,
 }
 

src/sources/aws_ecs_metrics/mod.rs

@@ -47,6 +47,7 @@ pub enum Version {
 /// Configuration for the `aws_ecs_metrics` source.
 #[configurable_component(source)]
 #[derive(Clone, Debug)]
+#[serde(deny_unknown_fields)]
 pub struct AwsEcsMetricsSourceConfig {
     /// Base URI of the task metadata endpoint.
     ///

src/sources/aws_s3/mod.rs

@@ -68,7 +68,7 @@ enum Strategy {
 // Maybe showing defaults at all, when there are required properties, doesn't actually make sense? :thinkies:
 #[configurable_component(source)]
 #[derive(Clone, Debug, Default)]
-#[serde(default)]
+#[serde(default, deny_unknown_fields)]
 pub struct AwsS3Config {
     #[serde(flatten)]
     region: RegionOrEndpoint,
@@ -92,9 +92,9 @@ pub struct AwsS3Config {
     #[serde(default)]
     auth: AwsAuthentication,
 
-    /// Multiline parsing configuration.
+    /// Multiline aggregation configuration.
     ///
-    /// If not specified, multiline parsing is disabled.
+    /// If not specified, multiline aggregation is disabled.
     multiline: Option<MultilineConfig>,
 
     #[configurable(derived)]

src/sources/aws_s3/sqs.rs

@@ -47,6 +47,7 @@ static SUPPORTED_S3S_EVENT_VERSION: Lazy<semver::VersionReq> =
 #[configurable_component]
 #[derive(Clone, Debug, Derivative)]
 #[derivative(Default)]
+#[serde(deny_unknown_fields)]
 pub(super) struct Config {
     /// The URL of the SQS queue to poll for bucket notifications.
     pub(super) queue_url: String,

src/sources/aws_sqs/config.rs

@@ -18,6 +18,7 @@ use crate::{
 #[configurable_component(source)]
 #[derive(Clone, Debug, Derivative)]
 #[derivative(Default)]
+#[serde(deny_unknown_fields)]
 pub struct AwsSqsConfig {
     #[serde(flatten)]
     pub region: RegionOrEndpoint,