chore(config): improve config schema output with more precise unevaluatedProperties + schema ref flattening

lib/vector-config-common/Cargo.toml

@@ -11,4 +11,5 @@ proc-macro2 = { version = "1.0", default-features = false }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0", default-features = false, features = ["std"] }
 syn = { version = "1.0", features = ["full", "extra-traits", "visit-mut", "visit"] }
+tracing = { version = "0.1.34", default-features = false }
 quote = { version = "1.0", default-features = false }

lib/vector-config-common/src/schema/json_schema.rs

@@ -1,8 +1,8 @@
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
-use std::ops::Deref;
+use std::{iter, ops::Deref};
 
-use super::{Map, Set};
+use super::{Map, Set, DEFINITIONS_PREFIX};
 
 /// A JSON Schema.
 #[allow(clippy::large_enum_variant)]
@@ -40,6 +40,26 @@ impl Schema {
         }
     }
 
+    /// Gets a reference to the inner schema object if this schema is a JSON Schema object.
+    ///
+    /// Otherwise, `None` is returned.
+    pub fn as_object(&self) -> Option<&SchemaObject> {
+        match self {
+            Schema::Object(schema) => Some(schema),
+            _ => None,
+        }
+    }
+
+    /// Gets a mutable reference to the inner schema object if this schema is a JSON Schema object.
+    ///
+    /// Otherwise, `None` is returned.
+    pub fn as_object_mut(&mut self) -> Option<&mut SchemaObject> {
+        match self {
+            Schema::Object(schema) => Some(schema),
+            _ => None,
+        }
+    }
+
     /// Converts the given schema (if it is a boolean schema) into an equivalent schema object.
     ///
     /// If the given schema is already a schema object, this has no effect.
@@ -539,6 +559,30 @@ pub enum SingleOrVec<T> {
     Vec(Vec<T>),
 }
 
+impl<T: Clone> Extend<T> for SingleOrVec<T> {
+    fn extend<I: IntoIterator<Item = T>>(&mut self, iter: I) {
+        match self {
+            Self::Single(item) => {
+                *self = Self::Vec(iter::once(*item.clone()).chain(iter.into_iter()).collect());
+            }
+            Self::Vec(items) => items.extend(iter),
+        }
+    }
+}
+
+impl<'a, T> IntoIterator for &'a SingleOrVec<T> {
+    type Item = &'a T;
+
+    type IntoIter = std::slice::Iter<'a, T>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            SingleOrVec::Single(item) => std::slice::from_ref(item.as_ref()).iter(),
+            SingleOrVec::Vec(items) => items.as_slice().iter(),
+        }
+    }
+}
+
 impl<T> From<T> for SingleOrVec<T> {
     fn from(single: T) -> Self {
         SingleOrVec::Single(Box::new(single))
@@ -567,3 +611,14 @@ fn is_none_or_default_true(field: &Option<Box<Schema>>) -> bool {
         Some(value) => matches!(value.as_ref(), Schema::Bool(true)),
     }
 }
+
+pub fn get_cleaned_schema_reference(schema_ref: &str) -> &str {
+    if let Some(cleaned) = schema_ref.strip_prefix(DEFINITIONS_PREFIX) {
+        cleaned
+    } else {
+        panic!(
+            "Tried to clean schema reference that does not start with the definition prefix: {}",
+            schema_ref
+        );
+    }
+}

lib/vector-config-common/src/schema/visit.rs

@@ -1,4 +1,6 @@
-use super::{Map, RootSchema, Schema, SchemaObject, SingleOrVec, DEFINITIONS_PREFIX};
+use tracing::debug;
+
+use super::{get_cleaned_schema_reference, Map, RootSchema, Schema, SchemaObject, SingleOrVec};
 
 /// Trait used to recursively modify a constructed schema and its subschemas.
 pub trait Visitor: std::fmt::Debug {
@@ -51,9 +53,15 @@ pub fn visit_schema_object<V: Visitor + ?Sized>(
     schema: &mut SchemaObject,
 ) {
     if schema.reference.is_some() {
-        with_resolved_schema_reference(definitions, schema, |defs, referenced_schema| {
-            v.visit_schema(defs, referenced_schema);
-        })
+        with_resolved_schema_reference(
+            definitions,
+            schema,
+            |defs, schema_ref, referenced_schema| {
+                debug!(referent = schema_ref, "Visiting schema reference.");
+
+                v.visit_schema(defs, referenced_schema);
+            },
+        )
     }
 
     if let Some(sub) = &mut schema.subschemas {
@@ -133,17 +141,17 @@ pub fn with_resolved_schema_reference<F>(
     schema: &mut SchemaObject,
     f: F,
 ) where
-    F: FnOnce(&mut Map<String, Schema>, &mut Schema),
+    F: FnOnce(&mut Map<String, Schema>, &str, &mut Schema),
 {
     if let Some(reference) = schema.reference.as_ref() {
-        let schema_def_key = reference.replace(DEFINITIONS_PREFIX, "");
+        let schema_def_key = get_cleaned_schema_reference(reference);
         let mut referenced_schema = definitions
-            .get(&schema_def_key)
+            .get(schema_def_key)
             .cloned()
             .expect("schema reference should exist");
 
-        f(definitions, &mut referenced_schema);
+        f(definitions, schema_def_key, &mut referenced_schema);
 
-        definitions.insert(schema_def_key, referenced_schema);
+        definitions.insert(schema_def_key.to_string(), referenced_schema);
     }
 }

lib/vector-config/Cargo.toml

@@ -23,6 +23,7 @@ serde_json = { version = "1.0", default-features = false, features = ["std"] }
 serde_with = { version = "2.3.1", default-features = false, features = ["std"] }
 snafu = { version = "0.7.4", default-features = false }
 toml = { version = "0.7.3", default-features = false }
+tracing = { version = "0.1.34", default-features = false }
 url = { version = "2.3.1", default-features = false, features = ["serde"] }
 vrl-compiler = { package = "vrl-compiler", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0" }
 vrl-core = { package = "vrl-core", git = "https://github.com/vectordotdev/vrl", rev = "v0.2.0", default-features = false, features = ["serde"] }

lib/vector-config/src/schema/helpers.rs

@@ -12,7 +12,7 @@ use crate::{
     num::ConfigurableNumber, Configurable, ConfigurableRef, GenerateError, Metadata, ToValue,
 };
 
-use super::visitors::DisallowedUnevaluatedPropertiesVisitor;
+use super::visitors::{DisallowedUnevaluatedPropertiesVisitor, InlineSingleUseReferencesVisitor};
 
 /// Applies metadata to the given schema.
 ///
@@ -480,7 +480,9 @@ pub fn generate_internal_tagged_variant_schema(
 }
 
 pub fn default_schema_settings() -> SchemaSettings {
-    SchemaSettings::new().with_visitor(DisallowedUnevaluatedPropertiesVisitor::from_settings)
+    SchemaSettings::new()
+        .with_visitor(InlineSingleUseReferencesVisitor::from_settings)
+        .with_visitor(DisallowedUnevaluatedPropertiesVisitor::from_settings)
 }
 
 pub fn generate_root_schema<T>() -> Result<RootSchema, GenerateError>