diff --git a/Cargo.toml b/Cargo.toml index 5d855f6fa302b..7d9320f6a5ba5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,7 +198,7 @@ serde = { version = "1.0.152", default-features = false, features = ["derive"] } serde-toml-merge = { version = "0.3.0", default-features = false } serde_bytes = { version = "0.11.8", default-features = false, features = ["std"], optional = true } serde_json = { version = "1.0.91", default-features = false, features = ["raw_value"] } -serde_with = { version = "2.2.0", default-features = false, features = ["macros", "std"]} +serde_with = { version = "2.2.0", default-features = false, features = ["macros", "std"] } serde_yaml = { version = "0.9.17", default-features = false } # Messagepack diff --git a/src/aws/auth.rs b/src/aws/auth.rs index 40fd956173583..31b8e55d764a9 100644 --- a/src/aws/auth.rs +++ b/src/aws/auth.rs @@ -56,9 +56,11 @@ pub enum AwsAuthentication { /// Authenticate using a fixed access key and secret pair. AccessKey { /// The AWS access key ID. + #[configurable(metadata(docs::examples = "AKIAIOSFODNN7EXAMPLE"))] access_key_id: SensitiveString, /// The AWS secret access key. + #[configurable(metadata(docs::examples = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"))] secret_access_key: SensitiveString, }, @@ -67,28 +69,42 @@ pub enum AwsAuthentication { /// Additionally, the specific credential profile to use can be set. File { /// Path to the credentials file. + #[configurable(metadata(docs::examples = "/my/aws/credentials"))] credentials_file: String, /// The credentials profile to use. + /// + /// Used to select AWS credentials from a provided credentials file. + #[configurable(metadata(docs::examples = "develop"))] profile: Option, }, /// Assume the given role ARN. Role { - /// The ARN of the role to assume. + /// The ARN of an [IAM role][iam_role] to assume. + /// + /// [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + #[configurable(metadata(docs::examples = "arn:aws:iam::123456789098:role/my_role"))] assume_role: String, /// Timeout for assuming the role, in seconds. + /// + /// Relevant when the default credentials chain is used or `assume_role`. + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::examples = 30))] load_timeout_secs: Option, /// Configuration for authenticating with AWS through IMDS. #[serde(default)] imds: ImdsAuthentication, - /// The AWS region to send STS requests to. + /// The [AWS region][aws_region] to send STS requests to. /// /// If not set, this will default to the configured region /// for the service itself. + /// + /// [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + #[configurable(metadata(docs::examples = "us-west-2"))] region: Option, }, @@ -96,6 +112,10 @@ pub enum AwsAuthentication { #[derivative(Default)] Default { /// Timeout for successfully loading any credentials, in seconds. + /// + /// Relevant when the default credentials chain is used or `assume_role`. + #[configurable(metadata(docs::type_unit = "seconds"))] + #[configurable(metadata(docs::examples = 30))] load_timeout_secs: Option, /// Configuration for authenticating with AWS through IMDS. diff --git a/src/aws/region.rs b/src/aws/region.rs index 50e64d9ad2e5b..1bb0abf9647f2 100644 --- a/src/aws/region.rs +++ b/src/aws/region.rs @@ -10,10 +10,14 @@ use vector_config::configurable_component; #[derive(Clone, Debug, Default, PartialEq, Eq)] #[serde(default)] pub struct RegionOrEndpoint { - /// The AWS region to use. + /// The [AWS region][aws_region] of the target service. + /// + /// [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + #[configurable(metadata(docs::examples = "us-east-1"))] pub region: Option, - /// The API endpoint of the service. + /// Custom endpoint for use with AWS-compatible services. + #[configurable(metadata(docs::examples = "http://127.0.0.0:5000/path/to/service"))] pub endpoint: Option, } diff --git a/src/sources/aws_s3/mod.rs b/src/sources/aws_s3/mod.rs index 69e568e5a51c9..65c73f45162af 100644 --- a/src/sources/aws_s3/mod.rs +++ b/src/sources/aws_s3/mod.rs @@ -34,7 +34,7 @@ pub enum Compression { /// The compression scheme of the object is determined from its `Content-Encoding` and /// `Content-Type` metadata, as well as the key suffix (for example, `.gz`). /// - /// It is set to 'none' if the compression scheme cannot be determined. + /// It is set to `none` if the compression scheme cannot be determined. #[derivative(Default)] Auto, /// Uncompressed. @@ -45,7 +45,7 @@ pub enum Compression { Zstd, } -/// Strategies for consuming objects from S3. +/// Strategies for consuming objects from AWS S3. #[configurable_component] #[derive(Clone, Copy, Debug, Derivative)] #[serde(rename_all = "lowercase")] @@ -86,6 +86,7 @@ pub struct AwsS3Config { /// /// [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html #[configurable(deprecated)] + #[configurable(metadata(docs::hidden))] assume_role: Option, #[configurable(derived)] @@ -95,6 +96,7 @@ pub struct AwsS3Config { /// Multiline aggregation configuration. /// /// If not specified, multiline aggregation is disabled. + #[configurable(derived)] multiline: Option, #[configurable(derived)] @@ -624,7 +626,7 @@ mod integration_tests { start_pattern: "abc".to_owned(), mode: line_agg::Mode::HaltWith, condition_pattern: "geh".to_owned(), - timeout_ms: 1000, + timeout_ms: Duration::from_millis(1000), }), logs.join("\n").into_bytes(), vec!["abc\ndef\ngeh".to_owned()], diff --git a/src/sources/aws_s3/sqs.rs b/src/sources/aws_s3/sqs.rs index d24e23704276b..d8234a296a74f 100644 --- a/src/sources/aws_s3/sqs.rs +++ b/src/sources/aws_s3/sqs.rs @@ -15,6 +15,7 @@ use codecs::{decoding::FramingError, BytesDeserializer, CharacterDelimitedDecode use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; use once_cell::sync::Lazy; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use tokio::{pin, select}; use tokio_util::codec::FramedRead; @@ -49,21 +50,27 @@ static SUPPORTED_S3_EVENT_VERSION: Lazy = // // TODO: It seems awfully likely that we could re-use the existing configuration type for the `aws_sqs` source in some // way, given the near 100% overlap in configurable values. +#[serde_as] #[configurable_component] #[derive(Clone, Debug, Derivative)] #[derivative(Default)] #[serde(deny_unknown_fields)] pub(super) struct Config { /// The URL of the SQS queue to poll for bucket notifications. + #[configurable(metadata( + docs::examples = "https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue" + ))] pub(super) queue_url: String, /// How long to wait while polling the queue for new messages, in seconds. /// - /// Generally should not be changed unless instructed to do so, as if messages are available, they will always be - /// consumed, regardless of the value of `poll_secs`. - // NOTE: We restrict this to u32 for safe conversion to i64 later. + /// Generally should not be changed unless instructed to do so, as if messages are available, + /// they will always be consumed, regardless of the value of `poll_secs`. + // NOTE: We restrict this to u32 for safe conversion to i32 later. + // NOTE: This value isn't used as a `Duration` downstream, so we don't bother using `serde_with` #[serde(default = "default_poll_secs")] #[derivative(Default(value = "default_poll_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] pub(super) poll_secs: u32, /// The visibility timeout to use for messages, in seconds. @@ -72,9 +79,11 @@ pub(super) struct Config { /// takes longer than `visibility_timeout_secs` to process and delete the message from the queue, it is made available again for another consumer. /// /// This can happen if there is an issue between consuming a message and deleting it. - // NOTE: We restrict this to u32 for safe conversion to i64 later. + // NOTE: We restrict this to u32 for safe conversion to i32 later. + // NOTE: This value isn't used as a `Duration` downstream, so we don't bother using `serde_with` #[serde(default = "default_visibility_timeout_secs")] #[derivative(Default(value = "default_visibility_timeout_secs()"))] + #[configurable(metadata(docs::type_unit = "seconds"))] pub(super) visibility_timeout_secs: u32, /// Whether to delete the message once it is processed. @@ -88,10 +97,13 @@ pub(super) struct Config { /// /// Defaults to the number of available CPUs on the system. /// - /// Should not typically need to be changed, but it can sometimes be beneficial to raise this value when there is a - /// high rate of messages being pushed into the queue and the objects being fetched are small. In these cases, - /// System resources may not be fully utilized without fetching more messages per second, as the SQS message - /// consumption rate affects the S3 object retrieval rate. + /// Should not typically need to be changed, but it can sometimes be beneficial to raise this + /// value when there is a high rate of messages being pushed into the queue and the objects + /// being fetched are small. In these cases, system resources may not be fully utilized without + /// fetching more messages per second, as the SQS message consumption rate affects the S3 object + /// retrieval rate. + #[configurable(metadata(docs::type_unit = "tasks"))] + #[configurable(metadata(docs::examples = 5))] pub(super) client_concurrency: Option, #[configurable(derived)] diff --git a/src/sources/docker_logs/mod.rs b/src/sources/docker_logs/mod.rs index 22dc23b65c71a..441bf1a89be91 100644 --- a/src/sources/docker_logs/mod.rs +++ b/src/sources/docker_logs/mod.rs @@ -131,6 +131,7 @@ pub struct DockerLogsConfig { /// Multiline aggregation configuration. /// /// If not specified, multiline aggregation is disabled. + #[configurable(derived)] multiline: Option, #[configurable(derived)] diff --git a/src/sources/docker_logs/tests.rs b/src/sources/docker_logs/tests.rs index 9d234e7452ad4..0f8cd0981c069 100644 --- a/src/sources/docker_logs/tests.rs +++ b/src/sources/docker_logs/tests.rs @@ -854,7 +854,7 @@ mod integration_tests { start_pattern: "^[^\\s]".to_owned(), condition_pattern: "^[\\s]+at".to_owned(), mode: line_agg::Mode::ContinueThrough, - timeout_ms: 10, + timeout_ms: Duration::from_millis(10), }), log_namespace: Some(true), ..DockerLogsConfig::default() @@ -926,7 +926,7 @@ mod integration_tests { start_pattern: "^[^\\s]".to_owned(), condition_pattern: "^[\\s]+at".to_owned(), mode: line_agg::Mode::ContinueThrough, - timeout_ms: 10, + timeout_ms: Duration::from_millis(10), }), ..DockerLogsConfig::default() }; diff --git a/src/sources/file.rs b/src/sources/file.rs index c4704f0e72819..fc41c1659d62d 100644 --- a/src/sources/file.rs +++ b/src/sources/file.rs @@ -1794,7 +1794,7 @@ mod tests { start_pattern: "INFO".to_owned(), condition_pattern: "INFO".to_owned(), mode: line_agg::Mode::HaltBefore, - timeout_ms: 25, // less than 50 in sleep() + timeout_ms: Duration::from_millis(25), // less than 50 in sleep() }), ..test_default_file_config(&dir) }; @@ -1853,7 +1853,7 @@ mod tests { start_pattern: "INFO".to_owned(), condition_pattern: "INFO".to_owned(), mode: line_agg::Mode::HaltBefore, - timeout_ms: 25, // less than 50 in sleep() + timeout_ms: Duration::from_millis(25), // less than 50 in sleep() }), ..test_default_file_config(&dir) }; diff --git a/src/sources/util/multiline_config.rs b/src/sources/util/multiline_config.rs index f35a0f0dddbd3..1a840fa8a6d67 100644 --- a/src/sources/util/multiline_config.rs +++ b/src/sources/util/multiline_config.rs @@ -1,12 +1,14 @@ use std::{convert::TryFrom, time::Duration}; use regex::bytes::Regex; +use serde_with::serde_as; use snafu::{ResultExt, Snafu}; use vector_config::configurable_component; use crate::line_agg; /// Configuration of multi-line aggregation. +#[serde_as] #[configurable_component] #[derive(Clone, Debug, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -36,9 +38,10 @@ pub struct MultilineConfig { /// The maximum amount of time to wait for the next additional line, in milliseconds. /// /// Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete. + #[serde_as(as = "serde_with::DurationMilliSeconds")] #[configurable(metadata(docs::examples = 1000))] #[configurable(metadata(docs::examples = 600000))] - pub timeout_ms: u64, + pub timeout_ms: Duration, } impl TryFrom<&MultilineConfig> for line_agg::Config { @@ -56,7 +59,7 @@ impl TryFrom<&MultilineConfig> for line_agg::Config { .with_context(|_| InvalidMultilineStartPatternSnafu { start_pattern })?; let condition_pattern = Regex::new(condition_pattern) .with_context(|_| InvalidMultilineConditionPatternSnafu { condition_pattern })?; - let timeout = Duration::from_millis(*timeout_ms); + let timeout = *timeout_ms; Ok(Self { start_pattern, diff --git a/website/cue/reference.cue b/website/cue/reference.cue index 2ad4df2ec99b7..eaaf07ba35d36 100644 --- a/website/cue/reference.cue +++ b/website/cue/reference.cue @@ -622,7 +622,7 @@ _values: { unit?: #Unit | null } -#Unit: "bytes" | "events" | "milliseconds" | "nanoseconds" | "requests" | "seconds" | "lines" | "concurrency" | "connections" +#Unit: "bytes" | "events" | "milliseconds" | "nanoseconds" | "requests" | "seconds" | "lines" | "concurrency" | "connections" | "tasks" administration: _ components: _ diff --git a/website/cue/reference/components/aws.cue b/website/cue/reference/components/aws.cue index 236eb037dc879..fafc9aa6c613f 100644 --- a/website/cue/reference/components/aws.cue +++ b/website/cue/reference/components/aws.cue @@ -2,121 +2,125 @@ package metadata components: _aws: { configuration: { - auth: { - common: false - description: "Options for the authentication strategy." - required: false - type: object: { - examples: [] - options: { - access_key_id: { - category: "Auth" - common: false - description: "The AWS access key id. Used for AWS authentication when communicating with AWS services." - required: false - type: string: { - default: null - examples: ["AKIAIOSFODNN7EXAMPLE"] + _aws_include: bool | *true + + if _aws_include { + auth: { + common: false + description: "Options for the authentication strategy." + required: false + type: object: { + examples: [] + options: { + access_key_id: { + category: "Auth" + common: false + description: "The AWS access key id. Used for AWS authentication when communicating with AWS services." + required: false + type: string: { + default: null + examples: ["AKIAIOSFODNN7EXAMPLE"] + } } - } - secret_access_key: { - category: "Auth" - common: false - description: "The AWS secret access key. Used for AWS authentication when communicating with AWS services." - required: false - type: string: { - default: null - examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] + secret_access_key: { + category: "Auth" + common: false + description: "The AWS secret access key. Used for AWS authentication when communicating with AWS services." + required: false + type: string: { + default: null + examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] + } } - } - assume_role: { - category: "Auth" - common: false - description: "The ARN of an [IAM role](\(urls.aws_iam_role)) to assume at startup." - required: false - type: string: { - default: null - examples: ["arn:aws:iam::123456789098:role/my_role"] + assume_role: { + category: "Auth" + common: false + description: "The ARN of an [IAM role](\(urls.aws_iam_role)) to assume at startup." + required: false + type: string: { + default: null + examples: ["arn:aws:iam::123456789098:role/my_role"] + } } - } - region: { - category: "Auth" - common: false - description: "The [AWS region](\(urls.aws_regions)) to send STS requests to. If not set, this will default to the configured region for the service itself." - required: false - type: string: { - default: null - examples: ["us-west-2"] + region: { + category: "Auth" + common: false + description: "The [AWS region](\(urls.aws_regions)) to send STS requests to. If not set, this will default to the configured region for the service itself." + required: false + type: string: { + default: null + examples: ["us-west-2"] + } } - } - load_timeout_secs: { - category: "Auth" - common: false - description: "The timeout for loading credentials. Relevant when the default credentials chain is used or `assume_role`." - required: false - type: uint: { - unit: "seconds" - default: 5 - examples: [30] + load_timeout_secs: { + category: "Auth" + common: false + description: "The timeout for loading credentials. Relevant when the default credentials chain is used or `assume_role`." + required: false + type: uint: { + unit: "seconds" + default: 5 + examples: [30] + } } - } - imds: { - description: "Configuration for authenticating with AWS through IMDS." - required: false - type: object: { - options: { - connect_timeout_seconds: { - description: "Connect timeout for IMDS." - required: false - type: uint: { - default: 1 - unit: "seconds" + imds: { + description: "Configuration for authenticating with AWS through IMDS." + required: false + type: object: { + options: { + connect_timeout_seconds: { + description: "Connect timeout for IMDS." + required: false + type: uint: { + default: 1 + unit: "seconds" + } } - } - max_attempts: { - description: "Number of IMDS retries for fetching tokens and metadata." - required: false - type: uint: default: 4 - } - read_timeout_seconds: { - description: "Read timeout for IMDS." - required: false - type: uint: { - default: 1 - unit: "seconds" + max_attempts: { + description: "Number of IMDS retries for fetching tokens and metadata." + required: false + type: uint: default: 4 + } + read_timeout_seconds: { + description: "Read timeout for IMDS." + required: false + type: uint: { + default: 1 + unit: "seconds" + } } } } } - } - profile: { - category: "Auth" - common: false - description: "The AWS profile name. Used to select AWS credentials from a provided credentials file." - required: false - type: string: { - default: "default" - examples: ["develop"] + profile: { + category: "Auth" + common: false + description: "The AWS profile name. Used to select AWS credentials from a provided credentials file." + required: false + type: string: { + default: "default" + examples: ["develop"] + } } } } } - } - endpoint: { - common: false - description: "Custom endpoint for use with AWS-compatible services." - required: false - type: string: { - default: null - examples: ["http://127.0.0.0:5000/path/to/service"] + endpoint: { + common: false + description: "Custom endpoint for use with AWS-compatible services." + required: false + type: string: { + default: null + examples: ["http://127.0.0.0:5000/path/to/service"] + } } - } - region: { - description: "The [AWS region](\(urls.aws_regions)) of the target service." - required: true - type: string: { - examples: ["us-east-1"] + region: { + description: "The [AWS region](\(urls.aws_regions)) of the target service." + required: true + type: string: { + examples: ["us-east-1"] + } } } } @@ -217,7 +221,7 @@ components: _aws: { { title: "Assuming roles" body: """ - Vector can assume an AWS IAM role via the [`assume_role`](#auth.assume_role) option. This is an + Vector can assume an AWS IAM role via the [`auth.assume_role`](#auth.assume_role) option. This is an optional setting that is helpful for a variety of use cases, such as cross account access. """ diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index d0a0999cb96f2..b4c10109817ee 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -34,17 +34,21 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -74,29 +78,42 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -297,9 +314,9 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } group_name: { description: """ @@ -311,9 +328,13 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { type: string: syntax: "template" } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: "Outbound HTTP request settings." diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue index a13f4544efece..b1db1b5a97c9f 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_metrics.cue @@ -44,17 +44,21 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -84,29 +88,42 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -177,14 +194,18 @@ base: components: sinks: aws_cloudwatch_metrics: configuration: { type: string: {} } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: """ diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue index 9c231a83a95eb..5423378543534 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_firehose.cue @@ -34,17 +34,21 @@ base: components: sinks: aws_kinesis_firehose: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -74,29 +78,42 @@ base: components: sinks: aws_kinesis_firehose: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -276,14 +293,18 @@ base: components: sinks: aws_kinesis_firehose: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: """ diff --git a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue index 94f9a2c0cff56..c6a7b1dd7fc3a 100644 --- a/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/base/aws_kinesis_streams.cue @@ -34,17 +34,21 @@ base: components: sinks: aws_kinesis_streams: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -74,29 +78,42 @@ base: components: sinks: aws_kinesis_streams: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -276,9 +293,9 @@ base: components: sinks: aws_kinesis_streams: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } partition_key_field: { description: """ @@ -290,9 +307,13 @@ base: components: sinks: aws_kinesis_streams: configuration: { type: string: {} } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: """ diff --git a/website/cue/reference/components/sinks/base/aws_s3.cue b/website/cue/reference/components/sinks/base/aws_s3.cue index b85e36f714a78..d5df31d453732 100644 --- a/website/cue/reference/components/sinks/base/aws_s3.cue +++ b/website/cue/reference/components/sinks/base/aws_s3.cue @@ -105,17 +105,21 @@ base: components: sinks: aws_s3: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -145,29 +149,42 @@ base: components: sinks: aws_s3: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -375,9 +392,9 @@ base: components: sinks: aws_s3: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } filename_append_uuid: { description: """ @@ -507,9 +524,13 @@ base: components: sinks: aws_s3: configuration: { type: string: syntax: "template" } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: """ diff --git a/website/cue/reference/components/sinks/base/aws_sqs.cue b/website/cue/reference/components/sinks/base/aws_sqs.cue index 8ec1accf9bc6a..e1fdb414164e0 100644 --- a/website/cue/reference/components/sinks/base/aws_sqs.cue +++ b/website/cue/reference/components/sinks/base/aws_sqs.cue @@ -44,17 +44,21 @@ base: components: sinks: aws_sqs: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -84,29 +88,42 @@ base: components: sinks: aws_sqs: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -227,9 +244,9 @@ base: components: sinks: aws_sqs: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } message_deduplication_id: { description: """ @@ -258,9 +275,13 @@ base: components: sinks: aws_sqs: configuration: { type: string: {} } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } request: { description: """ diff --git a/website/cue/reference/components/sinks/base/elasticsearch.cue b/website/cue/reference/components/sinks/base/elasticsearch.cue index e913a6949750b..60626359a433b 100644 --- a/website/cue/reference/components/sinks/base/elasticsearch.cue +++ b/website/cue/reference/components/sinks/base/elasticsearch.cue @@ -48,19 +48,23 @@ base: components: sinks: elasticsearch: configuration: { description: "The AWS access key ID." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -91,10 +95,17 @@ base: components: sinks: elasticsearch: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ relevant_when: "strategy = \"aws\"" required: false - type: uint: {} + type: uint: { + examples: [30] + unit: "seconds" + } } password: { description: "Basic authentication password." @@ -103,27 +114,33 @@ base: components: sinks: elasticsearch: configuration: { type: string: {} } profile: { - description: "The credentials profile to use." + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ relevant_when: "strategy = \"aws\"" required: false - type: string: {} + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ relevant_when: "strategy = \"aws\"" required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } strategy: { description: "The authentication strategy to use." @@ -146,14 +163,18 @@ base: components: sinks: elasticsearch: configuration: { required: false type: object: options: { endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } } } diff --git a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue index 84d4c57068b35..656a0b7211784 100644 --- a/website/cue/reference/components/sinks/base/prometheus_remote_write.cue +++ b/website/cue/reference/components/sinks/base/prometheus_remote_write.cue @@ -35,19 +35,23 @@ base: components: sinks: prometheus_remote_write: configuration: { description: "The AWS access key ID." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -78,10 +82,17 @@ base: components: sinks: prometheus_remote_write: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ relevant_when: "strategy = \"aws\"" required: false - type: uint: {} + type: uint: { + examples: [30] + unit: "seconds" + } } password: { description: "Basic authentication password." @@ -90,27 +101,33 @@ base: components: sinks: prometheus_remote_write: configuration: { type: string: {} } profile: { - description: "The credentials profile to use." + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ relevant_when: "strategy = \"aws\"" required: false - type: string: {} + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ relevant_when: "strategy = \"aws\"" required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." relevant_when: "strategy = \"aws\"" required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } strategy: { description: "The authentication strategy to use." @@ -144,14 +161,18 @@ base: components: sinks: prometheus_remote_write: configuration: { required: false type: object: options: { endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } } } diff --git a/website/cue/reference/components/sources/aws_s3.cue b/website/cue/reference/components/sources/aws_s3.cue index 8e77cacd1c244..253db87969552 100644 --- a/website/cue/reference/components/sources/aws_s3.cue +++ b/website/cue/reference/components/sources/aws_s3.cue @@ -4,6 +4,7 @@ components: sources: aws_s3: components._aws & { title: "AWS S3" features: { + auto_generated: true acknowledgements: true multiline: enabled: true collect: { @@ -44,75 +45,8 @@ components: sources: aws_s3: components._aws & { platform_name: null } - configuration: { - acknowledgements: configuration._source_acknowledgements - strategy: { - common: false - description: "The strategy to use to consume objects from AWS S3." - required: false - type: string: { - default: "sqs" - enum: { - sqs: "Consume S3 objects by polling for bucket notifications sent to an [AWS SQS queue](\(urls.aws_sqs))." - } - } - } - compression: { - common: false - description: "The compression format of the S3 objects.." - required: false - type: string: { - default: "text" - enum: { - auto: "Vector will try to determine the compression format of the object from its: `Content-Encoding` metadata, `Content-Type` metadata, and key suffix (e.g. `.gz`). It will fallback to 'none' if it cannot determine the compression." - gzip: "GZIP format." - zstd: "ZSTD format." - none: "Uncompressed." - } - } - } - sqs: { - common: true - description: "SQS strategy options. Required if strategy=`sqs`." - required: false - type: object: { - examples: [] - options: { - poll_secs: { - common: true - description: "How long to wait when polling SQS for new messages." - required: false - type: uint: { - default: 15 - unit: "seconds" - } - } - visibility_timeout_secs: { - common: false - description: "The visibility timeout to use for messages in seconds. This controls how long a message is left unavailable when a Vector receives it. If a `vector` does not delete the message before the timeout expires, it will be made reavailable for another consumer; this can happen if, for example, the `vector` process crashes." - required: false - warnings: ["Should be set higher than the length of time it takes to process an individual message to avoid that message being reprocessed."] - type: uint: { - default: 300 - unit: "seconds" - } - } - delete_message: { - common: true - description: "Whether to delete the message once Vector processes it. It can be useful to set this to `false` to debug or during initial Vector setup." - required: false - type: bool: default: true - } - queue_url: { - description: "The URL of the SQS queue to receive bucket notifications from." - required: true - type: string: { - examples: ["https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue"] - } - } - } - } - } + configuration: base.components.sources.aws_s3.configuration & { + _aws_include: false } output: logs: object: { diff --git a/website/cue/reference/components/sources/base/aws_s3.cue b/website/cue/reference/components/sources/base/aws_s3.cue index baa71581fbc93..4a8a2d832659f 100644 --- a/website/cue/reference/components/sources/base/aws_s3.cue +++ b/website/cue/reference/components/sources/base/aws_s3.cue @@ -22,16 +22,6 @@ base: components: sources: aws_s3: configuration: { type: bool: {} } } - assume_role: { - deprecated: true - description: """ - The ARN of an [IAM role][iam_role] to assume at startup. - - [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html - """ - required: false - type: string: {} - } auth: { description: "Configuration of the authentication strategy for interacting with AWS services." required: false @@ -39,17 +29,21 @@ base: components: sources: aws_s3: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -79,29 +73,42 @@ base: components: sources: aws_s3: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -117,7 +124,7 @@ base: components: sources: aws_s3: configuration: { The compression scheme of the object is determined from its `Content-Encoding` and `Content-Type` metadata, as well as the key suffix (for example, `.gz`). - It is set to 'none' if the compression scheme cannot be determined. + It is set to `none` if the compression scheme cannot be determined. """ gzip: "GZIP." none: "Uncompressed." @@ -126,9 +133,9 @@ base: components: sources: aws_s3: configuration: { } } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } multiline: { description: """ @@ -193,14 +200,21 @@ base: components: sources: aws_s3: configuration: { Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete. """ required: true - type: uint: examples: [1000, 600000] + type: uint: { + examples: [1000, 600000] + unit: "milliseconds" + } } } } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } sqs: { description: """ @@ -216,13 +230,17 @@ base: components: sources: aws_s3: configuration: { Defaults to the number of available CPUs on the system. - Should not typically need to be changed, but it can sometimes be beneficial to raise this value when there is a - high rate of messages being pushed into the queue and the objects being fetched are small. In these cases, - System resources may not be fully utilized without fetching more messages per second, as the SQS message - consumption rate affects the S3 object retrieval rate. + Should not typically need to be changed, but it can sometimes be beneficial to raise this + value when there is a high rate of messages being pushed into the queue and the objects + being fetched are small. In these cases, system resources may not be fully utilized without + fetching more messages per second, as the SQS message consumption rate affects the S3 object + retrieval rate. """ required: false - type: uint: {} + type: uint: { + examples: [5] + unit: "tasks" + } } delete_message: { description: """ @@ -237,16 +255,19 @@ base: components: sources: aws_s3: configuration: { description: """ How long to wait while polling the queue for new messages, in seconds. - Generally should not be changed unless instructed to do so, as if messages are available, they will always be - consumed, regardless of the value of `poll_secs`. + Generally should not be changed unless instructed to do so, as if messages are available, + they will always be consumed, regardless of the value of `poll_secs`. """ required: false - type: uint: default: 15 + type: uint: { + default: 15 + unit: "seconds" + } } queue_url: { description: "The URL of the SQS queue to poll for bucket notifications." required: true - type: string: {} + type: string: examples: ["https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue"] } tls_options: { description: "TLS configuration." @@ -343,7 +364,10 @@ base: components: sources: aws_s3: configuration: { This can happen if there is an issue between consuming a message and deleting it. """ required: false - type: uint: default: 300 + type: uint: { + default: 300 + unit: "seconds" + } } } } diff --git a/website/cue/reference/components/sources/base/aws_sqs.cue b/website/cue/reference/components/sources/base/aws_sqs.cue index 326458971fdc8..50f0efe252508 100644 --- a/website/cue/reference/components/sources/base/aws_sqs.cue +++ b/website/cue/reference/components/sources/base/aws_sqs.cue @@ -29,17 +29,21 @@ base: components: sources: aws_sqs: configuration: { access_key_id: { description: "The AWS access key ID." required: true - type: string: {} + type: string: examples: ["AKIAIOSFODNN7EXAMPLE"] } assume_role: { - description: "The ARN of the role to assume." - required: true - type: string: {} + description: """ + The ARN of an [IAM role][iam_role] to assume. + + [iam_role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html + """ + required: true + type: string: examples: ["arn:aws:iam::123456789098:role/my_role"] } credentials_file: { description: "Path to the credentials file." required: true - type: string: {} + type: string: examples: ["/my/aws/credentials"] } imds: { description: "Configuration for authenticating with AWS through IMDS." @@ -69,29 +73,42 @@ base: components: sources: aws_sqs: configuration: { } } load_timeout_secs: { - description: "Timeout for successfully loading any credentials, in seconds." - required: false - type: uint: {} + description: """ + Timeout for successfully loading any credentials, in seconds. + + Relevant when the default credentials chain is used or `assume_role`. + """ + required: false + type: uint: { + examples: [30] + unit: "seconds" + } } profile: { - description: "The credentials profile to use." - required: false - type: string: {} + description: """ + The credentials profile to use. + + Used to select AWS credentials from a provided credentials file. + """ + required: false + type: string: examples: ["develop"] } region: { description: """ - The AWS region to send STS requests to. + The [AWS region][aws_region] to send STS requests to. If not set, this will default to the configured region for the service itself. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints """ required: false - type: string: {} + type: string: examples: ["us-west-2"] } secret_access_key: { description: "The AWS secret access key." required: true - type: string: {} + type: string: examples: ["wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"] } } } @@ -168,9 +185,9 @@ base: components: sources: aws_sqs: configuration: { type: bool: default: true } endpoint: { - description: "The API endpoint of the service." + description: "Custom endpoint for use with AWS-compatible services." required: false - type: string: {} + type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } framing: { description: """ @@ -263,9 +280,13 @@ base: components: sources: aws_sqs: configuration: { type: string: {} } region: { - description: "The AWS region to use." - required: false - type: string: {} + description: """ + The [AWS region][aws_region] of the target service. + + [aws_region]: https://docs.aws.amazon.com/general/latest/gr/rande.html#regional-endpoints + """ + required: false + type: string: examples: ["us-east-1"] } tls: { description: "TLS configuration." diff --git a/website/cue/reference/components/sources/base/docker_logs.cue b/website/cue/reference/components/sources/base/docker_logs.cue index b35d6357cf2d7..6f33af0193c2f 100644 --- a/website/cue/reference/components/sources/base/docker_logs.cue +++ b/website/cue/reference/components/sources/base/docker_logs.cue @@ -141,7 +141,10 @@ base: components: sources: docker_logs: configuration: { Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete. """ required: true - type: uint: examples: [1000, 600000] + type: uint: { + examples: [1000, 600000] + unit: "milliseconds" + } } } } diff --git a/website/cue/reference/components/sources/base/file.cue b/website/cue/reference/components/sources/base/file.cue index 1d1f114b08a6c..999cfa97d13fe 100644 --- a/website/cue/reference/components/sources/base/file.cue +++ b/website/cue/reference/components/sources/base/file.cue @@ -298,7 +298,10 @@ base: components: sources: file: configuration: { Once this timeout is reached, the buffered message is guaranteed to be flushed, even if incomplete. """ required: true - type: uint: examples: [1000, 600000] + type: uint: { + examples: [1000, 600000] + unit: "milliseconds" + } } } }