Skip to content

Commit

Permalink
chore(config): Update field labels for commonly used sources and tran…
Browse files Browse the repository at this point in the history
…sforms (#17517)

This PR updates the commonly used sources and transforms' field labels
that needs a human_name label. Similar to this
[PR](#17475).

<!--
**Your PR title must conform to the conventional commit spec!**

  <type>(<scope>)!: <description>

  * `type` = chore, enhancement, feat, fix, docs
  * `!` = OPTIONAL: signals a breaking change
* `scope` = Optional when `type` is "chore" or "docs", available scopes
https://github.com/vectordotdev/vector/blob/master/.github/semantic.yml#L20
  * `description` = short description of the change

Examples:

  * enhancement(file source): Add `sort` option to sort discovered files
  * feat(new source): Initial `statsd` source
  * fix(file source): Fix a bug discovering new files
  * chore(external docs): Clarify `batch_size` option
-->

---------

Signed-off-by: Spencer Gilbert <spencer.gilbert@datadoghq.com>
Co-authored-by: Spencer Gilbert <spencer.gilbert@datadoghq.com>
  • Loading branch information
maycmlee and spencergilbert authored Jun 1, 2023
1 parent 23ed0e3 commit f523f70
Showing 20 changed files with 41 additions and 11 deletions.
2 changes: 1 addition & 1 deletion lib/vector-config-common/src/human_friendly.rs
Original file line number Diff line number Diff line change
@@ -48,7 +48,7 @@ static WELL_KNOWN_ACRONYMS: Lazy<HashSet<String>> = Lazy::new(|| {
"api", "amqp", "aws", "ec2", "ecs", "gcp", "hec", "http", "https", "nats", "nginx", "s3",
"sqs", "tls", "ssl", "otel", "gelf", "csv", "json", "rfc3339", "lz4", "us", "eu", "bsd",
"vrl", "tcp", "udp", "id", "uuid", "kms", "uri", "url", "acp", "uid", "ip", "pid",
"ndjson", "ewma", "rtt", "cpu", "acl", "imds", "acl", "alpn",
"ndjson", "ewma", "rtt", "cpu", "acl", "imds", "acl", "alpn", "sasl",
];

acronyms.iter().map(|s| s.to_lowercase()).collect()
1 change: 1 addition & 0 deletions lib/vector-core/src/tcp.rs
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@ use vector_config::configurable_component;
#[configurable_component]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[serde(deny_unknown_fields)]
#[configurable(metadata(docs::human_name = "Wait Time"))]
pub struct TcpKeepaliveConfig {
/// The time to wait before starting to send TCP keepalive probes on an idle connection.
#[configurable(metadata(docs::type_unit = "seconds"))]
1 change: 1 addition & 0 deletions src/sources/apache_metrics/mod.rs
Original file line number Diff line number Diff line change
@@ -42,6 +42,7 @@ pub struct ApacheMetricsConfig {
/// The interval between scrapes.
#[serde(default = "default_scrape_interval_secs")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
scrape_interval_secs: Duration,

/// The namespace of the metric.
1 change: 1 addition & 0 deletions src/sources/aws_ecs_metrics/mod.rs
Original file line number Diff line number Diff line change
@@ -87,6 +87,7 @@ pub struct AwsEcsMetricsSourceConfig {
/// The interval between scrapes, in seconds.
#[serde(default = "default_scrape_interval_secs")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
scrape_interval_secs: Duration,

/// The namespace of the metric.
4 changes: 2 additions & 2 deletions src/sources/aws_kinesis_firehose/mod.rs
Original file line number Diff line number Diff line change
@@ -37,15 +37,15 @@ pub struct AwsKinesisFirehoseConfig {
#[configurable(metadata(docs::examples = "localhost:443"))]
address: SocketAddr,

/// An optional access key to authenticate requests against.
/// An access key to authenticate requests against.
///
/// AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
/// configured, `access_key` should be set to the same value. Otherwise, all requests are allowed.
#[configurable(deprecated = "This option has been deprecated, use `access_keys` instead.")]
#[configurable(metadata(docs::examples = "A94A8FE5CCB19BA61C4C08"))]
access_key: Option<SensitiveString>,

/// An optional list of access keys to authenticate requests against.
/// A list of access keys to authenticate requests against.
///
/// AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
/// configured, `access_keys` should be set to the same value. Otherwise, all requests are allowed.
1 change: 1 addition & 0 deletions src/sources/aws_s3/sqs.rs
Original file line number Diff line number Diff line change
@@ -90,6 +90,7 @@ pub(super) struct Config {
#[serde(default = "default_visibility_timeout_secs")]
#[derivative(Default(value = "default_visibility_timeout_secs()"))]
#[configurable(metadata(docs::type_unit = "seconds"))]
#[configurable(metadata(docs::human_name = "Visibility Timeout"))]
pub(super) visibility_timeout_secs: u32,

/// Whether to delete the message once it is processed.
1 change: 1 addition & 0 deletions src/sources/aws_sqs/config.rs
Original file line number Diff line number Diff line change
@@ -58,6 +58,7 @@ pub struct AwsSqsConfig {
#[serde(default = "default_visibility_timeout_secs")]
#[derivative(Default(value = "default_visibility_timeout_secs()"))]
#[configurable(metadata(docs::type_unit = "seconds"))]
#[configurable(metadata(docs::human_name = "Visibility Timeout"))]
pub(super) visibility_timeout_secs: u32,

/// Whether to delete the message once it is processed.
4 changes: 4 additions & 0 deletions src/sources/gcp_pubsub.rs
Original file line number Diff line number Diff line change
@@ -163,13 +163,15 @@ pub struct PubsubConfig {
/// are all busy and so open a new stream.
#[serde(default = "default_poll_time")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Poll Time"))]
pub poll_time_seconds: Duration,

/// The acknowledgement deadline, in seconds, to use for this stream.
///
/// Messages that are not acknowledged when this deadline expires may be retransmitted.
#[serde(default = "default_ack_deadline")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[configurable(metadata(docs::human_name = "Acknowledgement Deadline"))]
pub ack_deadline_secs: Duration,

/// The acknowledgement deadline, in seconds, to use for this stream.
@@ -183,6 +185,7 @@ pub struct PubsubConfig {
/// The amount of time, in seconds, to wait between retry attempts after an error.
#[serde(default = "default_retry_delay")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Retry Delay"))]
pub retry_delay_secs: Duration,

/// The amount of time, in seconds, to wait between retry attempts after an error.
@@ -196,6 +199,7 @@ pub struct PubsubConfig {
/// `60`, you may see periodic errors sent from the server.
#[serde(default = "default_keepalive")]
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Keepalive"))]
pub keepalive_secs: Duration,

/// The namespace to use for logs. This overrides the global setting.
1 change: 1 addition & 0 deletions src/sources/http_client/client.rs
Original file line number Diff line number Diff line change
@@ -55,6 +55,7 @@ pub struct HttpClientConfig {
#[serde(default = "default_interval")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[serde(rename = "scrape_interval_secs")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
pub interval: Duration,

/// Custom parameters for the HTTP request query string.
4 changes: 4 additions & 0 deletions src/sources/kafka.rs
Original file line number Diff line number Diff line change
@@ -107,26 +107,30 @@ pub struct KafkaSourceConfig {
#[configurable(metadata(docs::examples = 5000, docs::examples = 10000))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_session_timeout_ms")]
#[configurable(metadata(docs::human_name = "Session Timeout"))]
session_timeout_ms: Duration,

/// Timeout for network requests.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 30000, docs::examples = 60000))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_socket_timeout_ms")]
#[configurable(metadata(docs::human_name = "Socket Timeout"))]
socket_timeout_ms: Duration,

/// Maximum time the broker may wait to fill the response.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 50, docs::examples = 100))]
#[configurable(metadata(docs::advanced))]
#[serde(default = "default_fetch_wait_max_ms")]
#[configurable(metadata(docs::human_name = "Max Fetch Wait Time"))]
fetch_wait_max_ms: Duration,

/// The frequency that the consumer offsets are committed (written) to offset storage.
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[serde(default = "default_commit_interval_ms")]
#[configurable(metadata(docs::examples = 5000, docs::examples = 10000))]
#[configurable(metadata(docs::human_name = "Commit Interval"))]
commit_interval_ms: Duration,

/// Overrides the name of the log field used to add the message key to each event.
1 change: 1 addition & 0 deletions src/sources/prometheus/scrape.rs
Original file line number Diff line number Diff line change
@@ -57,6 +57,7 @@ pub struct PrometheusScrapeConfig {
#[serde(default = "default_interval")]
#[serde_as(as = "serde_with::DurationSeconds<u64>")]
#[serde(rename = "scrape_interval_secs")]
#[configurable(metadata(docs::human_name = "Scrape Interval"))]
interval: Duration,

/// The tag name added to each event representing the scraped instance's `host:port`.
12 changes: 9 additions & 3 deletions src/sources/splunk_hec/acknowledgements.rs
Original file line number Diff line number Diff line change
@@ -27,28 +27,34 @@ pub struct HecAcknowledgementsConfig {
/// Enables end-to-end acknowledgements.
pub enabled: Option<bool>,

/// The maximum number of ack statuses pending query across all channels.
/// The maximum number of acknowledgement statuses pending query across all channels.
///
/// Equivalent to the `max_number_of_acked_requests_pending_query` Splunk HEC setting.
///
/// Minimum of `1`.
#[configurable(metadata(docs::human_name = "Max Number of Pending Acknowledgements"))]
pub max_pending_acks: NonZeroU64,

/// The maximum number of Splunk HEC channels clients can use with this source.
///
/// Minimum of `1`.
#[configurable(metadata(docs::human_name = "Max Number of Acknowledgement Channels"))]
pub max_number_of_ack_channels: NonZeroU64,

/// The maximum number of ack statuses pending query for a single channel.
/// The maximum number of acknowledgement statuses pending query for a single channel.
///
/// Equivalent to the `max_number_of_acked_requests_pending_query_per_ack_channel` Splunk HEC setting.
///
/// Minimum of `1`.
#[configurable(metadata(
docs::human_name = "Max Number of Pending Acknowledgements Per Channel"
))]
pub max_pending_acks_per_channel: NonZeroU64,

/// Whether or not to remove channels after idling for `max_idle_time` seconds.
///
/// A channel is idling if it is not used for sending data or querying ack statuses.
/// A channel is idling if it is not used for sending data or querying acknowledgement statuses.
#[configurable(metadata(docs::human_name = "Acknowledgement Idle Cleanup"))]
pub ack_idle_cleanup: bool,

/// The amount of time, in seconds, a channel is allowed to idle before removal.
1 change: 1 addition & 0 deletions src/sources/util/multiline_config.rs
Original file line number Diff line number Diff line change
@@ -41,6 +41,7 @@ pub struct MultilineConfig {
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[configurable(metadata(docs::examples = 1000))]
#[configurable(metadata(docs::examples = 600000))]
#[configurable(metadata(docs::human_name = "Timeout"))]
pub timeout_ms: Duration,
}

1 change: 1 addition & 0 deletions src/transforms/aggregate.rs
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@ pub struct AggregateConfig {
///
/// During this time frame, metrics with the same series data (name, namespace, tags, and so on) are aggregated.
#[serde(default = "default_interval_ms")]
#[configurable(metadata(docs::human_name = "Flush Interval"))]
pub interval_ms: u64,
}

2 changes: 2 additions & 0 deletions src/transforms/reduce/mod.rs
Original file line number Diff line number Diff line change
@@ -47,12 +47,14 @@ pub struct ReduceConfig {
#[serde(default = "default_expire_after_ms")]
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[derivative(Default(value = "default_expire_after_ms()"))]
#[configurable(metadata(docs::human_name = "Expire After"))]
pub expire_after_ms: Duration,

/// The interval to check for and flush any expired events, in milliseconds.
#[serde(default = "default_flush_period_ms")]
#[serde_as(as = "serde_with::DurationMilliSeconds<u64>")]
#[derivative(Default(value = "default_flush_period_ms()"))]
#[configurable(metadata(docs::human_name = "Flush Period"))]
pub flush_period_ms: Duration,

/// The maximum number of events to group together.
3 changes: 3 additions & 0 deletions src/transforms/remap.rs
Original file line number Diff line number Diff line change
@@ -102,6 +102,7 @@ pub struct RemapConfig {
/// Additionally, dropped events can potentially be diverted to a specially named output for
/// further logging and analysis by setting `reroute_dropped`.
#[serde(default = "crate::serde::default_false")]
#[configurable(metadata(docs::human_name = "Drop Event on Error"))]
pub drop_on_error: bool,

/// Drops any event that is manually aborted during processing.
@@ -117,6 +118,7 @@ pub struct RemapConfig {
///
/// [vrl_docs_abort]: https://vector.dev/docs/reference/vrl/expressions/#abort
#[serde(default = "crate::serde::default_true")]
#[configurable(metadata(docs::human_name = "Drop Event on Abort"))]
pub drop_on_abort: bool,

/// Reroutes dropped events to a named output instead of halting processing on them.
@@ -129,6 +131,7 @@ pub struct RemapConfig {
/// to a specially-named output, `dropped`. The original event is annotated with additional
/// fields describing why the event was dropped.
#[serde(default = "crate::serde::default_false")]
#[configurable(metadata(docs::human_name = "Reroute Dropped Events"))]
pub reroute_dropped: bool,

#[configurable(derived, metadata(docs::hidden))]
1 change: 1 addition & 0 deletions src/transforms/tag_cardinality_limit/config.rs
Original file line number Diff line number Diff line change
@@ -60,6 +60,7 @@ pub struct BloomFilterConfig {
/// The larger the cache size, the less likely it is to have a false positive, or a case where
/// we allow a new value for tag even after we have reached the configured limits.
#[serde(default = "default_cache_size")]
#[configurable(metadata(docs::human_name = "Cache Size per Key"))]
pub cache_size_per_key: usize,
}

1 change: 1 addition & 0 deletions src/transforms/throttle.rs
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ pub struct ThrottleConfig {

/// The time window in which the configured `threshold` is applied, in seconds.
#[serde_as(as = "serde_with::DurationSeconds<f64>")]
#[configurable(metadata(docs::human_name = "Time Window"))]
window_secs: Duration,

/// The value to group events into separate buckets to be rate limited independently.
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ base: components: sources: aws_kinesis_firehose: configuration: {
deprecated: true
deprecated_message: "This option has been deprecated, use `access_keys` instead."
description: """
An optional access key to authenticate requests against.
An access key to authenticate requests against.
AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
configured, `access_key` should be set to the same value. Otherwise, all requests are allowed.
@@ -15,7 +15,7 @@ base: components: sources: aws_kinesis_firehose: configuration: {
}
access_keys: {
description: """
An optional list of access keys to authenticate requests against.
A list of access keys to authenticate requests against.
AWS Kinesis Firehose can be configured to pass along a user-configurable access key with each request. If
configured, `access_keys` should be set to the same value. Otherwise, all requests are allowed.
6 changes: 3 additions & 3 deletions website/cue/reference/components/sources/base/splunk_hec.cue
Original file line number Diff line number Diff line change
@@ -9,7 +9,7 @@ base: components: sources: splunk_hec: configuration: {
description: """
Whether or not to remove channels after idling for `max_idle_time` seconds.
A channel is idling if it is not used for sending data or querying ack statuses.
A channel is idling if it is not used for sending data or querying acknowledgement statuses.
"""
required: false
type: bool: default: false
@@ -41,7 +41,7 @@ base: components: sources: splunk_hec: configuration: {
}
max_pending_acks: {
description: """
The maximum number of ack statuses pending query across all channels.
The maximum number of acknowledgement statuses pending query across all channels.
Equivalent to the `max_number_of_acked_requests_pending_query` Splunk HEC setting.
@@ -52,7 +52,7 @@ base: components: sources: splunk_hec: configuration: {
}
max_pending_acks_per_channel: {
description: """
The maximum number of ack statuses pending query for a single channel.
The maximum number of acknowledgement statuses pending query for a single channel.
Equivalent to the `max_number_of_acked_requests_pending_query_per_ack_channel` Splunk HEC setting.

0 comments on commit f523f70

Please sign in to comment.