From d02f435d3bf121b19e15de4f139c95a6d010b25c Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Fri, 6 Sep 2024 21:29:48 +0200 Subject: [PATCH] feat(operator): Add support for Loki OTLP limits config (#13446) Co-authored-by: Robert Jacob --- operator/apis/loki/v1/lokistack_types.go | 163 +++++ operator/apis/loki/v1/v1.go | 7 + .../apis/loki/v1/zz_generated.deepcopy.go | 127 ++++ .../loki-operator.clusterserviceversion.yaml | 117 +++- .../loki.grafana.com_lokistacks.yaml | 235 +++++++ .../loki-operator.clusterserviceversion.yaml | 117 +++- .../loki.grafana.com_lokistacks.yaml | 235 +++++++ .../loki-operator.clusterserviceversion.yaml | 117 +++- .../loki.grafana.com_lokistacks.yaml | 235 +++++++ .../bases/loki.grafana.com_lokistacks.yaml | 235 +++++++ .../loki-operator.clusterserviceversion.yaml | 115 ++++ .../loki-operator.clusterserviceversion.yaml | 115 ++++ .../loki-operator.clusterserviceversion.yaml | 115 ++++ operator/docs/operator/api.md | 333 ++++++++++ .../manifests/internal/config/build_test.go | 584 ++++++++++++++++++ .../internal/config/loki-config.yaml | 62 ++ .../internal/config/loki-runtime-config.yaml | 52 ++ operator/internal/validation/lokistack.go | 104 ++++ .../internal/validation/lokistack_test.go | 309 +++++++++ 19 files changed, 3374 insertions(+), 3 deletions(-) diff --git a/operator/apis/loki/v1/lokistack_types.go b/operator/apis/loki/v1/lokistack_types.go index 90cee75d9447..41f4ad95e6a8 100644 --- a/operator/apis/loki/v1/lokistack_types.go +++ b/operator/apis/loki/v1/lokistack_types.go @@ -791,6 +791,140 @@ type IngestionLimitSpec struct { PerStreamRateLimitBurst int32 `json:"perStreamRateLimitBurst,omitempty"` } +// OTLPAttributeAction defines the action to executed when indexing +// OTLP resource attributes. Resource attributes can be either added +// to the index, the chunk structured metadata or entirely dropped. +type OTLPAttributeAction string + +const ( + // OTLPAttributeActionIndexLabel stores a resource attribute as a label, which is part of the index identifying streams. + OTLPAttributeActionIndexLabel OTLPAttributeAction = "indexLabel" + // OTLPAttributeActionStructuredMetadata stores an attribute as structured metadata with each log entry. + OTLPAttributeActionStructuredMetadata OTLPAttributeAction = "structuredMetadata" + // OTLPAttributeActionDrop removes the matching attributes from the log entry. + OTLPAttributeActionDrop OTLPAttributeAction = "drop" +) + +// OTLPAttributesSpec contains the configuration for a set of attributes +// to store them as index labels or structured metadata or drop them altogether. +type OTLPAttributesSpec struct { + // Action defines the indexing action for the selected attributes. They + // can be either added to structured metadata or drop altogether. + // + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=structured_metadata;drop + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action" + Action OTLPAttributeAction `json:"action"` + + // Attributes allows choosing the attributes by listing their names. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names" + Attributes []string `json:"attributes,omitempty"` + + // Regex allows choosing the attributes by matching a regular expression. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression" + Regex string `json:"regex,omitempty"` +} + +// OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes +// to store them as index labels or structured metadata or drop them altogether. +type OTLPResourceAttributesConfigSpec struct { + // Action defines the indexing action for the selected resoure attributes. They + // can be either indexed as labels, added to structured metadata or drop altogether. + // + // +required + // +kubebuilder:validation:Required + // +kubebuilder:validation:Enum=index_label;structured_metadata;drop + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Action" + Action OTLPAttributeAction `json:"action,omitempty"` + + // Attributes is the list of attributes to configure indexing or drop them + // altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attribute Names" + Attributes []string `json:"attributes,omitempty"` + + // Regex allows choosing the attributes by matching a regular expression. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Regular Expression" + Regex string `json:"regex,omitempty"` +} + +// OTLPResourceAttributesSpec contains the configuration for resource attributes +// to store them as index labels or structured metadata or drop them altogether. +type OTLPResourceAttributesSpec struct { + // IgnoreDefaults controls whether to ignore the global configuration for resource attributes + // indexed as labels. + // + // If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors="urn:alm:descriptor:com.tectonic.ui:booleanSwitch",displayName="Ignore Global Defaults" + IgnoreDefaults bool `json:"ignoreDefaults,omitempty"` + + // Attributes contains the configuration for resource attributes + // to store them as index labels or structured metadata or drop them altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Attributes" + Attributes []OTLPResourceAttributesConfigSpec `json:"attributes,omitempty"` +} + +// GlobalOTLPSpec defines which resource, scope and log attributes to +// be stored as index or structured metadata or drop altogether for all +// tenants. +type GlobalOTLPSpec struct { + // IndexedResourceAttributes contains the global configuration for resource attributes + // to store them as index labels or structured metadata or drop them altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Indexed Resource Attributes" + IndexedResourceAttributes []string `json:"indexedResourceAttributes,omitempty"` + + OTLPSpec `json:",omitempty"` +} + +// OTLPSpec defines which resource, scope and log attributes to +// be stored as index or structured metadata or drop altogether +type OTLPSpec struct { + // ResourceAttributes contains the configuration for resource attributes + // to store them as index labels or structured metadata or drop them altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resource Attributes" + ResourceAttributes *OTLPResourceAttributesSpec `json:"resourceAttributes,omitempty"` + + // ScopeAttributes contains the configuration for scope attributes + // to store them as index labels or structured metadata or drop them altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Scope Attributes" + ScopeAttributes []OTLPAttributesSpec `json:"scopeAttributes,omitempty"` + + // LogAttributes contains the configuration for log attributes + // to store them as index labels or structured metadata or drop them altogether. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Log Attributes" + LogAttributes []OTLPAttributesSpec `json:"logAttributes,omitempty"` +} + // RetentionStreamSpec defines a log stream with separate retention time. type RetentionStreamSpec struct { // Days contains the number of days logs are kept. @@ -844,6 +978,14 @@ type LimitsTemplateSpec struct { // +kubebuilder:validation:Optional QueryLimits *QueryLimitSpec `json:"queries,omitempty"` + // OTLP to configure which resource, scope and log attributes + // to store as labels or structured metadata or drop them altogether + // for all tenants. + // + // +optional + // +kubebuilder:validation:Optional + OTLP *GlobalOTLPSpec `json:"otlp,omitempty"` + // Retention defines how long logs are kept in storage. // // +optional @@ -865,6 +1007,14 @@ type PerTenantLimitsTemplateSpec struct { // +kubebuilder:validation:Optional QueryLimits *PerTenantQueryLimitSpec `json:"queries,omitempty"` + // OTLP to configure which resource, scope and log attributes + // to store as labels or structured metadata or drop them altogether + // for a single tenants. + // + // +optional + // +kubebuilder:validation:Optional + OTLP *OTLPSpec `json:"otlp,omitempty"` + // Retention defines how long logs are kept in storage. // // +optional @@ -1313,3 +1463,16 @@ func (t BlockedQueryTypes) String() string { return strings.Join(res, ",") } + +func (a OTLPAttributeAction) Value() string { + switch a { + case OTLPAttributeActionIndexLabel: + return "index_label" + case OTLPAttributeActionStructuredMetadata: + return "structured_metadata" + case OTLPAttributeActionDrop: + return "drop" + default: + return string(a) + } +} diff --git a/operator/apis/loki/v1/v1.go b/operator/apis/loki/v1/v1.go index 8e86ea4b2498..a17e7244dfb5 100644 --- a/operator/apis/loki/v1/v1.go +++ b/operator/apis/loki/v1/v1.go @@ -84,6 +84,13 @@ var ( // ErrIPv6InstanceAddrTypeNotAllowed when the default InstanceAddrType is used with enableIPv6. ErrIPv6InstanceAddrTypeNotAllowed = errors.New(`instanceAddrType "default" cannot be used with enableIPv6 at the same time`) + // ErrOTLPResourceAttributesEmptyNotAllowed when the OTLP ResourceAttributes are empty even though ignoreDefaults is enabled. + ErrOTLPResourceAttributesEmptyNotAllowed = errors.New(`resourceAttributes cannot be empty when ignoreDefaults is true`) + // ErrOTLPResourceAttributesIndexLabelActionMissing when OTLP ResourceAttributes does not contain at least one index label when ignoreDefaults is enabled. + ErrOTLPResourceAttributesIndexLabelActionMissing = errors.New(`resourceAttributes does not contain at least one attributed mapped to "index_label"`) + // ErrOTLPAttributesSpecInvalid when the OTLPAttributesSpec attibutes and regex fields are both empty. + ErrOTLPAttributesSpecInvalid = errors.New(`attributes and regex cannot be empty at the same time`) + // ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing // matchers for a namespace. ErrRuleMustMatchNamespace = errors.New("rule needs to have a matcher for the namespace") diff --git a/operator/apis/loki/v1/zz_generated.deepcopy.go b/operator/apis/loki/v1/zz_generated.deepcopy.go index c7206c5ab660..faab229b2a56 100644 --- a/operator/apis/loki/v1/zz_generated.deepcopy.go +++ b/operator/apis/loki/v1/zz_generated.deepcopy.go @@ -504,6 +504,27 @@ func (in *ClusterProxy) DeepCopy() *ClusterProxy { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GlobalOTLPSpec) DeepCopyInto(out *GlobalOTLPSpec) { + *out = *in + if in.IndexedResourceAttributes != nil { + in, out := &in.IndexedResourceAttributes, &out.IndexedResourceAttributes + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.OTLPSpec.DeepCopyInto(&out.OTLPSpec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GlobalOTLPSpec. +func (in *GlobalOTLPSpec) DeepCopy() *GlobalOTLPSpec { + if in == nil { + return nil + } + out := new(GlobalOTLPSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HashRingSpec) DeepCopyInto(out *HashRingSpec) { *out = *in @@ -579,6 +600,11 @@ func (in *LimitsTemplateSpec) DeepCopyInto(out *LimitsTemplateSpec) { *out = new(QueryLimitSpec) **out = **in } + if in.OTLP != nil { + in, out := &in.OTLP, &out.OTLP + *out = new(GlobalOTLPSpec) + (*in).DeepCopyInto(*out) + } if in.Retention != nil { in, out := &in.Retention, &out.Retention *out = new(RetentionLimitSpec) @@ -1057,6 +1083,102 @@ func (in *OPASpec) DeepCopy() *OPASpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPAttributesSpec) DeepCopyInto(out *OTLPAttributesSpec) { + *out = *in + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPAttributesSpec. +func (in *OTLPAttributesSpec) DeepCopy() *OTLPAttributesSpec { + if in == nil { + return nil + } + out := new(OTLPAttributesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPResourceAttributesConfigSpec) DeepCopyInto(out *OTLPResourceAttributesConfigSpec) { + *out = *in + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPResourceAttributesConfigSpec. +func (in *OTLPResourceAttributesConfigSpec) DeepCopy() *OTLPResourceAttributesConfigSpec { + if in == nil { + return nil + } + out := new(OTLPResourceAttributesConfigSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPResourceAttributesSpec) DeepCopyInto(out *OTLPResourceAttributesSpec) { + *out = *in + if in.Attributes != nil { + in, out := &in.Attributes, &out.Attributes + *out = make([]OTLPResourceAttributesConfigSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPResourceAttributesSpec. +func (in *OTLPResourceAttributesSpec) DeepCopy() *OTLPResourceAttributesSpec { + if in == nil { + return nil + } + out := new(OTLPResourceAttributesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OTLPSpec) DeepCopyInto(out *OTLPSpec) { + *out = *in + if in.ResourceAttributes != nil { + in, out := &in.ResourceAttributes, &out.ResourceAttributes + *out = new(OTLPResourceAttributesSpec) + (*in).DeepCopyInto(*out) + } + if in.ScopeAttributes != nil { + in, out := &in.ScopeAttributes, &out.ScopeAttributes + *out = make([]OTLPAttributesSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.LogAttributes != nil { + in, out := &in.LogAttributes, &out.LogAttributes + *out = make([]OTLPAttributesSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPSpec. +func (in *OTLPSpec) DeepCopy() *OTLPSpec { + if in == nil { + return nil + } + out := new(OTLPSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectStorageSchema) DeepCopyInto(out *ObjectStorageSchema) { *out = *in @@ -1162,6 +1284,11 @@ func (in *PerTenantLimitsTemplateSpec) DeepCopyInto(out *PerTenantLimitsTemplate *out = new(PerTenantQueryLimitSpec) (*in).DeepCopyInto(*out) } + if in.OTLP != nil { + in, out := &in.OTLP, &out.OTLP + *out = new(OTLPSpec) + (*in).DeepCopyInto(*out) + } if in.Retention != nil { in, out := &in.Retention, &out.Retention *out = new(RetentionLimitSpec) diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index 89db0644458c..9df0d3707f9b 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.1 - createdAt: "2024-09-04T13:09:06Z" + createdAt: "2024-09-06T17:56:05Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -361,6 +361,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -457,6 +517,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index 7cfeec6d074f..e58e23492e5a 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -164,6 +164,127 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for all tenants. + properties: + indexedResourceAttributes: + description: |- + IndexedResourceAttributes contains the global configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + type: string + type: array + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. @@ -307,6 +428,120 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for a single tenants. + properties: + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index c431273519e6..305ab8785d31 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.1 - createdAt: "2024-09-04T13:09:04Z" + createdAt: "2024-09-06T17:56:04Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -354,6 +354,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -450,6 +510,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 234ec782eb1b..e4f08108b696 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -164,6 +164,127 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for all tenants. + properties: + indexedResourceAttributes: + description: |- + IndexedResourceAttributes contains the global configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + type: string + type: array + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. @@ -307,6 +428,120 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for a single tenants. + properties: + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index f943d3921ff3..62548fa8217d 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2024-09-04T13:09:07Z" + createdAt: "2024-09-06T17:56:07Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -374,6 +374,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -470,6 +530,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index 4ab2f8aaba2a..2084694a6db8 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -164,6 +164,127 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for all tenants. + properties: + indexedResourceAttributes: + description: |- + IndexedResourceAttributes contains the global configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + type: string + type: array + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. @@ -307,6 +428,120 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for a single tenants. + properties: + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 2429338bd3a6..0180fe98e7ae 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -146,6 +146,127 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for all tenants. + properties: + indexedResourceAttributes: + description: |- + IndexedResourceAttributes contains the global configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + type: string + type: array + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. @@ -289,6 +410,120 @@ spec: format: int32 type: integer type: object + otlp: + description: |- + OTLP to configure which resource, scope and log attributes + to store as labels or structured metadata or drop them altogether + for a single tenants. + properties: + logAttributes: + description: |- + LogAttributes contains the configuration for log attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + resourceAttributes: + description: |- + ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + attributes: + description: |- + Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected resoure attributes. They + can be either indexed as labels, added to structured metadata or drop altogether. + enum: + - index_label + - structured_metadata + - drop + type: string + attributes: + description: |- + Attributes is the list of attributes to configure indexing or drop them + altogether. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + type: object + type: array + ignoreDefaults: + description: |- + IgnoreDefaults controls whether to ignore the global configuration for resource attributes + indexed as labels. + + + If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label. + type: boolean + type: object + scopeAttributes: + description: |- + ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + items: + description: |- + OTLPAttributesSpec contains the configuration for a set of attributes + to store them as index labels or structured metadata or drop them altogether. + properties: + action: + description: |- + Action defines the indexing action for the selected attributes. They + can be either added to structured metadata or drop altogether. + enum: + - structured_metadata + - drop + type: string + attributes: + description: Attributes allows choosing the attributes + by listing their names. + items: + type: string + type: array + regex: + description: Regex allows choosing the attributes + by matching a regular expression. + type: string + required: + - action + type: object + type: array + type: object queries: description: QueryLimits defines the limit applied on querying log streams. diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index b655b250aea3..35946e8e05d9 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -274,6 +274,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -370,6 +430,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml index 7d12fc8ddaad..8e42ee8f4770 100644 --- a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml @@ -267,6 +267,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -363,6 +423,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index d55686c3addc..3d25c9f4c244 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -286,6 +286,66 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: IndexedResourceAttributes contains the global configuration for + resource attributes to store them as index labels or structured metadata + or drop them altogether. + displayName: Indexed Resource Attributes + path: limits.global.otlp.indexedResourceAttributes + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.global.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.global.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.global.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.global.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.global.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.global.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.global.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.global.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.global.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.global.otlp.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -382,6 +442,61 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: LogAttributes contains the configuration for log attributes to + store them as index labels or structured metadata or drop them altogether. + displayName: Log Attributes + path: limits.tenants.otlp.logAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.logAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.logAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.logAttributes[0].regex + - description: ResourceAttributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Resource Attributes + path: limits.tenants.otlp.resourceAttributes + - description: Attributes contains the configuration for resource attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Attributes + path: limits.tenants.otlp.resourceAttributes.attributes + - description: Action defines the indexing action for the selected resoure attributes. + They can be either indexed as labels, added to structured metadata or drop + altogether. + displayName: Action + path: limits.tenants.otlp.resourceAttributes.attributes[0].action + - description: Attributes is the list of attributes to configure indexing or + drop them altogether. + displayName: Attribute Names + path: limits.tenants.otlp.resourceAttributes.attributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.resourceAttributes.attributes[0].regex + - description: "IgnoreDefaults controls whether to ignore the global configuration + for resource attributes indexed as labels. \n If IgnoreDefaults is true, + then this spec needs to contain at least one mapping to a index label." + displayName: Ignore Global Defaults + path: limits.tenants.otlp.resourceAttributes.ignoreDefaults + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: ScopeAttributes contains the configuration for scope attributes + to store them as index labels or structured metadata or drop them altogether. + displayName: Scope Attributes + path: limits.tenants.otlp.scopeAttributes + - description: Action defines the indexing action for the selected attributes. + They can be either added to structured metadata or drop altogether. + displayName: Action + path: limits.tenants.otlp.scopeAttributes[0].action + - description: Attributes allows choosing the attributes by listing their names. + displayName: Attribute Names + path: limits.tenants.otlp.scopeAttributes[0].attributes + - description: Regex allows choosing the attributes by matching a regular expression. + displayName: Regular Expression + path: limits.tenants.otlp.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 3e2cb6a850ac..ee4ee5c5a62b 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -1146,6 +1146,51 @@ a secret. This mode is only supported for certain object storage types in certai +## GlobalOTLPSpec { #loki-grafana-com-v1-GlobalOTLPSpec } +

+(Appears on:LimitsTemplateSpec) +

+
+

GlobalOTLPSpec defines which resource, scope and log attributes to +be stored as index or structured metadata or drop altogether for all +tenants.

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+indexedResourceAttributes
+ +[]string + +		 +(Optional) +

IndexedResourceAttributes contains the global configuration for resource attributes +to store them as index labels or structured metadata or drop them altogether.

+
+OTLPSpec
+ + +OTLPSpec + + +		 +
+ ## HashRingSpec { #loki-grafana-com-v1-HashRingSpec }

(Appears on:LokiStackSpec) @@ -1473,6 +1518,22 @@ QueryLimitSpec +otlp
+ + +GlobalOTLPSpec + + + + +(Optional) +

OTLP to configure which resource, scope and log attributes +to store as labels or structured metadata or drop them altogether +for all tenants.

+ + + + retention
@@ -2601,6 +2662,262 @@ string +## OTLPAttributeAction { #loki-grafana-com-v1-OTLPAttributeAction } +(string alias) +

+(Appears on:OTLPAttributesSpec, OTLPResourceAttributesConfigSpec) +

+
+

OTLPAttributeAction defines the action to executed when indexing +OTLP resource attributes. Resource attributes can be either added +to the index, the chunk structured metadata or entirely dropped.

+
+ + + + + + + + + + + + + + +
ValueDescription

"drop"

OTLPAttributeActionDrop removes the matching attributes from the log entry.

+

"indexLabel"

OTLPAttributeActionIndexLabel stores a resource attribute as a label, which is part of the index identifying streams.

+

"structuredMetadata"

OTLPAttributeActionStructuredMetadata stores an attribute as structured metadata with each log entry.

+
+ +## OTLPAttributesSpec { #loki-grafana-com-v1-OTLPAttributesSpec } +

+(Appears on:OTLPSpec) +

+
+

OTLPAttributesSpec contains the configuration for a set of attributes +to store them as index labels or structured metadata or drop them altogether.

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+action
+ + +OTLPAttributeAction + + +		 +

Action defines the indexing action for the selected attributes. They +can be either added to structured metadata or drop altogether.

+
+attributes
+ +[]string + +		 +(Optional) +

Attributes allows choosing the attributes by listing their names.

+
+regex
+ +string + +		 +(Optional) +

Regex allows choosing the attributes by matching a regular expression.

+
+ +## OTLPResourceAttributesConfigSpec { #loki-grafana-com-v1-OTLPResourceAttributesConfigSpec } +

+(Appears on:OTLPResourceAttributesSpec) +

+
+

OTLPResourceAttributesConfigSpec contains the configuration for a set of resource attributes +to store them as index labels or structured metadata or drop them altogether.

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+action
+ + +OTLPAttributeAction + + +		 +

Action defines the indexing action for the selected resoure attributes. They +can be either indexed as labels, added to structured metadata or drop altogether.

+
+attributes
+ +[]string + +		 +(Optional) +

Attributes is the list of attributes to configure indexing or drop them +altogether.

+
+regex
+ +string + +		 +(Optional) +

Regex allows choosing the attributes by matching a regular expression.

+
+ +## OTLPResourceAttributesSpec { #loki-grafana-com-v1-OTLPResourceAttributesSpec } +

+(Appears on:OTLPSpec) +

+
+

OTLPResourceAttributesSpec contains the configuration for resource attributes +to store them as index labels or structured metadata or drop them altogether.

+
+ + + + + + + + + + + + + + + + + +
FieldDescription
+ignoreDefaults
+ +bool + +		 +(Optional) +

IgnoreDefaults controls whether to ignore the global configuration for resource attributes +indexed as labels.

+

If IgnoreDefaults is true, then this spec needs to contain at least one mapping to a index label.

+
+attributes
+ + +[]OTLPResourceAttributesConfigSpec + + +		 +(Optional) +

Attributes contains the configuration for resource attributes +to store them as index labels or structured metadata or drop them altogether.

+
+ +## OTLPSpec { #loki-grafana-com-v1-OTLPSpec } +

+(Appears on:GlobalOTLPSpec, PerTenantLimitsTemplateSpec) +

+
+

OTLPSpec defines which resource, scope and log attributes to +be stored as index or structured metadata or drop altogether

+
+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+resourceAttributes
+ + +OTLPResourceAttributesSpec + + +		 +(Optional) +

ResourceAttributes contains the configuration for resource attributes +to store them as index labels or structured metadata or drop them altogether.

+
+scopeAttributes
+ + +[]OTLPAttributesSpec + + +		 +(Optional) +

ScopeAttributes contains the configuration for scope attributes +to store them as index labels or structured metadata or drop them altogether.

+
+logAttributes
+ + +[]OTLPAttributesSpec + + +		 +(Optional) +

LogAttributes contains the configuration for log attributes +to store them as index labels or structured metadata or drop them altogether.

+
+ ## ObjectStorageSchema { #loki-grafana-com-v1-ObjectStorageSchema }

(Appears on:LokiStackStorageStatus, ObjectStorageSpec) @@ -2953,6 +3270,22 @@ PerTenantQueryLimitSpec +otlp
+ + +OTLPSpec + + + + +(Optional) +

OTLP to configure which resource, scope and log attributes +to store as labels or structured metadata or drop them altogether +for a single tenants.

+ + + + retention
diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 5e119e6661d5..016e42253f21 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -6012,3 +6012,587 @@ overrides: require.YAMLEq(t, expCfg, string(cfg)) require.YAMLEq(t, expRCfg, string(rCfg)) } + +func TestBuild_ConfigAndRuntimeConfig_OTLPConfigGenerated(t *testing.T) { + expCfg := ` +--- +auth_enabled: true +chunk_store_config: + chunk_cache_config: + embedded_cache: + enabled: true + max_size_mb: 500 +common: + storage: + s3: + endpoint: http://test.default.svc.cluster.local.:9000 + bucketnames: loki + region: us-east + access_key_id: ${AWS_ACCESS_KEY_ID} + secret_access_key: ${AWS_ACCESS_KEY_SECRET} + s3forcepathstyle: true + compactor_grpc_address: loki-compactor-grpc-lokistack-dev.default.svc.cluster.local:9095 + ring: + kvstore: + store: memberlist + heartbeat_period: 5s + heartbeat_timeout: 1m + instance_port: 9095 +compactor: + compaction_interval: 2h + working_directory: /tmp/loki/compactor +distributor: + otlp_config: + default_resource_attributes_as_index_labels: + - foo.bar + - bar.baz +frontend: + tail_proxy_url: http://loki-querier-http-lokistack-dev.default.svc.cluster.local:3100 + compress_responses: true + max_outstanding_per_tenant: 4096 + log_queries_longer_than: 5s +frontend_worker: + frontend_address: loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local:9095 + grpc_client_config: + max_send_msg_size: 104857600 +ingester: + chunk_block_size: 262144 + chunk_encoding: snappy + chunk_idle_period: 1h + chunk_retain_period: 5m + chunk_target_size: 2097152 + flush_op_timeout: 10m + lifecycler: + final_sleep: 0s + join_after: 30s + num_tokens: 512 + ring: + replication_factor: 1 + max_chunk_age: 2h + wal: + enabled: true + dir: /tmp/wal + replay_memory_ceiling: 2147483648 +ingester_client: + grpc_client_config: + max_recv_msg_size: 67108864 + remote_timeout: 1s +# NOTE: Keep the order of keys as in Loki docs +# to enable easy diffs when vendoring newer +# Loki releases. +# (See https://grafana.com/docs/loki/latest/configuration/#limits_config) +# +# Values for not exposed fields are taken from the grafana/loki production +# configuration manifests. +# (See https://github.com/grafana/loki/blob/main/production/ksonnet/loki/config.libsonnet) +limits_config: + ingestion_rate_strategy: global + ingestion_rate_mb: 4 + ingestion_burst_size_mb: 6 + max_label_name_length: 1024 + max_label_value_length: 2048 + max_label_names_per_series: 30 + reject_old_samples: true + reject_old_samples_max_age: 168h + creation_grace_period: 10m + # Keep max_streams_per_user always to 0 to default + # using max_global_streams_per_user always. + # (See https://github.com/grafana/loki/blob/main/pkg/ingester/limiter.go#L73) + max_streams_per_user: 0 + max_line_size: 256000 + max_entries_limit_per_query: 5000 + max_global_streams_per_user: 0 + max_chunks_per_query: 2000000 + max_query_length: 721h + max_query_parallelism: 32 + tsdb_max_query_parallelism: 512 + max_query_series: 500 + cardinality_limit: 100000 + max_streams_matchers_per_query: 1000 + max_cache_freshness_per_query: 10m + split_queries_by_interval: 30m + query_timeout: 1m + volume_enabled: true + volume_max_series: 1000 + per_stream_rate_limit: 5MB + per_stream_rate_limit_burst: 15MB + shard_streams: + enabled: true + desired_rate: 3MB + allow_structured_metadata: true + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: + - res.foo.bar + - res.bar.baz + regex: .* + - action: structured_metadata + attributes: + - res.service.env + regex: .* + scope_attributes: + - action: index_label + attributes: + - scope.foo.bar + - scope.bar.baz + regex: .* + - action: structured_metadata + attributes: + - scope.service.env + regex: .* + log_attributes: + - action: index_label + attributes: + - log.foo.bar + - log.bar.baz + regex: .* + - action: structured_metadata + attributes: + - log.service.env + regex: .* +memberlist: + abort_if_cluster_join_fails: true + advertise_port: 7946 + bind_port: 7946 + join_members: + - loki-gossip-ring-lokistack-dev.default.svc.cluster.local:7946 + max_join_backoff: 1m + max_join_retries: 10 + min_join_backoff: 1s +querier: + engine: + max_look_back_period: 30s + extra_query_delay: 0s + max_concurrent: 2 + query_ingesters_within: 3h + tail_max_duration: 1h +query_range: + align_queries_with_step: true + cache_results: true + max_retries: 5 + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 500 + parallelise_shardable_queries: true +schema_config: + configs: + - from: "2024-01-01" + index: + period: 24h + prefix: index_ + object_store: s3 + schema: v13 + store: tsdb +ruler: + enable_api: true + enable_sharding: true + evaluation_interval: 1m + poll_interval: 1m + external_url: http://alert.me/now + external_labels: + key1: val1 + key2: val2 + alertmanager_url: http://alerthost1,http://alerthost2 + enable_alertmanager_v2: true + enable_alertmanager_discovery: true + alertmanager_refresh_interval: 1m + notification_queue_capacity: 1000 + notification_timeout: 1m + alertmanager_client: + tls_cert_path: "custom/path" + tls_key_path: "custom/key" + tls_ca_path: "custom/CA" + tls_server_name: "custom-servername" + tls_insecure_skip_verify: false + basic_auth_password: "pass" + basic_auth_username: "user" + credentials: "creds" + credentials_file: "cred/file" + type: "auth" + for_outage_tolerance: 10m + for_grace_period: 5m + resend_delay: 2m + remote_write: + enabled: true + config_refresh_period: 1m + client: + name: remote-write-me + url: http://remote.write.me + remote_timeout: 10s + proxy_url: http://proxy.through.me + follow_redirects: true + headers: + more: foryou + less: forme + authorization: + type: bearer + credentials: supersecret + queue_config: + capacity: 1000 + max_shards: 100 + min_shards: 50 + max_samples_per_send: 1000 + batch_send_deadline: 10s + min_backoff: 30ms + max_backoff: 100ms + wal: + dir: /tmp/wal + truncate_frequency: 60m + min_age: 5m + max_age: 4h + rule_path: /tmp/loki + storage: + type: local + local: + directory: /tmp/rules + ring: + kvstore: + store: memberlist +server: + graceful_shutdown_timeout: 5s + grpc_server_min_time_between_pings: '10s' + grpc_server_ping_without_stream_allowed: true + grpc_server_max_concurrent_streams: 1000 + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + http_listen_port: 3100 + http_server_idle_timeout: 30s + http_server_read_timeout: 30s + http_server_write_timeout: 10m0s + log_level: info +storage_config: + tsdb_shipper: + active_index_directory: /tmp/loki/tsdb-index + cache_location: /tmp/loki/tsdb-cache + cache_ttl: 24h + resync_interval: 5m + index_gateway_client: + server_address: dns:///loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local:9095 +tracing: + enabled: false +analytics: + reporting_enabled: true +` + expRCfg := ` +--- +overrides: + test-a: + otlp_config: + resource_attributes: + ignore_defaults: true + attributes_config: + - action: index_label + attributes: + - res.foo.bar + - res.bar.baz + regex: .* + - action: structured_metadata + attributes: + - res.service.env + regex: .* + scope_attributes: + - action: index_label + attributes: + - scope.foo.bar + - scope.bar.baz + regex: .* + - action: structured_metadata + attributes: + - scope.service.env + regex: .* + log_attributes: + - action: index_label + attributes: + - log.foo.bar + - log.bar.baz + regex: .* + - action: structured_metadata + attributes: + - log.service.env + regex: .* +` + opts := Options{ + Stack: lokiv1.LokiStackSpec{ + Replication: &lokiv1.ReplicationSpec{ + Factor: 1, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + IngestionLimits: &lokiv1.IngestionLimitSpec{ + IngestionRate: 4, + IngestionBurstSize: 6, + MaxLabelNameLength: 1024, + MaxLabelValueLength: 2048, + MaxLabelNamesPerSeries: 30, + MaxGlobalStreamsPerTenant: 0, + MaxLineSize: 256000, + PerStreamRateLimit: 5, + PerStreamRateLimitBurst: 15, + PerStreamDesiredRate: 3, + }, + OTLP: &lokiv1.GlobalOTLPSpec{ + IndexedResourceAttributes: []string{ + "foo.bar", + "bar.baz", + }, + OTLPSpec: lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "res.foo.bar", + "res.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "res.service.env", + }, + Regex: ".*", + }, + }, + }, + ScopeAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "scope.foo.bar", + "scope.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "scope.service.env", + }, + Regex: ".*", + }, + }, + LogAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "log.foo.bar", + "log.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "log.service.env", + }, + Regex: ".*", + }, + }, + }, + }, + QueryLimits: &lokiv1.QueryLimitSpec{ + MaxEntriesLimitPerQuery: 5000, + MaxChunksPerQuery: 2000000, + MaxQuerySeries: 500, + QueryTimeout: "1m", + CardinalityLimit: 100000, + MaxVolumeSeries: 1000, + }, + }, + }, + }, + Namespace: "test-ns", + Name: "test", + Compactor: Address{ + FQDN: "loki-compactor-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + FrontendWorker: Address{ + FQDN: "loki-query-frontend-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + GossipRing: GossipRing{ + InstancePort: 9095, + BindPort: 7946, + MembersDiscoveryAddr: "loki-gossip-ring-lokistack-dev.default.svc.cluster.local", + }, + Querier: Address{ + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, + }, + IndexGateway: Address{ + FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", + Port: 9095, + }, + Ruler: Ruler{ + Enabled: true, + RulesStorageDirectory: "/tmp/rules", + EvaluationInterval: "1m", + PollInterval: "1m", + AlertManager: &AlertManagerConfig{ + Notifier: &NotifierConfig{ + TLS: TLSConfig{ + ServerName: ptr.To("custom-servername"), + CertPath: ptr.To("custom/path"), + KeyPath: ptr.To("custom/key"), + CAPath: ptr.To("custom/CA"), + InsecureSkipVerify: ptr.To(false), + }, + BasicAuth: BasicAuth{ + Username: ptr.To("user"), + Password: ptr.To("pass"), + }, + HeaderAuth: HeaderAuth{ + CredentialsFile: ptr.To("cred/file"), + Type: ptr.To("auth"), + Credentials: ptr.To("creds"), + }, + }, + ExternalURL: "http://alert.me/now", + ExternalLabels: map[string]string{ + "key1": "val1", + "key2": "val2", + }, + Hosts: "http://alerthost1,http://alerthost2", + EnableV2: true, + EnableDiscovery: true, + RefreshInterval: "1m", + QueueCapacity: 1000, + Timeout: "1m", + ForOutageTolerance: "10m", + ForGracePeriod: "5m", + ResendDelay: "2m", + }, + RemoteWrite: &RemoteWriteConfig{ + Enabled: true, + RefreshPeriod: "1m", + Client: &RemoteWriteClientConfig{ + Name: "remote-write-me", + URL: "http://remote.write.me", + RemoteTimeout: "10s", + Headers: map[string]string{ + "more": "foryou", + "less": "forme", + }, + ProxyURL: "http://proxy.through.me", + FollowRedirects: true, + BearerToken: "supersecret", + }, + Queue: &RemoteWriteQueueConfig{ + Capacity: 1000, + MaxShards: 100, + MinShards: 50, + MaxSamplesPerSend: 1000, + BatchSendDeadline: "10s", + MinBackOffPeriod: "30ms", + MaxBackOffPeriod: "100ms", + }, + }, + }, + StorageDirectory: "/tmp/loki", + MaxConcurrent: MaxConcurrent{ + AvailableQuerierCPUCores: 2, + }, + WriteAheadLog: WriteAheadLog{ + Directory: "/tmp/wal", + IngesterMemoryRequest: 4 * 1024 * 1024 * 1024, + }, + ObjectStorage: storage.Options{ + SharedStore: lokiv1.ObjectStorageSecretS3, + S3: &storage.S3StorageConfig{ + Endpoint: "http://test.default.svc.cluster.local.:9000", + Region: "us-east", + Buckets: "loki", + ForcePathStyle: true, + }, + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-01-01", + }, + }, + AllowStructuredMetadata: true, + }, + Shippers: []string{"tsdb"}, + EnableRemoteReporting: true, + HTTPTimeouts: HTTPTimeoutConfig{ + IdleTimeout: 30 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 10 * time.Minute, + }, + Overrides: map[string]LokiOverrides{ + "test-a": { + Limits: lokiv1.PerTenantLimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "res.foo.bar", + "res.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "res.service.env", + }, + Regex: ".*", + }, + }, + }, + ScopeAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "scope.foo.bar", + "scope.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "scope.service.env", + }, + Regex: ".*", + }, + }, + LogAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + Attributes: []string{ + "log.foo.bar", + "log.bar.baz", + }, + Regex: ".*", + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{ + "log.service.env", + }, + Regex: ".*", + }, + }, + }, + }, + }, + }, + } + cfg, rCfg, err := Build(opts) + require.NoError(t, err) + require.YAMLEq(t, expCfg, string(cfg)) + require.YAMLEq(t, expRCfg, string(rCfg)) +} diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index dfdf5c900325..01ee6fe9b707 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -107,6 +107,16 @@ compactor: {{- end }} delete_request_store: {{.ObjectStorage.SharedStore}} {{- end }} +{{- if $l := .Stack.Limits.Global }} +{{- with $l.OTLP }}{{- with .IndexedResourceAttributes }} +distributor: + otlp_config: + default_resource_attributes_as_index_labels: + {{- range . }} + - {{ . }} + {{- end }} +{{- end }} +{{- end }}{{- end }} frontend: tail_proxy_url: {{ .Querier.Protocol }}://{{ .Querier.FQDN }}:{{ .Querier.Port }} {{- if .Gates.HTTPEncryption }} @@ -222,6 +232,58 @@ limits_config: shard_streams: enabled: true desired_rate: {{ . }}MB +{{- end }} +{{- with .Stack.Limits.Global.OTLP }} + otlp_config: + {{- with .ResourceAttributes }} + resource_attributes: + ignore_defaults: {{ .IgnoreDefaults }} + {{- with .Attributes}} + attributes_config: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .ScopeAttributes }} + scope_attributes: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with .LogAttributes }} + log_attributes: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- end }} allow_structured_metadata: {{ .ObjectStorage.AllowStructuredMetadata }} {{- with .GossipRing }} diff --git a/operator/internal/manifests/internal/config/loki-runtime-config.yaml b/operator/internal/manifests/internal/config/loki-runtime-config.yaml index 7d5b5e242108..21b75508dc7a 100644 --- a/operator/internal/manifests/internal/config/loki-runtime-config.yaml +++ b/operator/internal/manifests/internal/config/loki-runtime-config.yaml @@ -73,6 +73,58 @@ overrides: {{- end }} {{- end}} {{- end -}} + {{- if $l := $spec.OTLP }} + otlp_config: + {{- with $l.ResourceAttributes }} + resource_attributes: + ignore_defaults: {{ .IgnoreDefaults }} + {{- with .Attributes}} + attributes_config: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end}} + {{- with $l.ScopeAttributes }} + scope_attributes: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- with $l.LogAttributes }} + log_attributes: + {{- range . }} + - action: {{ .Action.Value }} + {{- with .Regex }} + regex: {{ . }} + {{- end }} + {{- with .Attributes }} + attributes: + {{- range . }} + - {{ . }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- with $spec.Retention }} retention_period: {{ .Days }}d {{- with .Streams }} diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index 3246b7ada106..e6c84458c080 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -78,6 +78,16 @@ func (v *LokiStackValidator) validate(ctx context.Context, obj runtime.Object) ( allErrs = append(allErrs, errors...) } + if stack.Spec.Limits != nil { + if stack.Spec.Limits.Global != nil && stack.Spec.Limits.Global.OTLP != nil { + allErrs = append(allErrs, v.validateGlobalOTLPSpec(stack.Spec.Limits.Global.OTLP)...) + } + + if stack.Spec.Limits.Tenants != nil { + allErrs = append(allErrs, v.validatePerTenantOTLPSpec(stack.Spec.Limits.Tenants)...) + } + } + if v.ExtendedValidator != nil { allErrs = append(allErrs, v.ExtendedValidator(ctx, stack)...) } @@ -93,6 +103,100 @@ func (v *LokiStackValidator) validate(ctx context.Context, obj runtime.Object) ( ) } +func (v *LokiStackValidator) validateGlobalOTLPSpec(s *lokiv1.GlobalOTLPSpec) field.ErrorList { + basePath := field.NewPath("spec", "limits", "global") + + return v.validateOTLPSpec(basePath, &s.OTLPSpec) +} + +func (v *LokiStackValidator) validatePerTenantOTLPSpec(tenants map[string]lokiv1.PerTenantLimitsTemplateSpec) field.ErrorList { + var allErrs field.ErrorList + + for key, tenant := range tenants { + basePath := field.NewPath("spec", "limits", "tenants").Key(key) + allErrs = append(allErrs, v.validateOTLPSpec(basePath, tenant.OTLP)...) + } + + return allErrs +} + +func (v *LokiStackValidator) validateOTLPSpec(parent *field.Path, s *lokiv1.OTLPSpec) field.ErrorList { + var allErrs field.ErrorList + + if s.ResourceAttributes != nil && s.ResourceAttributes.IgnoreDefaults { + switch { + case len(s.ResourceAttributes.Attributes) == 0: + allErrs = append(allErrs, + field.Invalid( + parent.Child("otlp", "resourceAttributes"), + []lokiv1.OTLPAttributesSpec{}, + lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), + ), + ) + default: + var indexLabelActionFound bool + for _, attr := range s.ResourceAttributes.Attributes { + if attr.Action == lokiv1.OTLPAttributeActionIndexLabel { + indexLabelActionFound = true + break + } + } + + if !indexLabelActionFound { + allErrs = append(allErrs, + field.Invalid( + parent.Child("otlp", "resourceAttributes"), + s.ResourceAttributes.Attributes, + lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), + ), + ) + } + + for idx, attr := range s.ResourceAttributes.Attributes { + if len(attr.Attributes) == 0 && attr.Regex == "" { + allErrs = append(allErrs, + field.Invalid( + parent.Child("otlp", "resourceAttributes").Index(idx), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + ) + } + } + } + } + + if len(s.ScopeAttributes) != 0 { + for idx, attr := range s.ScopeAttributes { + if len(attr.Attributes) == 0 && attr.Regex == "" { + allErrs = append(allErrs, + field.Invalid( + parent.Child("otlp", "scopeAttributes").Index(idx), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + ) + } + } + } + + if len(s.LogAttributes) != 0 { + for idx, attr := range s.LogAttributes { + if len(attr.Attributes) == 0 && attr.Regex == "" { + allErrs = append(allErrs, + field.Invalid( + parent.Child("otlp", "logAttributes").Index(idx), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + ) + } + } + } + + return allErrs +} + func (v *LokiStackValidator) validateHashRingSpec(s lokiv1.LokiStackSpec) field.ErrorList { if s.HashRing == nil { return nil diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go index e0419cd39565..06f31507cef9 100644 --- a/operator/internal/validation/lokistack_test.go +++ b/operator/internal/validation/lokistack_test.go @@ -390,6 +390,315 @@ var ltt = []struct { }, ), }, + { + desc: "enabling global limits OTLP IgnoreDefaults without resource attributes", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), + []lokiv1.OTLPAttributesSpec{}, + lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), + ), + }, + ), + }, + { + desc: "enabling global limits OTLP IgnoreDefaults without index label action for resource attributes", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{"test"}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), + []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + Attributes: []string{"test"}, + }, + }, + lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), + ), + }, + ), + }, + { + desc: "enabling global limits OTLP IgnoreDefaults with invalid resource attributes config", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes"), + []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + }, + }, + lokiv1.ErrOTLPResourceAttributesIndexLabelActionMissing.Error(), + ), + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes").Index(0), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + }, + ), + }, + { + desc: "enabling global limits OTLP with invalid resource attributes config", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + Attributes: []lokiv1.OTLPResourceAttributesConfigSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + }, + }, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "resourceAttributes").Index(0), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + }, + ), + }, + { + desc: "invalid global OTLP scope attribute specs", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + ScopeAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + }, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "scopeAttributes").Index(0), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "scopeAttributes").Index(1), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + }, + ), + }, + { + desc: "invalid global OTLP log attribute specs", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.GlobalOTLPSpec{ + OTLPSpec: lokiv1.OTLPSpec{ + LogAttributes: []lokiv1.OTLPAttributesSpec{ + { + Action: lokiv1.OTLPAttributeActionIndexLabel, + }, + { + Action: lokiv1.OTLPAttributeActionStructuredMetadata, + }, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "logAttributes").Index(0), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "logAttributes").Index(1), + []string{}, + lokiv1.ErrOTLPAttributesSpecInvalid.Error(), + ), + }, + ), + }, + { + desc: "enabling per-tenant limits OTLP IgnoreDefaults without resource attributes", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2020-10-11", + }, + }, + }, + Limits: &lokiv1.LimitsSpec{ + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "tenant-a": { + OTLP: &lokiv1.OTLPSpec{ + ResourceAttributes: &lokiv1.OTLPResourceAttributesSpec{ + IgnoreDefaults: true, + }, + }, + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "tenants").Key("tenant-a").Child("otlp", "resourceAttributes"), + []lokiv1.OTLPAttributesSpec{}, + lokiv1.ErrOTLPResourceAttributesEmptyNotAllowed.Error(), + ), + }, + ), + }, } func TestLokiStackValidationWebhook_ValidateCreate(t *testing.T) {