From f043e370e79674ddf292aabf92b5f5fc0be8940f Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 15:15:32 +0100 Subject: [PATCH 01/16] feat(operator): Add configuration option for dropping OTLP attributes --- operator/api/loki/v1/lokistack_types.go | 11 ++ .../loki-operator.clusterserviceversion.yaml | 98 +++++++++++- .../loki.grafana.com_lokistacks.yaml | 146 +++++++++++++++++- .../loki-operator.clusterserviceversion.yaml | 98 +++++++++++- .../loki.grafana.com_lokistacks.yaml | 146 +++++++++++++++++- .../loki-operator.clusterserviceversion.yaml | 98 +++++++++++- .../loki.grafana.com_lokistacks.yaml | 146 +++++++++++++++++- .../bases/loki.grafana.com_lokistacks.yaml | 146 +++++++++++++++++- .../loki-operator.clusterserviceversion.yaml | 96 +++++++++++- .../loki-operator.clusterserviceversion.yaml | 96 +++++++++++- .../loki-operator.clusterserviceversion.yaml | 96 +++++++++++- 11 files changed, 1134 insertions(+), 43 deletions(-) diff --git a/operator/api/loki/v1/lokistack_types.go b/operator/api/loki/v1/lokistack_types.go index 8965eb24aad41..ac3b8f92a8be1 100644 --- a/operator/api/loki/v1/lokistack_types.go +++ b/operator/api/loki/v1/lokistack_types.go @@ -837,8 +837,19 @@ type OTLPSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Stream Labels" StreamLabels *OTLPStreamLabelSpec `json:"streamLabels,omitempty"` + // Drop configures which attributes are dropped from the log entry. + // + // +optional + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Dropped Attributes" + Drop *OTLPMetadataSpec `json:"drop,omitempty"` + // StructuredMetadata configures which attributes are saved in structured metadata. // + // Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + // necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + // the attributes from the entry, they need to be listed in Drop instead. + // // +optional // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Structured Metadata" diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index a3b6d4175fb2a..c1498334c2bcf 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.7.1 - createdAt: "2025-01-16T18:30:58Z" + createdAt: "2025-01-20T14:14:46Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -373,6 +373,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -389,8 +428,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -534,6 +578,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -550,8 +633,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index 00eeb8dc5893a..6879d508090ca 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -170,6 +170,71 @@ spec: Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even enforce the use of some required attributes. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -196,8 +261,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log @@ -413,6 +482,71 @@ spec: The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -439,8 +573,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index 81477cf1b4b27..d9d91351eaac1 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.7.1 - createdAt: "2025-01-16T18:30:55Z" + createdAt: "2025-01-20T14:14:43Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -366,6 +366,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -382,8 +421,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -527,6 +571,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -543,8 +626,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 9bb07dc5c18fb..215c705e3e487 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -171,6 +171,71 @@ spec: Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even enforce the use of some required attributes. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -197,8 +262,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log @@ -414,6 +483,71 @@ spec: The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -440,8 +574,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index 1ad0ef93ad36c..fff95045ec15a 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2025-01-16T18:31:01Z" + createdAt: "2025-01-20T14:14:49Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -386,6 +386,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -402,8 +441,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -547,6 +591,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -563,8 +646,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index 277350061c87f..d98545093d3b5 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -170,6 +170,71 @@ spec: Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even enforce the use of some required attributes. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -196,8 +261,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log @@ -413,6 +482,71 @@ spec: The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -439,8 +573,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 02dbdf73a3927..401e4369aa2a9 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -152,6 +152,71 @@ spec: Tenancy modes can provide a default OTLP configuration, when no custom OTLP configuration is set or even enforce the use of some required attributes. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names of + resource attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of scope + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead of + as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -178,8 +243,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log @@ -395,6 +464,71 @@ spec: The per-tenant configuration for OTLP attributes will be merged with the global configuration. properties: + drop: + description: Drop configures which attributes are dropped + from the log entry. + properties: + logAttributes: + description: LogAttributes lists the names of log + attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + resourceAttributes: + description: ResourceAttributes lists the names + of resource attributes that should be included + in structured metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + scopeAttributes: + description: ScopeAttributes lists the names of + scope attributes that should be included in structured + metadata. + items: + properties: + name: + description: Name contains either a verbatim + name of an attribute or a regular expression + matching many attributes. + type: string + regex: + description: If Regex is true, then Name is + treated as a regular expression instead + of as a verbatim attribute name. + type: boolean + required: + - name + type: object + type: array + type: object streamLabels: description: StreamLabels configures which resource attributes are converted to Loki stream labels. @@ -421,8 +555,12 @@ spec: type: array type: object structuredMetadata: - description: StructuredMetadata configures which attributes - are saved in structured metadata. + description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. properties: logAttributes: description: LogAttributes lists the names of log diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index e9819e168ccbd..11123b1b4c4f8 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -288,6 +288,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -304,8 +343,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -449,6 +493,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -465,8 +548,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be diff --git a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml index 47d07b984e6d2..dd06dfcb5b2b9 100644 --- a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml @@ -281,6 +281,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -297,8 +336,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -442,6 +486,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -458,8 +541,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index f7a44332d7705..ee79e9226ae6c 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -300,6 +300,45 @@ spec: path: limits.global.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.global.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.global.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.global.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.global.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.global.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.global.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -316,8 +355,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.global.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be @@ -461,6 +505,45 @@ spec: path: limits.tenants.ingestion.perStreamRateLimitBurst x-descriptors: - urn:alm:descriptor:com.tectonic.ui:number + - description: Drop configures which attributes are dropped from the log entry. + displayName: Dropped Attributes + path: limits.tenants.otlp.drop + - description: LogAttributes lists the names of log attributes that should be + included in structured metadata. + displayName: Log Attributes + path: limits.tenants.otlp.drop.logAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.logAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.logAttributes[0].regex + - description: ResourceAttributes lists the names of resource attributes that + should be included in structured metadata. + displayName: Resource Attributes + path: limits.tenants.otlp.drop.resourceAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.resourceAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.resourceAttributes[0].regex + - description: ScopeAttributes lists the names of scope attributes that should + be included in structured metadata. + displayName: Scope Attributes + path: limits.tenants.otlp.drop.scopeAttributes + - description: Name contains either a verbatim name of an attribute or a regular + expression matching many attributes. + displayName: Name + path: limits.tenants.otlp.drop.scopeAttributes[0].name + - description: If Regex is true, then Name is treated as a regular expression + instead of as a verbatim attribute name. + displayName: Treat name as regular expression + path: limits.tenants.otlp.drop.scopeAttributes[0].regex - description: StreamLabels configures which resource attributes are converted to Loki stream labels. displayName: Stream Labels @@ -477,8 +560,13 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: StructuredMetadata configures which attributes are saved in structured - metadata. + - description: |- + StructuredMetadata configures which attributes are saved in structured metadata. + + + Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not + necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove + the attributes from the entry, they need to be listed in Drop instead. displayName: Structured Metadata path: limits.tenants.otlp.structuredMetadata - description: LogAttributes lists the names of log attributes that should be From c62b991f525a12139638d0e2e8936db30513d7c1 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 15:29:36 +0100 Subject: [PATCH 02/16] Add support in implementation --- operator/internal/manifests/config_otlp.go | 38 +++++++++++++++++++ .../manifests/internal/config/options.go | 1 + 2 files changed, 39 insertions(+) diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go index 8530b7e63cfb6..3b6c069448937 100644 --- a/operator/internal/manifests/config_otlp.go +++ b/operator/internal/manifests/config_otlp.go @@ -92,6 +92,23 @@ func convertTenantAttributeReferences(otlpSpec *lokiv1.OTLPSpec, base *config.OT convertAttributeReferences(streamLabels.ResourceAttributes, config.OTLPAttributeActionStreamLabel)...) } + if dropLabels := otlpSpec.Drop; dropLabels != nil { + if resAttr := dropLabels.ResourceAttributes; len(resAttr) > 0 { + result.ResourceAttributes = append(result.ResourceAttributes, + convertAttributeReferences(resAttr, config.OTLPAttributeActionDrop)...) + } + + if scopeAttr := dropLabels.ScopeAttributes; len(scopeAttr) > 0 { + result.ScopeAttributes = append(result.ScopeAttributes, + convertAttributeReferences(scopeAttr, config.OTLPAttributeActionDrop)...) + } + + if logAttr := dropLabels.LogAttributes; len(logAttr) > 0 { + result.LogAttributes = append(result.LogAttributes, + convertAttributeReferences(logAttr, config.OTLPAttributeActionDrop)...) + } + } + if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil { if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { result.ResourceAttributes = append(result.ResourceAttributes, @@ -233,6 +250,27 @@ func otlpAttributeConfig(ls *lokiv1.LokiStackSpec) config.OTLPAttributeConfig { } } + if dropLabels := globalOTLP.Drop; dropLabels != nil { + if result.Global == nil { + result.Global = &config.OTLPTenantAttributeConfig{} + } + + if resAttr := dropLabels.ResourceAttributes; len(resAttr) > 0 { + result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, + convertAttributeReferences(resAttr, config.OTLPAttributeActionDrop)...) + } + + if scopeAttr := dropLabels.ScopeAttributes; len(scopeAttr) > 0 { + result.Global.ScopeAttributes = append(result.Global.ScopeAttributes, + convertAttributeReferences(scopeAttr, config.OTLPAttributeActionDrop)...) + } + + if logAttr := dropLabels.LogAttributes; len(logAttr) > 0 { + result.Global.LogAttributes = append(result.Global.LogAttributes, + convertAttributeReferences(logAttr, config.OTLPAttributeActionDrop)...) + } + } + if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil { if result.Global == nil { result.Global = &config.OTLPTenantAttributeConfig{} diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index 365a768724eba..c3e81bfa45ceb 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -269,6 +269,7 @@ type OTLPAttributeAction string const ( OTLPAttributeActionStreamLabel OTLPAttributeAction = "index_label" OTLPAttributeActionMetadata OTLPAttributeAction = "structured_metadata" + OTLPAttributeActionDrop OTLPAttributeAction = "drop" ) type OTLPAttribute struct { From 708f32d6993853163538c59b945e672be14812ba Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 18:00:24 +0100 Subject: [PATCH 03/16] Refactor OTLP validation --- operator/internal/validation/lokistack.go | 58 ++++++++++++----------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index c5eacd8d57299..aba73ea8220e7 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -126,46 +126,50 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe hasGlobalStreamLabels = v.hasOTLPStreamLabel(spec.Limits.Global.OTLP) } - if hasGlobalStreamLabels { - // When the global configuration has at least one stream label, then the configuration will be valid - return nil + errList := field.ErrorList{} + if !hasGlobalStreamLabels && spec.Limits.Tenants == nil { + // No tenant config and no global stream labels -> error + errList = append(errList, field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPGlobalNoStreamLabel.Error(), + ), + ) } - if spec.Limits.Tenants == nil { - // No tenant config and no global stream labels -> error - return field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"), - nil, - lokiv1.ErrOTLPGlobalNoStreamLabel.Error(), - ), - } + errList = append(errList, v.validateOTLPTenantConfiguration(spec, hasGlobalStreamLabels)...) + return errList +} + +func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiStackSpec, hasGlobalStreamLabel bool) (errList field.ErrorList) { + if spec.Limits == nil || spec.Limits.Tenants == nil { + return nil } - errList := field.ErrorList{} + errList = field.ErrorList{} for _, tenant := range spec.Tenants.Authentication { tenantName := tenant.TenantName tenantLimits, ok := spec.Limits.Tenants[tenantName] if !ok || tenantLimits.OTLP == nil { - // No tenant limits defined and no global stream labels -> error - errList = append(errList, field.Invalid( - field.NewPath("spec", "limits", "tenants", tenantName, "otlp"), - nil, - lokiv1.ErrOTLPTenantMissing.Error(), - )) + if !hasGlobalStreamLabel { + // No tenant limits defined and no global stream labels -> error + errList = append(errList, field.Invalid( + field.NewPath("spec", "limits", "tenants", tenantName, "otlp"), + nil, + lokiv1.ErrOTLPTenantMissing.Error(), + )) + } continue } - if v.hasOTLPStreamLabel(tenantLimits.OTLP) { - continue + if !hasGlobalStreamLabel && !v.hasOTLPStreamLabel(tenantLimits.OTLP) { + errList = append(errList, field.Invalid( + field.NewPath("spec", "limits", "tenants", tenantName, "otlp", "streamLabels", "resourceAttributes"), + nil, + lokiv1.ErrOTLPTenantNoStreamLabel.Error(), + )) } - - errList = append(errList, field.Invalid( - field.NewPath("spec", "limits", "tenants", tenantName, "otlp", "streamLabels", "resourceAttributes"), - nil, - lokiv1.ErrOTLPTenantNoStreamLabel.Error(), - )) } return errList From 4c7741b6dabc095597824c149d66fa8cfcf6bb1d Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 19:19:29 +0100 Subject: [PATCH 04/16] Add validation for global drop configuration --- operator/api/loki/v1/v1.go | 2 + operator/internal/validation/lokistack.go | 77 ++++++++++++++++++- .../internal/validation/lokistack_test.go | 49 ++++++++++++ 3 files changed, 125 insertions(+), 3 deletions(-) diff --git a/operator/api/loki/v1/v1.go b/operator/api/loki/v1/v1.go index 77ae0fa417ce8..0cf1e8e584661 100644 --- a/operator/api/loki/v1/v1.go +++ b/operator/api/loki/v1/v1.go @@ -90,6 +90,8 @@ var ( ErrOTLPTenantMissing = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must have an OTLP configuration") // ErrOTLPTenantNoStreamLabel when a tenant is defined but has no stream labels and there also no global stream labels. ErrOTLPTenantNoStreamLabel = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must define at least one stream label") + // ErrOTLPInvalidDrop when a OTLP configuration tries to drop an attribute also listed as stream label or structured metadata + ErrOTLPInvalidDrop = errors.New("can not drop OTLP attributes also listed as stream label or structured metadata") // ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing // matchers for a namespace. diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index aba73ea8220e7..976e92fa82223 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -122,19 +122,19 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe } hasGlobalStreamLabels := false + errList := field.ErrorList{} if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil { hasGlobalStreamLabels = v.hasOTLPStreamLabel(spec.Limits.Global.OTLP) + errList = append(errList, v.checkOTLPInvalidDrop(spec.Limits.Global.OTLP, field.NewPath("spec", "limits", "global", "otlp"))...) } - errList := field.ErrorList{} if !hasGlobalStreamLabels && spec.Limits.Tenants == nil { // No tenant config and no global stream labels -> error errList = append(errList, field.Invalid( field.NewPath("spec", "limits", "global", "otlp", "streamLabels", "resourceAttributes"), nil, lokiv1.ErrOTLPGlobalNoStreamLabel.Error(), - ), - ) + )) } errList = append(errList, v.validateOTLPTenantConfiguration(spec, hasGlobalStreamLabels)...) @@ -170,6 +170,8 @@ func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiSt lokiv1.ErrOTLPTenantNoStreamLabel.Error(), )) } + + errList = append(errList, v.checkOTLPInvalidDrop(tenantLimits.OTLP, field.NewPath("spec", "limits", "tenants", tenantName, "otlp"))...) } return errList @@ -187,6 +189,75 @@ func (v *LokiStackValidator) hasOTLPStreamLabel(otlp *lokiv1.OTLPSpec) bool { return len(otlp.StreamLabels.ResourceAttributes) > 0 } +func (v *LokiStackValidator) checkOTLPInvalidDrop(otlp *lokiv1.OTLPSpec, basePath *field.Path) field.ErrorList { + if otlp.Drop == nil { + return nil + } + + errList := field.ErrorList{} + if streamLabels := otlp.StreamLabels; streamLabels != nil { + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "resourceAttributes"), + otlp.Drop.ResourceAttributes, + streamLabels.ResourceAttributes, + )...) + } + + if metadata := otlp.StructuredMetadata; metadata != nil { + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "resourceAttributes"), + otlp.Drop.ResourceAttributes, + metadata.ResourceAttributes, + )...) + + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "scopeAttributes"), + otlp.Drop.ScopeAttributes, + metadata.ScopeAttributes, + )...) + + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "logAttributes"), + otlp.Drop.LogAttributes, + metadata.LogAttributes, + )...) + } + + return errList +} + +func (v *LokiStackValidator) checkOTLPInvalidDropReference(basePath *field.Path, dropList []lokiv1.OTLPAttributeReference, keepLists ...[]lokiv1.OTLPAttributeReference) field.ErrorList { + attributeNames := map[string]bool{} + errList := field.ErrorList{} + for _, keeps := range keepLists { + for _, attr := range keeps { + if attr.Regex { + // skip regular expressions for this check + continue + } + attributeNames[attr.Name] = true + } + } + + for i, attr := range dropList { + if attr.Regex { + continue + } + + if !attributeNames[attr.Name] { + continue + } + + errList = append(errList, field.Invalid( + basePath.Index(i), + attr.Name, + lokiv1.ErrOTLPInvalidDrop.Error(), + )) + } + + return errList +} + func (v *LokiStackValidator) validateHashRingSpec(s lokiv1.LokiStackSpec) field.ErrorList { if s.HashRing == nil { return nil diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go index 18434c5fa20b2..d2cd420c18f54 100644 --- a/operator/internal/validation/lokistack_test.go +++ b/operator/internal/validation/lokistack_test.go @@ -654,6 +654,55 @@ var ltt = []struct { }, ), }, + { + desc: "lokistack with custom OTLP configuration listing an attribute as both stream-label and drop", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, + }, + }, + Drop: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, + }, + }, + }, + }, + }, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-22", + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "drop", "resourceAttributes").Index(0), + "global.stream.label", + lokiv1.ErrOTLPInvalidDrop.Error(), + ), + }, + ), + }, } func TestLokiStackValidationWebhook_ValidateCreate(t *testing.T) { From 6e1fd39688ccc1c0ded1cddcb5442b96ed465658 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 19:46:16 +0100 Subject: [PATCH 05/16] Extend validation to support global configuration inheritance --- operator/internal/validation/lokistack.go | 53 ++++++++++++++++++----- 1 file changed, 42 insertions(+), 11 deletions(-) diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index 976e92fa82223..58fffbbda5067 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -123,9 +123,12 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe hasGlobalStreamLabels := false errList := field.ErrorList{} + var globalOtlp *lokiv1.OTLPSpec if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil { - hasGlobalStreamLabels = v.hasOTLPStreamLabel(spec.Limits.Global.OTLP) - errList = append(errList, v.checkOTLPInvalidDrop(spec.Limits.Global.OTLP, field.NewPath("spec", "limits", "global", "otlp"))...) + globalOtlp = spec.Limits.Global.OTLP + + hasGlobalStreamLabels = v.hasOTLPStreamLabel(globalOtlp) + errList = append(errList, v.checkOTLPInvalidDrop(field.NewPath("spec", "limits", "global", "otlp"), globalOtlp, nil)...) } if !hasGlobalStreamLabels && spec.Limits.Tenants == nil { @@ -137,11 +140,11 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe )) } - errList = append(errList, v.validateOTLPTenantConfiguration(spec, hasGlobalStreamLabels)...) + errList = append(errList, v.validateOTLPTenantConfiguration(spec, globalOtlp, hasGlobalStreamLabels)...) return errList } -func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiStackSpec, hasGlobalStreamLabel bool) (errList field.ErrorList) { +func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiStackSpec, globalOtlp *lokiv1.OTLPSpec, hasGlobalStreamLabel bool) (errList field.ErrorList) { if spec.Limits == nil || spec.Limits.Tenants == nil { return nil } @@ -171,7 +174,7 @@ func (v *LokiStackValidator) validateOTLPTenantConfiguration(spec *lokiv1.LokiSt )) } - errList = append(errList, v.checkOTLPInvalidDrop(tenantLimits.OTLP, field.NewPath("spec", "limits", "tenants", tenantName, "otlp"))...) + errList = append(errList, v.checkOTLPInvalidDrop(field.NewPath("spec", "limits", "tenants", tenantName, "otlp"), tenantLimits.OTLP, globalOtlp)...) } return errList @@ -189,44 +192,72 @@ func (v *LokiStackValidator) hasOTLPStreamLabel(otlp *lokiv1.OTLPSpec) bool { return len(otlp.StreamLabels.ResourceAttributes) > 0 } -func (v *LokiStackValidator) checkOTLPInvalidDrop(otlp *lokiv1.OTLPSpec, basePath *field.Path) field.ErrorList { +func (v *LokiStackValidator) checkOTLPInvalidDrop(basePath *field.Path, otlp, inheritedOtlp *lokiv1.OTLPSpec) field.ErrorList { if otlp.Drop == nil { return nil } errList := field.ErrorList{} if streamLabels := otlp.StreamLabels; streamLabels != nil { + resourceAttributes := [][]lokiv1.OTLPAttributeReference{ + streamLabels.ResourceAttributes, + } + if inheritedOtlp != nil && inheritedOtlp.StreamLabels != nil && len(inheritedOtlp.StreamLabels.ResourceAttributes) > 0 { + resourceAttributes = append(resourceAttributes, inheritedOtlp.StreamLabels.ResourceAttributes) + } + errList = append(errList, v.checkOTLPInvalidDropReference( basePath.Child("drop", "resourceAttributes"), otlp.Drop.ResourceAttributes, - streamLabels.ResourceAttributes, + resourceAttributes, )...) } if metadata := otlp.StructuredMetadata; metadata != nil { + resourceAttributes := [][]lokiv1.OTLPAttributeReference{ + metadata.ResourceAttributes, + } + if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.ResourceAttributes) > 0 { + resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) + } + errList = append(errList, v.checkOTLPInvalidDropReference( basePath.Child("drop", "resourceAttributes"), otlp.Drop.ResourceAttributes, - metadata.ResourceAttributes, + resourceAttributes, )...) + scopeAttributes := [][]lokiv1.OTLPAttributeReference{ + metadata.ScopeAttributes, + } + if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.ScopeAttributes) > 0 { + resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) + } + errList = append(errList, v.checkOTLPInvalidDropReference( basePath.Child("drop", "scopeAttributes"), otlp.Drop.ScopeAttributes, - metadata.ScopeAttributes, + scopeAttributes, )...) + logAttributes := [][]lokiv1.OTLPAttributeReference{ + metadata.LogAttributes, + } + if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.LogAttributes) > 0 { + resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) + } + errList = append(errList, v.checkOTLPInvalidDropReference( basePath.Child("drop", "logAttributes"), otlp.Drop.LogAttributes, - metadata.LogAttributes, + logAttributes, )...) } return errList } -func (v *LokiStackValidator) checkOTLPInvalidDropReference(basePath *field.Path, dropList []lokiv1.OTLPAttributeReference, keepLists ...[]lokiv1.OTLPAttributeReference) field.ErrorList { +func (v *LokiStackValidator) checkOTLPInvalidDropReference(basePath *field.Path, dropList []lokiv1.OTLPAttributeReference, keepLists [][]lokiv1.OTLPAttributeReference) field.ErrorList { attributeNames := map[string]bool{} errList := field.ErrorList{} for _, keeps := range keepLists { From a1b18d4dac991588cd6b8decb2f77d17a2437cc6 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 20:11:26 +0100 Subject: [PATCH 06/16] Fix tenant validation when it does not contain own attribute list --- operator/internal/validation/lokistack.go | 93 +++++++++---------- .../internal/validation/lokistack_test.go | 72 ++++++++++++++ 2 files changed, 118 insertions(+), 47 deletions(-) diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index 58fffbbda5067..eb01c0f5c6a70 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -198,68 +198,67 @@ func (v *LokiStackValidator) checkOTLPInvalidDrop(basePath *field.Path, otlp, in } errList := field.ErrorList{} + streamAttributes := [][]lokiv1.OTLPAttributeReference{} if streamLabels := otlp.StreamLabels; streamLabels != nil { - resourceAttributes := [][]lokiv1.OTLPAttributeReference{ - streamLabels.ResourceAttributes, - } - if inheritedOtlp != nil && inheritedOtlp.StreamLabels != nil && len(inheritedOtlp.StreamLabels.ResourceAttributes) > 0 { - resourceAttributes = append(resourceAttributes, inheritedOtlp.StreamLabels.ResourceAttributes) - } - - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "resourceAttributes"), - otlp.Drop.ResourceAttributes, - resourceAttributes, - )...) + streamAttributes = append(streamAttributes, streamLabels.ResourceAttributes) } + if inheritedOtlp != nil && inheritedOtlp.StreamLabels != nil && len(inheritedOtlp.StreamLabels.ResourceAttributes) > 0 { + streamAttributes = append(streamAttributes, inheritedOtlp.StreamLabels.ResourceAttributes) + } + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "resourceAttributes"), + otlp.Drop.ResourceAttributes, + streamAttributes, + )...) + resourceAttributes := [][]lokiv1.OTLPAttributeReference{} + scopeAttributes := [][]lokiv1.OTLPAttributeReference{} + logAttributes := [][]lokiv1.OTLPAttributeReference{} if metadata := otlp.StructuredMetadata; metadata != nil { - resourceAttributes := [][]lokiv1.OTLPAttributeReference{ - metadata.ResourceAttributes, - } - if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.ResourceAttributes) > 0 { - resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) - } + resourceAttributes = append(resourceAttributes, metadata.ResourceAttributes) + scopeAttributes = append(scopeAttributes, metadata.ScopeAttributes) + logAttributes = append(logAttributes, metadata.LogAttributes) + } - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "resourceAttributes"), - otlp.Drop.ResourceAttributes, - resourceAttributes, - )...) + if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil { + resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) + scopeAttributes = append(scopeAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) + logAttributes = append(logAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) + } - scopeAttributes := [][]lokiv1.OTLPAttributeReference{ - metadata.ScopeAttributes, - } - if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.ScopeAttributes) > 0 { - resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) - } + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "resourceAttributes"), + otlp.Drop.ResourceAttributes, + resourceAttributes, + )...) - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "scopeAttributes"), - otlp.Drop.ScopeAttributes, - scopeAttributes, - )...) + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "scopeAttributes"), + otlp.Drop.ScopeAttributes, + scopeAttributes, + )...) - logAttributes := [][]lokiv1.OTLPAttributeReference{ - metadata.LogAttributes, - } - if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil && len(inheritedOtlp.StructuredMetadata.LogAttributes) > 0 { - resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) - } - - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "logAttributes"), - otlp.Drop.LogAttributes, - logAttributes, - )...) - } + errList = append(errList, v.checkOTLPInvalidDropReference( + basePath.Child("drop", "logAttributes"), + otlp.Drop.LogAttributes, + logAttributes, + )...) return errList } func (v *LokiStackValidator) checkOTLPInvalidDropReference(basePath *field.Path, dropList []lokiv1.OTLPAttributeReference, keepLists [][]lokiv1.OTLPAttributeReference) field.ErrorList { + if len(dropList) == 0 { + return nil + } + + if len(keepLists) == 0 { + return nil + } + attributeNames := map[string]bool{} errList := field.ErrorList{} + for _, keeps := range keepLists { for _, attr := range keeps { if attr.Regex { diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go index d2cd420c18f54..369417aea1384 100644 --- a/operator/internal/validation/lokistack_test.go +++ b/operator/internal/validation/lokistack_test.go @@ -703,6 +703,78 @@ var ltt = []struct { }, ), }, + { + desc: "lokistack with custom OTLP configuration trying to drop a global metadata attribute in a tenant", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + StreamLabels: &lokiv1.OTLPStreamLabelSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "global.stream.label", + }, + }, + }, + StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "log.attribute", + }, + }, + }, + }, + }, + Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ + "test-tenant": { + OTLP: &lokiv1.OTLPSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "resource.attribute", + }, + }, + LogAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "log.attribute", + }, + }, + }, + }, + }, + }, + }, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-22", + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.Static, + Authentication: []lokiv1.AuthenticationSpec{ + { + TenantName: "test-tenant", + }, + }, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "tenants", "test-tenant", "otlp", "drop", "logAttributes").Index(0), + "log.attribute", + lokiv1.ErrOTLPInvalidDrop.Error(), + ), + }, + ), + }, } func TestLokiStackValidationWebhook_ValidateCreate(t *testing.T) { From 4861ff18524c086a65ae0e69201a18eb5f3f6396 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Mon, 20 Jan 2025 20:55:56 +0100 Subject: [PATCH 07/16] Add validation for dropping OpenShift required OTLP attributes --- operator/internal/validation/lokistack.go | 3 +- .../internal/validation/openshift/otlp.go | 96 +++++++++++++++++++ .../validation/openshift/otlp_test.go | 81 ++++++++++++++++ 3 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 operator/internal/validation/openshift/otlp.go create mode 100644 operator/internal/validation/openshift/otlp_test.go diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index eb01c0f5c6a70..ec5607f79c262 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -13,6 +13,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/webhook/admission" lokiv1 "github.com/grafana/loki/operator/api/loki/v1" + "github.com/grafana/loki/operator/internal/validation/openshift" ) // objectStorageSchemaMap defines the type for mapping a schema version with a date @@ -108,7 +109,7 @@ func (v *LokiStackValidator) validateOTLPConfiguration(spec *lokiv1.LokiStackSpe if spec.Tenants.Mode == lokiv1.OpenshiftLogging { // This tenancy mode always provides stream labels - return nil + return openshift.ValidateOTLPInvalidDrop(spec) } if spec.Tenants.Mode == lokiv1.OpenshiftNetwork { diff --git a/operator/internal/validation/openshift/otlp.go b/operator/internal/validation/openshift/otlp.go new file mode 100644 index 0000000000000..60866b13f6f3e --- /dev/null +++ b/operator/internal/validation/openshift/otlp.go @@ -0,0 +1,96 @@ +package openshift + +import ( + "k8s.io/apimachinery/pkg/util/validation/field" + + lokiv1 "github.com/grafana/loki/operator/api/loki/v1" +) + +var ( + requiredStreamLabels = []string{ + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + } + + optionalStreamLabels = []string{ + "k8s.container.name", + "k8s.cronjob.name", + "k8s.daemonset.name", + "k8s.deployment.name", + "k8s.job.name", + "k8s.node.name", + "k8s.pod.name", + "k8s.statefulset.name", + "kubernetes.container_name", + "kubernetes.host", + "kubernetes.pod_name", + "service.name", + } +) + +// ValidateOTLPInvalidDrop validates that a spec does not drop required OTLP attributes in the openshift-logging tenancy mode. +func ValidateOTLPInvalidDrop(spec *lokiv1.LokiStackSpec) field.ErrorList { + if spec.Limits == nil { + return nil + } + + requiredAttributes := map[string]bool{} + for _, label := range requiredStreamLabels { + requiredAttributes[label] = true + } + + disableRecommendedAttributes := false + if spec.Tenants != nil && spec.Tenants.Openshift != nil && spec.Tenants.Openshift.OTLP != nil { + disableRecommendedAttributes = spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes + } + if !disableRecommendedAttributes { + for _, label := range optionalStreamLabels { + requiredAttributes[label] = true + } + } + + errList := field.ErrorList{} + if spec.Limits.Global != nil && spec.Limits.Global.OTLP != nil { + errList = append(errList, validateOTLPSpec(requiredAttributes, spec.Limits.Global.OTLP, field.NewPath("spec", "limits", "global", "otlp"))...) + } + + if len(spec.Limits.Tenants) > 0 { + for name, tenant := range spec.Limits.Tenants { + if tenant.OTLP != nil { + errList = append(errList, validateOTLPSpec(requiredAttributes, tenant.OTLP, field.NewPath("spec", "limits", "tenants", name, "otlp"))...) + } + } + } + + return errList +} + +func validateOTLPSpec(requiredAttributes map[string]bool, otlp *lokiv1.OTLPSpec, basePath *field.Path) field.ErrorList { + if otlp.Drop == nil { + return nil + } + + errList := field.ErrorList{} + for i, attr := range otlp.Drop.ResourceAttributes { + if attr.Regex { + continue + } + + if !requiredAttributes[attr.Name] { + continue + } + + errList = append(errList, field.Invalid( + basePath.Child("drop", "resourceAttributes").Index(i).Child("name"), + attr.Name, + lokiv1.ErrOTLPInvalidDrop.Error(), + )) + } + + return errList +} diff --git a/operator/internal/validation/openshift/otlp_test.go b/operator/internal/validation/openshift/otlp_test.go new file mode 100644 index 0000000000000..23047621f6afb --- /dev/null +++ b/operator/internal/validation/openshift/otlp_test.go @@ -0,0 +1,81 @@ +package openshift + +import ( + "testing" + + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + + lokiv1 "github.com/grafana/loki/operator/api/loki/v1" +) + +func TestValidateOTLPInvalidDrop(t *testing.T) { + tt := []struct { + desc string + spec lokiv1.LokiStack + err *apierrors.StatusError + }{ + { + desc: "lokistack using openshift-logging tenancy mode trying to remove a required stream label", + spec: lokiv1.LokiStack{ + Spec: lokiv1.LokiStackSpec{ + Limits: &lokiv1.LimitsSpec{ + Global: &lokiv1.LimitsTemplateSpec{ + OTLP: &lokiv1.OTLPSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ + ResourceAttributes: []lokiv1.OTLPAttributeReference{ + { + Name: "kubernetes.namespace_name", + }, + }, + }, + }, + }, + }, + Storage: lokiv1.ObjectStorageSpec{ + Schemas: []lokiv1.ObjectStorageSchema{ + { + Version: lokiv1.ObjectStorageSchemaV13, + EffectiveDate: "2024-10-22", + }, + }, + }, + Tenants: &lokiv1.TenantsSpec{ + Mode: lokiv1.OpenshiftLogging, + }, + }, + }, + err: apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + field.ErrorList{ + field.Invalid( + field.NewPath("spec", "limits", "global", "otlp", "drop", "resourceAttributes").Index(0).Child("name"), + "kubernetes.namespace_name", + lokiv1.ErrOTLPInvalidDrop.Error(), + ), + }, + ), + }, + } + + for _, tc := range tt { + t.Run(tc.desc, func(t *testing.T) { + t.Parallel() + + errList := ValidateOTLPInvalidDrop(&tc.spec.Spec) + if tc.err != nil { + testErr := apierrors.NewInvalid( + schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, + "testing-stack", + errList, + ) + require.Equal(t, tc.err, testErr) + } else { + require.Nil(t, errList) + } + }) + } +} From 61a7d98e11320d4b74a9af96f6e262cb0756d2d8 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 13:42:53 +0100 Subject: [PATCH 08/16] Update API docs --- operator/docs/operator/api.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 7816be60e804c..4bb265312d513 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -2756,6 +2756,20 @@ OTLPStreamLabelSpec +drop
+ + +OTLPMetadataSpec + + + + +(Optional) +

Drop configures which attributes are dropped from the log entry.

+ + + + structuredMetadata
@@ -2766,6 +2780,9 @@ OTLPMetadataSpec (Optional)

StructuredMetadata configures which attributes are saved in structured metadata.

+

Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not +necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove +the attributes from the entry, they need to be listed in Drop instead.

From 69fc46f9907f4f83fc69b8158d0c1b07cf48a7c2 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 13:44:39 +0100 Subject: [PATCH 09/16] Fix generated API code --- operator/api/loki/v1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/operator/api/loki/v1/zz_generated.deepcopy.go b/operator/api/loki/v1/zz_generated.deepcopy.go index 99c8caa49432b..0eafed540163a 100644 --- a/operator/api/loki/v1/zz_generated.deepcopy.go +++ b/operator/api/loki/v1/zz_generated.deepcopy.go @@ -1115,6 +1115,11 @@ func (in *OTLPSpec) DeepCopyInto(out *OTLPSpec) { *out = new(OTLPStreamLabelSpec) (*in).DeepCopyInto(*out) } + if in.Drop != nil { + in, out := &in.Drop, &out.Drop + *out = new(OTLPMetadataSpec) + (*in).DeepCopyInto(*out) + } if in.StructuredMetadata != nil { in, out := &in.StructuredMetadata, &out.StructuredMetadata *out = new(OTLPMetadataSpec) From 6fe2420a490b7bc2db56ea80d2b33db18f15835a Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 13:48:07 +0100 Subject: [PATCH 10/16] Ignore lint errors about deprecated API attribute --- operator/internal/manifests/config_otlp.go | 4 ++-- operator/internal/validation/lokistack.go | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go index 3b6c069448937..b01e30e2aefa0 100644 --- a/operator/internal/manifests/config_otlp.go +++ b/operator/internal/manifests/config_otlp.go @@ -109,7 +109,7 @@ func convertTenantAttributeReferences(otlpSpec *lokiv1.OTLPSpec, base *config.OT } } - if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil { + if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil { // nolint:staticcheck if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { result.ResourceAttributes = append(result.ResourceAttributes, convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...) @@ -271,7 +271,7 @@ func otlpAttributeConfig(ls *lokiv1.LokiStackSpec) config.OTLPAttributeConfig { } } - if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil { + if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil { // nolint:staticcheck if result.Global == nil { result.Global = &config.OTLPTenantAttributeConfig{} } diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index ec5607f79c262..751628aa9958a 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -215,16 +215,16 @@ func (v *LokiStackValidator) checkOTLPInvalidDrop(basePath *field.Path, otlp, in resourceAttributes := [][]lokiv1.OTLPAttributeReference{} scopeAttributes := [][]lokiv1.OTLPAttributeReference{} logAttributes := [][]lokiv1.OTLPAttributeReference{} - if metadata := otlp.StructuredMetadata; metadata != nil { + if metadata := otlp.StructuredMetadata; metadata != nil { // nolint:staticcheck resourceAttributes = append(resourceAttributes, metadata.ResourceAttributes) scopeAttributes = append(scopeAttributes, metadata.ScopeAttributes) logAttributes = append(logAttributes, metadata.LogAttributes) } - if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil { - resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) - scopeAttributes = append(scopeAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) - logAttributes = append(logAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) + if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil { // nolint:staticcheck + resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) // nolint:staticcheck + scopeAttributes = append(scopeAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) // nolint:staticcheck + logAttributes = append(logAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) // nolint:staticcheck } errList = append(errList, v.checkOTLPInvalidDropReference( From d0f411765ef36f1f22ab462463bc7535ed8d9387 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 16:04:28 +0100 Subject: [PATCH 11/16] Remove StructuredMetadata from OTLP configuration options --- operator/api/loki/v1/lokistack_types.go | 11 -- operator/api/loki/v1/zz_generated.deepcopy.go | 5 - .../loki-operator.clusterserviceversion.yaml | 92 +----------- .../loki.grafana.com_lokistacks.yaml | 138 ------------------ .../loki-operator.clusterserviceversion.yaml | 92 +----------- .../loki.grafana.com_lokistacks.yaml | 138 ------------------ .../loki-operator.clusterserviceversion.yaml | 92 +----------- .../loki.grafana.com_lokistacks.yaml | 138 ------------------ .../bases/loki.grafana.com_lokistacks.yaml | 138 ------------------ .../loki-operator.clusterserviceversion.yaml | 90 ------------ .../loki-operator.clusterserviceversion.yaml | 90 ------------ .../loki-operator.clusterserviceversion.yaml | 90 ------------ operator/docs/operator/api.md | 17 --- 13 files changed, 3 insertions(+), 1128 deletions(-) diff --git a/operator/api/loki/v1/lokistack_types.go b/operator/api/loki/v1/lokistack_types.go index ac3b8f92a8be1..801cad5c5f430 100644 --- a/operator/api/loki/v1/lokistack_types.go +++ b/operator/api/loki/v1/lokistack_types.go @@ -843,17 +843,6 @@ type OTLPSpec struct { // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Dropped Attributes" Drop *OTLPMetadataSpec `json:"drop,omitempty"` - - // StructuredMetadata configures which attributes are saved in structured metadata. - // - // Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - // necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - // the attributes from the entry, they need to be listed in Drop instead. - // - // +optional - // +kubebuilder:validation:Optional - // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Structured Metadata" - StructuredMetadata *OTLPMetadataSpec `json:"structuredMetadata,omitempty"` } type OTLPStreamLabelSpec struct { diff --git a/operator/api/loki/v1/zz_generated.deepcopy.go b/operator/api/loki/v1/zz_generated.deepcopy.go index 0eafed540163a..e214c1094fc84 100644 --- a/operator/api/loki/v1/zz_generated.deepcopy.go +++ b/operator/api/loki/v1/zz_generated.deepcopy.go @@ -1120,11 +1120,6 @@ func (in *OTLPSpec) DeepCopyInto(out *OTLPSpec) { *out = new(OTLPMetadataSpec) (*in).DeepCopyInto(*out) } - if in.StructuredMetadata != nil { - in, out := &in.StructuredMetadata, &out.StructuredMetadata - *out = new(OTLPMetadataSpec) - (*in).DeepCopyInto(*out) - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OTLPSpec. diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index c1498334c2bcf..7b93b9c98a0ac 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.7.1 - createdAt: "2025-01-20T14:14:46Z" + createdAt: "2025-01-21T12:58:10Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -428,51 +428,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -633,51 +588,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml index 6879d508090ca..b634e217c4872 100644 --- a/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community-openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -260,75 +260,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names of - resource attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of scope - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying @@ -572,75 +503,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names - of resource attributes that should be included - in structured metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of - scope attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index d9d91351eaac1..6759feaa08f3a 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.7.1 - createdAt: "2025-01-20T14:14:43Z" + createdAt: "2025-01-21T12:58:07Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -421,51 +421,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -626,51 +581,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml index 215c705e3e487..8bdfa823f1a6f 100644 --- a/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/community/manifests/loki.grafana.com_lokistacks.yaml @@ -261,75 +261,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names of - resource attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of scope - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying @@ -573,75 +504,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names - of resource attributes that should be included - in structured metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of - scope attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index fff95045ec15a..e5f13852e042f 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2025-01-20T14:14:49Z" + createdAt: "2025-01-21T12:58:13Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -441,51 +441,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -646,51 +601,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml index d98545093d3b5..31d88f30362ea 100644 --- a/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml +++ b/operator/bundle/openshift/manifests/loki.grafana.com_lokistacks.yaml @@ -260,75 +260,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names of - resource attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of scope - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying @@ -572,75 +503,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names - of resource attributes that should be included - in structured metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of - scope attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml index 401e4369aa2a9..69e21bd380288 100644 --- a/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml +++ b/operator/config/crd/bases/loki.grafana.com_lokistacks.yaml @@ -242,75 +242,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names of - resource attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of scope - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead of - as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying @@ -554,75 +485,6 @@ spec: type: object type: array type: object - structuredMetadata: - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - properties: - logAttributes: - description: LogAttributes lists the names of log - attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - resourceAttributes: - description: ResourceAttributes lists the names - of resource attributes that should be included - in structured metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - scopeAttributes: - description: ScopeAttributes lists the names of - scope attributes that should be included in structured - metadata. - items: - properties: - name: - description: Name contains either a verbatim - name of an attribute or a regular expression - matching many attributes. - type: string - regex: - description: If Regex is true, then Name is - treated as a regular expression instead - of as a verbatim attribute name. - type: boolean - required: - - name - type: object - type: array - type: object type: object queries: description: QueryLimits defines the limit applied on querying diff --git a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml index 11123b1b4c4f8..5a554543c25c9 100644 --- a/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community-openshift/bases/loki-operator.clusterserviceversion.yaml @@ -343,51 +343,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -548,51 +503,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml index dd06dfcb5b2b9..92449288ba336 100644 --- a/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/community/bases/loki-operator.clusterserviceversion.yaml @@ -336,51 +336,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -541,51 +496,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml index ee79e9226ae6c..4626d7024cf12 100644 --- a/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml +++ b/operator/config/manifests/openshift/bases/loki-operator.clusterserviceversion.yaml @@ -355,51 +355,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.global.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.global.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.global.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.global.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.global.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.global.otlp.structuredMetadata.scopeAttributes[0].regex - description: CardinalityLimit defines the cardinality limit for index queries. displayName: Cardinality Limit path: limits.global.queries.cardinalityLimit @@ -560,51 +515,6 @@ spec: instead of as a verbatim attribute name. displayName: Treat name as regular expression path: limits.tenants.otlp.streamLabels.resourceAttributes[0].regex - - description: |- - StructuredMetadata configures which attributes are saved in structured metadata. - - - Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not - necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove - the attributes from the entry, they need to be listed in Drop instead. - displayName: Structured Metadata - path: limits.tenants.otlp.structuredMetadata - - description: LogAttributes lists the names of log attributes that should be - included in structured metadata. - displayName: Log Attributes - path: limits.tenants.otlp.structuredMetadata.logAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.logAttributes[0].regex - - description: ResourceAttributes lists the names of resource attributes that - should be included in structured metadata. - displayName: Resource Attributes - path: limits.tenants.otlp.structuredMetadata.resourceAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.resourceAttributes[0].regex - - description: ScopeAttributes lists the names of scope attributes that should - be included in structured metadata. - displayName: Scope Attributes - path: limits.tenants.otlp.structuredMetadata.scopeAttributes - - description: Name contains either a verbatim name of an attribute or a regular - expression matching many attributes. - displayName: Name - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].name - - description: If Regex is true, then Name is treated as a regular expression - instead of as a verbatim attribute name. - displayName: Treat name as regular expression - path: limits.tenants.otlp.structuredMetadata.scopeAttributes[0].regex - description: Blocked defines the list of rules to block matching queries. displayName: Blocked path: limits.tenants.queries.blocked diff --git a/operator/docs/operator/api.md b/operator/docs/operator/api.md index 4bb265312d513..e158eb6c5a67e 100644 --- a/operator/docs/operator/api.md +++ b/operator/docs/operator/api.md @@ -2768,23 +2768,6 @@ OTLPMetadataSpec

Drop configures which attributes are dropped from the log entry.

- - -structuredMetadata
- - -OTLPMetadataSpec - - - - -(Optional) -

StructuredMetadata configures which attributes are saved in structured metadata.

-

Deprecated: Using StructuredMetadata to specify which attributes are saved as structured metadata is not -necessary, as this is the default behavior. Not listing attributes in StructuredMetadata will not remove -the attributes from the entry, they need to be listed in Drop instead.

- - From a424841020c5def5cdda305b763f8655a980f0b6 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 16:13:51 +0100 Subject: [PATCH 12/16] Remove StructuredMetadata from implementation and tests --- operator/internal/manifests/config_otlp.go | 38 --- .../internal/manifests/config_otlp_test.go | 216 +++--------------- operator/internal/validation/lokistack.go | 33 --- .../internal/validation/lokistack_test.go | 102 +-------- 4 files changed, 30 insertions(+), 359 deletions(-) diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go index b01e30e2aefa0..644ca9fd3f9b7 100644 --- a/operator/internal/manifests/config_otlp.go +++ b/operator/internal/manifests/config_otlp.go @@ -109,23 +109,6 @@ func convertTenantAttributeReferences(otlpSpec *lokiv1.OTLPSpec, base *config.OT } } - if structuredMetadata := otlpSpec.StructuredMetadata; structuredMetadata != nil { // nolint:staticcheck - if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { - result.ResourceAttributes = append(result.ResourceAttributes, - convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...) - } - - if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 { - result.ScopeAttributes = append(result.ScopeAttributes, - convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...) - } - - if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 { - result.LogAttributes = append(result.LogAttributes, - convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...) - } - } - return result } @@ -270,27 +253,6 @@ func otlpAttributeConfig(ls *lokiv1.LokiStackSpec) config.OTLPAttributeConfig { convertAttributeReferences(logAttr, config.OTLPAttributeActionDrop)...) } } - - if structuredMetadata := globalOTLP.StructuredMetadata; structuredMetadata != nil { // nolint:staticcheck - if result.Global == nil { - result.Global = &config.OTLPTenantAttributeConfig{} - } - - if resAttr := structuredMetadata.ResourceAttributes; len(resAttr) > 0 { - result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, - convertAttributeReferences(resAttr, config.OTLPAttributeActionMetadata)...) - } - - if scopeAttr := structuredMetadata.ScopeAttributes; len(scopeAttr) > 0 { - result.Global.ScopeAttributes = append(result.Global.ScopeAttributes, - convertAttributeReferences(scopeAttr, config.OTLPAttributeActionMetadata)...) - } - - if logAttr := structuredMetadata.LogAttributes; len(logAttr) > 0 { - result.Global.LogAttributes = append(result.Global.LogAttributes, - convertAttributeReferences(logAttr, config.OTLPAttributeActionMetadata)...) - } - } } for tenant, tenantLimits := range ls.Limits.Tenants { diff --git a/operator/internal/manifests/config_otlp_test.go b/operator/internal/manifests/config_otlp_test.go index 73f4970c8dbf0..6b809ef4c37c0 100644 --- a/operator/internal/manifests/config_otlp_test.go +++ b/operator/internal/manifests/config_otlp_test.go @@ -81,36 +81,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, }, { - desc: "global metadata", - spec: lokiv1.LokiStackSpec{ - Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.OTLPSpec{ - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "metadata", - }, - }, - }, - }, - }, - }, - }, - wantConfig: config.OTLPAttributeConfig{ - RemoveDefaultLabels: true, - Global: &config.OTLPTenantAttributeConfig{ - ResourceAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"metadata"}, - }, - }, - }, - }, - }, - { - desc: "global combined", + desc: "global with drop", spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ @@ -120,38 +91,12 @@ func TestOtlpAttributeConfig(t *testing.T) { { Name: "stream.label", }, - { - Name: "stream\\.label\\.regex\\..+", - Regex: true, - }, }, }, - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Name: "resource.metadata", - }, - { - Name: "resource.metadata\\.other\\..+", - Regex: true, - }, - }, - ScopeAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "scope.metadata", - }, - { - Name: "scope.metadata\\.other\\..+", - Regex: true, - }, - }, - LogAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "log.metadata", - }, - { - Name: "log.metadata\\.other\\..+", - Regex: true, + Name: "drop.attribute", }, }, }, @@ -164,40 +109,16 @@ func TestOtlpAttributeConfig(t *testing.T) { Global: &config.OTLPTenantAttributeConfig{ ResourceAttributes: []config.OTLPAttribute{ { - Action: config.OTLPAttributeActionStreamLabel, - Names: []string{"stream.label"}, + Action: config.OTLPAttributeActionDrop, + Names: []string{ + "drop.attribute", + }, }, { Action: config.OTLPAttributeActionStreamLabel, - Regex: "stream\\.label\\.regex\\..+", - }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"resource.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "resource.metadata\\.other\\..+", - }, - }, - ScopeAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"scope.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "scope.metadata\\.other\\..+", - }, - }, - LogAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"log.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "log.metadata\\.other\\..+", + Names: []string{ + "stream.label", + }, }, }, }, @@ -271,40 +192,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, }, { - desc: "tenant metadata", - spec: lokiv1.LokiStackSpec{ - Limits: &lokiv1.LimitsSpec{ - Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ - "test-tenant": { - OTLP: &lokiv1.OTLPSpec{ - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "tenant.metadata", - }, - }, - }, - }, - }, - }, - }, - }, - wantConfig: config.OTLPAttributeConfig{ - RemoveDefaultLabels: true, - Tenants: map[string]*config.OTLPTenantAttributeConfig{ - "test-tenant": { - ResourceAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"tenant.metadata"}, - }, - }, - }, - }, - }, - }, - { - desc: "tenant combined", + desc: "tenant with drop", spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ @@ -313,40 +201,14 @@ func TestOtlpAttributeConfig(t *testing.T) { StreamLabels: &lokiv1.OTLPStreamLabelSpec{ ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Name: "tenant.stream.label", - }, - { - Name: `tenant\.stream\.label\.regex\..+`, - Regex: true, + Name: "stream.label", }, }, }, - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ ResourceAttributes: []lokiv1.OTLPAttributeReference{ { - Name: "tenant.resource.metadata", - }, - { - Name: `tenant\.resource.metadata\.other\..+`, - Regex: true, - }, - }, - ScopeAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "tenant.scope.metadata", - }, - { - Name: `tenant\.scope\.metadata\.other\..+`, - Regex: true, - }, - }, - LogAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "tenant.log.metadata", - }, - { - Name: `tenant\.log\.metadata\.other\..+`, - Regex: true, + Name: "drop.attribute", }, }, }, @@ -361,40 +223,16 @@ func TestOtlpAttributeConfig(t *testing.T) { "test-tenant": { ResourceAttributes: []config.OTLPAttribute{ { - Action: config.OTLPAttributeActionStreamLabel, - Names: []string{"tenant.stream.label"}, + Action: config.OTLPAttributeActionDrop, + Names: []string{ + "drop.attribute", + }, }, { Action: config.OTLPAttributeActionStreamLabel, - Regex: "tenant\\.stream\\.label\\.regex\\..+", - }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"tenant.resource.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `tenant\.resource.metadata\.other\..+`, - }, - }, - ScopeAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"tenant.scope.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `tenant\.scope\.metadata\.other\..+`, - }, - }, - LogAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"tenant.log.metadata"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `tenant\.log\.metadata\.other\..+`, + Names: []string{ + "stream.label", + }, }, }, }, @@ -490,7 +328,7 @@ func TestOtlpAttributeConfig(t *testing.T) { wantConfig: openshift.DefaultOTLPAttributes(true), }, { - desc: "openshift-logging defaults with additional custom attributes", + desc: "openshift-logging defaults with drop", spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ @@ -502,7 +340,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, }, }, - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ LogAttributes: []lokiv1.OTLPAttributeReference{ { Name: "custom.log.metadata", @@ -554,7 +392,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, LogAttributes: []config.OTLPAttribute{ { - Action: config.OTLPAttributeActionMetadata, + Action: config.OTLPAttributeActionDrop, Names: []string{"custom.log.metadata"}, }, }, @@ -579,7 +417,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, LogAttributes: []config.OTLPAttribute{ { - Action: config.OTLPAttributeActionMetadata, + Action: config.OTLPAttributeActionDrop, Names: []string{"custom.log.metadata"}, }, }, @@ -603,7 +441,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, }, }, - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ + Drop: &lokiv1.OTLPMetadataSpec{ LogAttributes: []lokiv1.OTLPAttributeReference{ { Name: "custom.log.metadata", @@ -642,7 +480,7 @@ func TestOtlpAttributeConfig(t *testing.T) { }, LogAttributes: []config.OTLPAttribute{ { - Action: config.OTLPAttributeActionMetadata, + Action: config.OTLPAttributeActionDrop, Names: []string{"custom.log.metadata"}, }, }, diff --git a/operator/internal/validation/lokistack.go b/operator/internal/validation/lokistack.go index 751628aa9958a..8ec73456afec3 100644 --- a/operator/internal/validation/lokistack.go +++ b/operator/internal/validation/lokistack.go @@ -212,39 +212,6 @@ func (v *LokiStackValidator) checkOTLPInvalidDrop(basePath *field.Path, otlp, in streamAttributes, )...) - resourceAttributes := [][]lokiv1.OTLPAttributeReference{} - scopeAttributes := [][]lokiv1.OTLPAttributeReference{} - logAttributes := [][]lokiv1.OTLPAttributeReference{} - if metadata := otlp.StructuredMetadata; metadata != nil { // nolint:staticcheck - resourceAttributes = append(resourceAttributes, metadata.ResourceAttributes) - scopeAttributes = append(scopeAttributes, metadata.ScopeAttributes) - logAttributes = append(logAttributes, metadata.LogAttributes) - } - - if inheritedOtlp != nil && inheritedOtlp.StructuredMetadata != nil { // nolint:staticcheck - resourceAttributes = append(resourceAttributes, inheritedOtlp.StructuredMetadata.ResourceAttributes) // nolint:staticcheck - scopeAttributes = append(scopeAttributes, inheritedOtlp.StructuredMetadata.ScopeAttributes) // nolint:staticcheck - logAttributes = append(logAttributes, inheritedOtlp.StructuredMetadata.LogAttributes) // nolint:staticcheck - } - - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "resourceAttributes"), - otlp.Drop.ResourceAttributes, - resourceAttributes, - )...) - - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "scopeAttributes"), - otlp.Drop.ScopeAttributes, - scopeAttributes, - )...) - - errList = append(errList, v.checkOTLPInvalidDropReference( - basePath.Child("drop", "logAttributes"), - otlp.Drop.LogAttributes, - logAttributes, - )...) - return errList } diff --git a/operator/internal/validation/lokistack_test.go b/operator/internal/validation/lokistack_test.go index 369417aea1384..b51098e586045 100644 --- a/operator/internal/validation/lokistack_test.go +++ b/operator/internal/validation/lokistack_test.go @@ -440,15 +440,7 @@ var ltt = []struct { }, Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-tenant": { - OTLP: &lokiv1.OTLPSpec{ - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "custom.resource.attribute", - }, - }, - }, - }, + OTLP: &lokiv1.OTLPSpec{}, }, }, }, @@ -517,15 +509,7 @@ var ltt = []struct { Spec: lokiv1.LokiStackSpec{ Limits: &lokiv1.LimitsSpec{ Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.OTLPSpec{ - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "custom.resource.attribute", - }, - }, - }, - }, + OTLP: &lokiv1.OTLPSpec{}, }, }, Storage: lokiv1.ObjectStorageSpec{ @@ -612,15 +596,7 @@ var ltt = []struct { Limits: &lokiv1.LimitsSpec{ Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ "test-tenant": { - OTLP: &lokiv1.OTLPSpec{ - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "tenant.resource.attribute", - }, - }, - }, - }, + OTLP: &lokiv1.OTLPSpec{}, }, }, }, @@ -703,78 +679,6 @@ var ltt = []struct { }, ), }, - { - desc: "lokistack with custom OTLP configuration trying to drop a global metadata attribute in a tenant", - spec: lokiv1.LokiStack{ - Spec: lokiv1.LokiStackSpec{ - Limits: &lokiv1.LimitsSpec{ - Global: &lokiv1.LimitsTemplateSpec{ - OTLP: &lokiv1.OTLPSpec{ - StreamLabels: &lokiv1.OTLPStreamLabelSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "global.stream.label", - }, - }, - }, - StructuredMetadata: &lokiv1.OTLPMetadataSpec{ - LogAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "log.attribute", - }, - }, - }, - }, - }, - Tenants: map[string]lokiv1.PerTenantLimitsTemplateSpec{ - "test-tenant": { - OTLP: &lokiv1.OTLPSpec{ - Drop: &lokiv1.OTLPMetadataSpec{ - ResourceAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "resource.attribute", - }, - }, - LogAttributes: []lokiv1.OTLPAttributeReference{ - { - Name: "log.attribute", - }, - }, - }, - }, - }, - }, - }, - Storage: lokiv1.ObjectStorageSpec{ - Schemas: []lokiv1.ObjectStorageSchema{ - { - Version: lokiv1.ObjectStorageSchemaV13, - EffectiveDate: "2024-10-22", - }, - }, - }, - Tenants: &lokiv1.TenantsSpec{ - Mode: lokiv1.Static, - Authentication: []lokiv1.AuthenticationSpec{ - { - TenantName: "test-tenant", - }, - }, - }, - }, - }, - err: apierrors.NewInvalid( - schema.GroupKind{Group: "loki.grafana.com", Kind: "LokiStack"}, - "testing-stack", - field.ErrorList{ - field.Invalid( - field.NewPath("spec", "limits", "tenants", "test-tenant", "otlp", "drop", "logAttributes").Index(0), - "log.attribute", - lokiv1.ErrOTLPInvalidDrop.Error(), - ), - }, - ), - }, } func TestLokiStackValidationWebhook_ValidateCreate(t *testing.T) { From 4784d9d8968cef9583f7a73be0934a81d48f0a42 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Tue, 21 Jan 2025 16:17:03 +0100 Subject: [PATCH 13/16] Remove structured_metadata attributes from defaults --- .../internal/manifests/config_otlp_test.go | 44 --------------- .../manifests/internal/config/build_test.go | 24 ++++---- .../manifests/internal/config/options.go | 1 - operator/internal/manifests/openshift/otlp.go | 56 ------------------- 4 files changed, 12 insertions(+), 113 deletions(-) diff --git a/operator/internal/manifests/config_otlp_test.go b/operator/internal/manifests/config_otlp_test.go index 6b809ef4c37c0..0b9d36ba98350 100644 --- a/operator/internal/manifests/config_otlp_test.go +++ b/operator/internal/manifests/config_otlp_test.go @@ -509,22 +509,10 @@ func TestSortOTLPAttributes(t *testing.T) { { desc: "sort", attrs: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"test.a"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.c", - }, { Action: config.OTLPAttributeActionStreamLabel, Names: []string{"test.b"}, }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.a", - }, { Action: config.OTLPAttributeActionStreamLabel, Regex: "test.regex.b", @@ -539,18 +527,6 @@ func TestSortOTLPAttributes(t *testing.T) { Action: config.OTLPAttributeActionStreamLabel, Regex: "test.regex.b", }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"test.a"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.a", - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.c", - }, }, }, { @@ -583,26 +559,14 @@ func TestSortOTLPAttributes(t *testing.T) { Action: config.OTLPAttributeActionStreamLabel, Names: []string{"test.c"}, }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"test.d", "test.e"}, - }, { Action: config.OTLPAttributeActionStreamLabel, Names: []string{"test.b"}, }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"test.f"}, - }, { Action: config.OTLPAttributeActionStreamLabel, Regex: "test.regex.b", }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.a", - }, }, wantAttrs: []config.OTLPAttribute{ { @@ -613,14 +577,6 @@ func TestSortOTLPAttributes(t *testing.T) { Action: config.OTLPAttributeActionStreamLabel, Regex: "test.regex.b", }, - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{"test.d", "test.e", "test.f"}, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: "test.regex.a", - }, }, }, } diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 1bf465b5e08c0..3d9c076656bbd 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -6164,16 +6164,16 @@ limits_config: attributes: - res.foo.bar - res.bar.baz - - action: structured_metadata + - action: drop attributes: - res.service.env scope_attributes: - - action: structured_metadata + - action: drop attributes: - scope.foo.bar - scope.bar.baz log_attributes: - - action: structured_metadata + - action: drop attributes: - log.foo.bar - log.bar.baz @@ -6313,16 +6313,16 @@ overrides: attributes: - res.foo.bar - res.bar.baz - - action: structured_metadata + - action: drop attributes: - res.service.env scope_attributes: - - action: structured_metadata + - action: drop attributes: - scope.foo.bar - scope.bar.baz log_attributes: - - action: structured_metadata + - action: drop attributes: - log.foo.bar - log.bar.baz @@ -6499,7 +6499,7 @@ overrides: }, }, { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "res.service.env", }, @@ -6507,7 +6507,7 @@ overrides: }, ScopeAttributes: []OTLPAttribute{ { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "scope.foo.bar", "scope.bar.baz", @@ -6516,7 +6516,7 @@ overrides: }, LogAttributes: []OTLPAttribute{ { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "log.foo.bar", "log.bar.baz", @@ -6535,7 +6535,7 @@ overrides: }, }, { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "res.service.env", }, @@ -6543,7 +6543,7 @@ overrides: }, ScopeAttributes: []OTLPAttribute{ { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "scope.foo.bar", "scope.bar.baz", @@ -6552,7 +6552,7 @@ overrides: }, LogAttributes: []OTLPAttribute{ { - Action: OTLPAttributeActionMetadata, + Action: OTLPAttributeActionDrop, Names: []string{ "log.foo.bar", "log.bar.baz", diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index c3e81bfa45ceb..5f837cea72e7f 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -268,7 +268,6 @@ type OTLPAttributeAction string const ( OTLPAttributeActionStreamLabel OTLPAttributeAction = "index_label" - OTLPAttributeActionMetadata OTLPAttributeAction = "structured_metadata" OTLPAttributeActionDrop OTLPAttributeAction = "drop" ) diff --git a/operator/internal/manifests/openshift/otlp.go b/operator/internal/manifests/openshift/otlp.go index b74aa2e40aa7b..74c1eb5e02da9 100644 --- a/operator/internal/manifests/openshift/otlp.go +++ b/operator/internal/manifests/openshift/otlp.go @@ -48,61 +48,5 @@ func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig { ) slices.Sort(result.Global.ResourceAttributes[0].Names) - result.Global.ResourceAttributes = append(result.Global.ResourceAttributes, - config.OTLPAttribute{ - Action: config.OTLPAttributeActionMetadata, - Names: []string{ - "k8s.node.uid", - "k8s.pod.uid", - "k8s.replicaset.name", - "process.command_line", - "process.executable.name", - "process.executable.path", - "process.pid", - }, - }, - config.OTLPAttribute{ - Action: config.OTLPAttributeActionMetadata, - Regex: `k8s\.pod\.labels\..+`, - }, - config.OTLPAttribute{ - Action: config.OTLPAttributeActionMetadata, - Regex: `openshift\.labels\..+`, - }, - ) - - result.Global.LogAttributes = []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionMetadata, - Names: []string{ - "k8s.event.level", - "k8s.event.object_ref.api.group", - "k8s.event.object_ref.api.version", - "k8s.event.object_ref.name", - "k8s.event.object_ref.resource", - "k8s.event.request.uri", - "k8s.event.response.code", - "k8s.event.stage", - "k8s.event.user_agent", - "k8s.user.groups", - "k8s.user.username", - "level", - "log.iostream", - }, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `k8s\.event\.annotations\..+`, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `systemd\.t\..+`, - }, - { - Action: config.OTLPAttributeActionMetadata, - Regex: `systemd\.u\..+`, - }, - } - return result } From 07618ae42663d927ba462c69213222e3a332200d Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Thu, 23 Jan 2025 15:14:24 +0100 Subject: [PATCH 14/16] Update text for validation error --- operator/api/loki/v1/v1.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/api/loki/v1/v1.go b/operator/api/loki/v1/v1.go index 0cf1e8e584661..3c5fa54c60621 100644 --- a/operator/api/loki/v1/v1.go +++ b/operator/api/loki/v1/v1.go @@ -90,8 +90,8 @@ var ( ErrOTLPTenantMissing = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must have an OTLP configuration") // ErrOTLPTenantNoStreamLabel when a tenant is defined but has no stream labels and there also no global stream labels. ErrOTLPTenantNoStreamLabel = errors.New("if no global OTLP configuration is present which defines at least one stream label, every tenant must define at least one stream label") - // ErrOTLPInvalidDrop when a OTLP configuration tries to drop an attribute also listed as stream label or structured metadata - ErrOTLPInvalidDrop = errors.New("can not drop OTLP attributes also listed as stream label or structured metadata") + // ErrOTLPInvalidDrop when a OTLP configuration tries to drop an attribute also listed as a stream label + ErrOTLPInvalidDrop = errors.New("can not drop OTLP attribute also listed as a stream label") // ErrRuleMustMatchNamespace indicates that an expression used in an alerting or recording rule is missing // matchers for a namespace. From ededa50736439462bd9bd6f08d239a52879674b1 Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Thu, 23 Jan 2025 15:56:34 +0100 Subject: [PATCH 15/16] Move list of OTLP attributes to common package --- operator/internal/manifests/openshift/otlp.go | 24 ++--------- .../internal/manifests/openshift/otlp/otlp.go | 28 +++++++++++++ .../internal/validation/openshift/otlp.go | 41 ++++--------------- 3 files changed, 38 insertions(+), 55 deletions(-) create mode 100644 operator/internal/manifests/openshift/otlp/otlp.go diff --git a/operator/internal/manifests/openshift/otlp.go b/operator/internal/manifests/openshift/otlp.go index 74c1eb5e02da9..707f4da768f15 100644 --- a/operator/internal/manifests/openshift/otlp.go +++ b/operator/internal/manifests/openshift/otlp.go @@ -4,6 +4,7 @@ import ( "slices" "github.com/grafana/loki/operator/internal/manifests/internal/config" + "github.com/grafana/loki/operator/internal/manifests/openshift/otlp" ) // DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging. @@ -14,15 +15,7 @@ func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig { ResourceAttributes: []config.OTLPAttribute{ { Action: config.OTLPAttributeActionStreamLabel, - Names: []string{ - "k8s.namespace.name", - "kubernetes.namespace_name", - "log_source", - "log_type", - "openshift.cluster.uid", - "openshift.log.source", - "openshift.log.type", - }, + Names: otlp.RequiredAttributes, }, }, }, @@ -33,18 +26,7 @@ func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig { } result.Global.ResourceAttributes[0].Names = append(result.Global.ResourceAttributes[0].Names, - "k8s.container.name", - "k8s.cronjob.name", - "k8s.daemonset.name", - "k8s.deployment.name", - "k8s.job.name", - "k8s.node.name", - "k8s.pod.name", - "k8s.statefulset.name", - "kubernetes.container_name", - "kubernetes.host", - "kubernetes.pod_name", - "service.name", + otlp.RecommendedAttributes..., ) slices.Sort(result.Global.ResourceAttributes[0].Names) diff --git a/operator/internal/manifests/openshift/otlp/otlp.go b/operator/internal/manifests/openshift/otlp/otlp.go new file mode 100644 index 0000000000000..1ff28d1fb7c03 --- /dev/null +++ b/operator/internal/manifests/openshift/otlp/otlp.go @@ -0,0 +1,28 @@ +package otlp + +var ( + RequiredAttributes = []string{ + "k8s.namespace.name", + "kubernetes.namespace_name", + "log_source", + "log_type", + "openshift.cluster.uid", + "openshift.log.source", + "openshift.log.type", + } + + RecommendedAttributes = []string{ + "k8s.container.name", + "k8s.cronjob.name", + "k8s.daemonset.name", + "k8s.deployment.name", + "k8s.job.name", + "k8s.node.name", + "k8s.pod.name", + "k8s.statefulset.name", + "kubernetes.container_name", + "kubernetes.host", + "kubernetes.pod_name", + "service.name", + } +) diff --git a/operator/internal/validation/openshift/otlp.go b/operator/internal/validation/openshift/otlp.go index 60866b13f6f3e..ae8b07beaaef1 100644 --- a/operator/internal/validation/openshift/otlp.go +++ b/operator/internal/validation/openshift/otlp.go @@ -4,33 +4,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" lokiv1 "github.com/grafana/loki/operator/api/loki/v1" -) - -var ( - requiredStreamLabels = []string{ - "k8s.namespace.name", - "kubernetes.namespace_name", - "log_source", - "log_type", - "openshift.cluster.uid", - "openshift.log.source", - "openshift.log.type", - } - - optionalStreamLabels = []string{ - "k8s.container.name", - "k8s.cronjob.name", - "k8s.daemonset.name", - "k8s.deployment.name", - "k8s.job.name", - "k8s.node.name", - "k8s.pod.name", - "k8s.statefulset.name", - "kubernetes.container_name", - "kubernetes.host", - "kubernetes.pod_name", - "service.name", - } + "github.com/grafana/loki/operator/internal/manifests/openshift/otlp" ) // ValidateOTLPInvalidDrop validates that a spec does not drop required OTLP attributes in the openshift-logging tenancy mode. @@ -40,16 +14,15 @@ func ValidateOTLPInvalidDrop(spec *lokiv1.LokiStackSpec) field.ErrorList { } requiredAttributes := map[string]bool{} - for _, label := range requiredStreamLabels { + for _, label := range otlp.RequiredAttributes { requiredAttributes[label] = true } - disableRecommendedAttributes := false - if spec.Tenants != nil && spec.Tenants.Openshift != nil && spec.Tenants.Openshift.OTLP != nil { - disableRecommendedAttributes = spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes - } - if !disableRecommendedAttributes { - for _, label := range optionalStreamLabels { + if spec.Tenants != nil && + spec.Tenants.Openshift != nil && + spec.Tenants.Openshift.OTLP != nil && + !spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes { + for _, label := range otlp.RecommendedAttributes { requiredAttributes[label] = true } } From 54029d50673e04985fa9920e8e426bd89e343afc Mon Sep 17 00:00:00 2001 From: Robert Jacob Date: Thu, 23 Jan 2025 16:43:56 +0100 Subject: [PATCH 16/16] Refactor DefaultOTLPAttributes --- operator/internal/manifests/config_otlp.go | 14 ++++++-- .../internal/manifests/config_otlp_test.go | 26 ++++++++++++-- operator/internal/manifests/openshift/otlp.go | 34 ------------------- .../internal/manifests/openshift/otlp/otlp.go | 21 ++++++++++-- .../internal/validation/openshift/otlp.go | 18 +++++----- 5 files changed, 62 insertions(+), 51 deletions(-) delete mode 100644 operator/internal/manifests/openshift/otlp.go diff --git a/operator/internal/manifests/config_otlp.go b/operator/internal/manifests/config_otlp.go index 644ca9fd3f9b7..04414ad3e48ff 100644 --- a/operator/internal/manifests/config_otlp.go +++ b/operator/internal/manifests/config_otlp.go @@ -6,7 +6,7 @@ import ( lokiv1 "github.com/grafana/loki/operator/api/loki/v1" "github.com/grafana/loki/operator/internal/manifests/internal/config" - "github.com/grafana/loki/operator/internal/manifests/openshift" + "github.com/grafana/loki/operator/internal/manifests/openshift/otlp" ) func defaultOTLPAttributeConfig(ts *lokiv1.TenantsSpec) config.OTLPAttributeConfig { @@ -19,7 +19,17 @@ func defaultOTLPAttributeConfig(ts *lokiv1.TenantsSpec) config.OTLPAttributeConf disableRecommended = ts.Openshift.OTLP.DisableRecommendedAttributes } - return openshift.DefaultOTLPAttributes(disableRecommended) + return config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: otlp.DefaultOTLPAttributes(disableRecommended), + }, + }, + }, + } } func convertAttributeReferences(refs []lokiv1.OTLPAttributeReference, action config.OTLPAttributeAction) []config.OTLPAttribute { diff --git a/operator/internal/manifests/config_otlp_test.go b/operator/internal/manifests/config_otlp_test.go index 0b9d36ba98350..88f3e28ef3127 100644 --- a/operator/internal/manifests/config_otlp_test.go +++ b/operator/internal/manifests/config_otlp_test.go @@ -7,7 +7,7 @@ import ( lokiv1 "github.com/grafana/loki/operator/api/loki/v1" "github.com/grafana/loki/operator/internal/manifests/internal/config" - "github.com/grafana/loki/operator/internal/manifests/openshift" + "github.com/grafana/loki/operator/internal/manifests/openshift/otlp" ) func TestOtlpAttributeConfig(t *testing.T) { @@ -311,7 +311,17 @@ func TestOtlpAttributeConfig(t *testing.T) { Mode: lokiv1.OpenshiftLogging, }, }, - wantConfig: openshift.DefaultOTLPAttributes(false), + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: otlp.DefaultOTLPAttributes(false), + }, + }, + }, + }, }, { desc: "openshift-logging defaults without recommended", @@ -325,7 +335,17 @@ func TestOtlpAttributeConfig(t *testing.T) { }, }, }, - wantConfig: openshift.DefaultOTLPAttributes(true), + wantConfig: config.OTLPAttributeConfig{ + RemoveDefaultLabels: true, + Global: &config.OTLPTenantAttributeConfig{ + ResourceAttributes: []config.OTLPAttribute{ + { + Action: config.OTLPAttributeActionStreamLabel, + Names: otlp.DefaultOTLPAttributes(true), + }, + }, + }, + }, }, { desc: "openshift-logging defaults with drop", diff --git a/operator/internal/manifests/openshift/otlp.go b/operator/internal/manifests/openshift/otlp.go deleted file mode 100644 index 707f4da768f15..0000000000000 --- a/operator/internal/manifests/openshift/otlp.go +++ /dev/null @@ -1,34 +0,0 @@ -package openshift - -import ( - "slices" - - "github.com/grafana/loki/operator/internal/manifests/internal/config" - "github.com/grafana/loki/operator/internal/manifests/openshift/otlp" -) - -// DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging. -func DefaultOTLPAttributes(disableRecommended bool) config.OTLPAttributeConfig { - result := config.OTLPAttributeConfig{ - RemoveDefaultLabels: true, - Global: &config.OTLPTenantAttributeConfig{ - ResourceAttributes: []config.OTLPAttribute{ - { - Action: config.OTLPAttributeActionStreamLabel, - Names: otlp.RequiredAttributes, - }, - }, - }, - } - - if disableRecommended { - return result - } - - result.Global.ResourceAttributes[0].Names = append(result.Global.ResourceAttributes[0].Names, - otlp.RecommendedAttributes..., - ) - slices.Sort(result.Global.ResourceAttributes[0].Names) - - return result -} diff --git a/operator/internal/manifests/openshift/otlp/otlp.go b/operator/internal/manifests/openshift/otlp/otlp.go index 1ff28d1fb7c03..96d9cab6d24c2 100644 --- a/operator/internal/manifests/openshift/otlp/otlp.go +++ b/operator/internal/manifests/openshift/otlp/otlp.go @@ -1,7 +1,11 @@ package otlp +import ( + "slices" +) + var ( - RequiredAttributes = []string{ + requiredAttributes = []string{ "k8s.namespace.name", "kubernetes.namespace_name", "log_source", @@ -11,7 +15,7 @@ var ( "openshift.log.type", } - RecommendedAttributes = []string{ + recommendedAttributes = []string{ "k8s.container.name", "k8s.cronjob.name", "k8s.daemonset.name", @@ -26,3 +30,16 @@ var ( "service.name", } ) + +// DefaultOTLPAttributes provides the required/recommended set of OTLP attributes for OpenShift Logging. +func DefaultOTLPAttributes(disableRecommended bool) []string { + result := append([]string{}, requiredAttributes...) + if disableRecommended { + return result + } + + result = append(result, recommendedAttributes...) + slices.Sort(result) + + return result +} diff --git a/operator/internal/validation/openshift/otlp.go b/operator/internal/validation/openshift/otlp.go index ae8b07beaaef1..9ddf86cd79da1 100644 --- a/operator/internal/validation/openshift/otlp.go +++ b/operator/internal/validation/openshift/otlp.go @@ -13,18 +13,16 @@ func ValidateOTLPInvalidDrop(spec *lokiv1.LokiStackSpec) field.ErrorList { return nil } - requiredAttributes := map[string]bool{} - for _, label := range otlp.RequiredAttributes { - requiredAttributes[label] = true - } - + disableRecommendedAttributes := false if spec.Tenants != nil && spec.Tenants.Openshift != nil && - spec.Tenants.Openshift.OTLP != nil && - !spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes { - for _, label := range otlp.RecommendedAttributes { - requiredAttributes[label] = true - } + spec.Tenants.Openshift.OTLP != nil { + disableRecommendedAttributes = spec.Tenants.Openshift.OTLP.DisableRecommendedAttributes + } + + requiredAttributes := map[string]bool{} + for _, label := range otlp.DefaultOTLPAttributes(disableRecommendedAttributes) { + requiredAttributes[label] = true } errList := field.ErrorList{}