diff --git a/fleet/lib/fluent-bit-external/fleet.yaml b/fleet/lib/fluent-bit-external/fleet.yaml index fa8a32190a..3b1cedab3c 100644 --- a/fleet/lib/fluent-bit-external/fleet.yaml +++ b/fleet/lib/fluent-bit-external/fleet.yaml @@ -17,3 +17,9 @@ dependsOn: - selector: matchLabels: bundle: fluent-bit-external-pre + +targetCustomizations: + - name: Ayekan + clusterSelector: + matchLabels: + name: ayekan \ No newline at end of file diff --git a/fleet/lib/fluent-bit-external/values.yaml b/fleet/lib/fluent-bit-external/values.yaml index 71f0311ff5..9d7b5faec9 100644 --- a/fleet/lib/fluent-bit-external/values.yaml +++ b/fleet/lib/fluent-bit-external/values.yaml @@ -360,6 +360,59 @@ config: Json_date_key date Json_date_format epoch + + # Loki syslog logs output + [Output] + Name loki + Match_Regex (?:rsyslog)\.(.*) + Host loki.kueyen.dev.lsst.org + Port 443 + tls.verify false + Labels job="syslog", host="$host", ident="$ident" + Label_Keys facility,priority,program + Remove_Keys time,message + Line_Format json + Buffer_Size 1M + + # Loki firewall logs output + [Output] + Name loki + Match_Regex (?:firewall)\.(.*) + Host loki.kueyen.dev.lsst.org + Port 443 + tls.verify false + Labels job="firewall", host="$host" + Label_Keys facility,priority,program + Remove_Keys time + Line_Format json + Buffer_Size 1M + + # Loki networking logs output + [Output] + Name loki + Match_Regex (?:network)\.(.*) + Host loki.kueyen.dev.lsst.org + Port 443 + tls.verify false + Labels job="network", host="$host" + Label_Keys facility,priority,program + Remove_Keys time + Line_Format json + Buffer_Size 1M + + # Loki sudo logs output + [Output] + Name loki + Match sudo.* + Host loki.kueyen.dev.lsst.org + Port 443 + tls.verify false + Labels job="sudo", host="$host", user="$user", command="$command" + Label_Keys tty,working_directory,sudo_user + Remove_Keys time + Line_Format json + Buffer_Size 1M + ## https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/classic-mode/upstream-servers ## This configuration is deprecated, please use `extraFiles` instead. upstream: {} diff --git a/fleet/lib/fluent-operator/client/clusterfilter.yaml b/fleet/lib/fluent-operator/client/clusterfilter.yaml new file mode 100644 index 0000000000..d32b3fad81 --- /dev/null +++ b/fleet/lib/fluent-operator/client/clusterfilter.yaml @@ -0,0 +1,45 @@ +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterFilter +metadata: + name: kubernetes-metadata + namespace: fluent-operator +spec: + filters: + - kubernetes: + labels: true + annotations: false + mergeLog: true + keepLog: false + k8sLoggingParser: true + k8sLoggingExclude: true + - lua: + script: | + function containerd(tag, timestamp, record) + if(record["logtag"]~=nil) then + timeStr = os.date("!*t", timestamp["sec"]) + t = string.format("%4d-%02d-%02dT%02d:%02d:%02d.%sZ", + timeStr["year"], timeStr["month"], timeStr["day"], + timeStr["hour"], timeStr["min"], timeStr["sec"], + timestamp["nsec"]) + record["time"] = t + record["log"] = record["message"] + record["message"] = nil + return 1, timestamp, record + else + return 0, timestamp, record + end + end + call: containerd + - nest: + operation: lift + nested_under: kubernetes + add_prefix: kubernetes_ + - modify: + add: + - cluster: ${CLUSTER_NAME} + remove: + - stream + - kubernetes_pod_id + - kubernetes_host + - kubernetes_container_hash + match: kube.* \ No newline at end of file diff --git a/fleet/lib/fluent-operator/client/clustermultilineparser.yaml b/fleet/lib/fluent-operator/client/clustermultilineparser.yaml new file mode 100644 index 0000000000..2c79de5998 --- /dev/null +++ b/fleet/lib/fluent-operator/client/clustermultilineparser.yaml @@ -0,0 +1,21 @@ +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterMultilineParser +metadata: + name: multiline-parser + namespace: fluent-operator + labels: + fluentbit.fluent.io/enabled: "true" +spec: + parsers: + - name: multiline-regex + type: regex + # PatrĂ³n para logs de Java/Go stack traces + rules: + - state: start_state + pattern: '^\d{4}-\d{2}-\d{2}\s\d{2}:\d{2}:\d{2}' + - state: cont + pattern: '^\s+at' + - state: cont + pattern: '^\s+...' + - state: cont + pattern: '^Caused by:' \ No newline at end of file diff --git a/fleet/lib/fluent-operator/client/clusteroutput-fluentbit-loki.yaml b/fleet/lib/fluent-operator/client/clusteroutput-fluentbit-loki.yaml new file mode 100644 index 0000000000..2622fa44b0 --- /dev/null +++ b/fleet/lib/fluent-operator/client/clusteroutput-fluentbit-loki.yaml @@ -0,0 +1,31 @@ +apiVersion: fluentbit.fluent.io/v1alpha2 +kind: ClusterOutput +metadata: + labels: + fluentbit.fluent.io/enabled: "true" + name: fluentbit-output-loki + namespace: fluent-operator +spec: + matchRegex: (?:kube|service)\.(.*) + loki: + host: loki.kueyen.dev.lsst.org + port: 443 + tls: + verify: false + labels: + - job=fluentbit + - k8s_namespace=$kubernetes['namespace_name'] + - k8s_pod_name=$kubernetes['pod_name'] + - k8s_container_name=$kubernetes['container_name'] + - k8s_host=$kubernetes['host'] + - k8s_image=$kubernetes['container_image'] + - cluster="${CLUSTER_NAME}" + label_keys: + - app + - release + - component + - k8s-app + line_format: json + auto_kubernetes_labels: true + retry_limit: 3 + buffer_size: "100MB" diff --git a/fleet/lib/fluent-operator/client/configmap-cluster-metadata.yaml b/fleet/lib/fluent-operator/client/configmap-cluster-metadata.yaml new file mode 100644 index 0000000000..fc5fa03796 --- /dev/null +++ b/fleet/lib/fluent-operator/client/configmap-cluster-metadata.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-metadata + namespace: fluent-operator +data: + CLUSTER_NAME: ${CLUSTER_NAME} diff --git a/fleet/lib/fluent-operator/client/kustomization.yaml b/fleet/lib/fluent-operator/client/kustomization.yaml new file mode 100644 index 0000000000..6f0172b9bf --- /dev/null +++ b/fleet/lib/fluent-operator/client/kustomization.yaml @@ -0,0 +1,5 @@ +--- +resources: + - clusteroutput-fluentbit-loki.yaml + - configmap-cluster-metadata.yaml + - clusterfilter.yaml \ No newline at end of file diff --git a/fleet/lib/fluent-operator/fleet.yaml b/fleet/lib/fluent-operator/fleet.yaml new file mode 100644 index 0000000000..3df4f613a2 --- /dev/null +++ b/fleet/lib/fluent-operator/fleet.yaml @@ -0,0 +1,48 @@ +--- +defaultNamespace: &name fluent-operator +labels: + bundle: *name +namespaceLabels: + lsst.io/discover: "true" +kustomize: + dir: client +helm: + chart: *name + releaseName: *name + repo: https://fluent.github.io/helm-charts + version: 3.2.0 + waitForJobs: true + values: + clusterName: ${CLUSTER_NAME} + fluentbit: + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + labels: + lsst.team.io/infra: "true" + lsst.io/role: logging + lsst.io/component: fluent-bit + lsst.io/env: kube + lsst.io/project: o11y + extraEnv: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CLUSTER_NAME + valueFrom: + configMapKeyRef: + name: cluster-metadata + key: CLUSTER_NAME + +targetCustomizations: + - name: kueyen + clusterName: kueyen + - name: ayekan + clusterName: ayekan + - name: ruka + clusterName: ruka