From e94152cb995f82f2e183603ccb14ba1056c90edd Mon Sep 17 00:00:00 2001 From: WrenIX Date: Fri, 18 Apr 2025 00:33:48 +0200 Subject: [PATCH] fix(infra-logging): migrate from logging-operator to vector-agent --- infra-logging/config/vector.yaml | 53 +++++ .../templates/configmap_init_crd.yaml | 3 +- .../templates/flow/kube-system/coredns.yaml | 45 ---- .../templates/flow/kube-system/klog.yaml | 52 ----- .../logging-operator/flow/event-tailer.yaml | 29 --- .../logging-operator/flow/fluentbit.yaml | 36 ---- .../flow/logging-operator.yaml | 34 --- .../templates/logging-operator/release.yaml | 202 ------------------ infra-logging/templates/loki/release.yaml | 4 +- infra-logging/templates/vector/configmap.yaml | 6 + .../templates/vector/prometheus-rule.yaml | 24 +++ infra-logging/templates/vector/release.yaml | 42 ++++ .../{logging-operator => vector}/repo.yaml | 5 +- infra-logging/values.yaml | 3 - 14 files changed, 130 insertions(+), 408 deletions(-) create mode 100644 infra-logging/config/vector.yaml delete mode 100644 infra-logging/templates/flow/kube-system/coredns.yaml delete mode 100644 infra-logging/templates/flow/kube-system/klog.yaml delete mode 100644 infra-logging/templates/logging-operator/flow/event-tailer.yaml delete mode 100644 infra-logging/templates/logging-operator/flow/fluentbit.yaml delete mode 100644 infra-logging/templates/logging-operator/flow/logging-operator.yaml delete mode 100644 infra-logging/templates/logging-operator/release.yaml create mode 100644 infra-logging/templates/vector/configmap.yaml create mode 100644 infra-logging/templates/vector/prometheus-rule.yaml create mode 100644 infra-logging/templates/vector/release.yaml rename infra-logging/templates/{logging-operator => vector}/repo.yaml (55%) diff --git a/infra-logging/config/vector.yaml b/infra-logging/config/vector.yaml new file mode 100644 index 0000000..df9170b --- /dev/null +++ b/infra-logging/config/vector.yaml @@ -0,0 +1,53 @@ +data_dir: /vector-data-dir +api: + enabled: true + address: 127.0.0.1:8686 + playground: false +sources: + kubernetes_logs: + type: kubernetes_logs + host_metrics: + filesystem: + devices: + excludes: [binfmt_misc] + filesystems: + excludes: [binfmt_misc] + mountpoints: + excludes: ["*/proc/sys/fs/binfmt_misc"] + type: host_metrics + internal_metrics: + type: internal_metrics +transforms: + logs: + type: remap + inputs: + - kubernetes_logs + source: |- + if !exists(.pod_namespace) { + .pod_namespace = "unknown" + } + if !exists(.metadata) { + .metadata = { + "not found": "unknown" + } + } +sinks: + prom_exporter: + type: prometheus_exporter + inputs: [host_metrics, internal_metrics] + address: 0.0.0.0:9090 + {{- if .Values.loki.enabled }} + loki: + type: loki + inputs: + - logs + endpoint: http://loki:3100 + encoding: + codec: logfmt + labels: + {{` + "pod_labels_*": "{{ kubernetes.pod_labels }}" + "namespace": "{{ kubernetes.pod_namespace }}" + "*": "{{ metadata }}" + `}} + {{- end }} diff --git a/infra-logging/templates/configmap_init_crd.yaml b/infra-logging/templates/configmap_init_crd.yaml index f950f5e..709841b 100644 --- a/infra-logging/templates/configmap_init_crd.yaml +++ b/infra-logging/templates/configmap_init_crd.yaml @@ -6,9 +6,8 @@ metadata: namespace: "{{ .Values.init.namespace }}" data: {{- if and - (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") + (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PodMonitor") (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") - (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }} init: "-1" {{- else }} diff --git a/infra-logging/templates/flow/kube-system/coredns.yaml b/infra-logging/templates/flow/kube-system/coredns.yaml deleted file mode 100644 index 59f9065..0000000 --- a/infra-logging/templates/flow/kube-system/coredns.yaml +++ /dev/null @@ -1,45 +0,0 @@ -{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }} ---- -apiVersion: logging.banzaicloud.io/v1beta1 -kind: Flow -metadata: - name: coredns - namespace: kube-system -spec: - match: - - select: - labels: - k8s-app: "coredns" - filters: - - tag_normaliser: {} - - parser: - reserve_data: true - remove_key_name_field: true - parse: - type: "multi_format" - patterns: - - format: "regexp" - expression: '^\[(?.*)\] \[?(?.*)\]?:(?.*) - (?.*) "(?.*) (?.*) (?.*)\.? (?.*) (?.*) (?.*) (?.*)" (?.*) (?.*) (?.*) (?.*)s' - types: "source.port:integer,dns.id:integer,coredns.query.size:integer,coredns.dnssec_ok:bool,bufsize:integer,dns.header_flags:array,coredns.response.size:integer,coredns.duration:float" - - format: "none" - - record_transformer: - enable_ruby: true - records: - - source.ip: '${ record["source.address"] }' - dns.header_flags: '${ !(record["dns.header_flags"].nil?) ? record["dns.header_flags"].map(&:upcase) : nil }' - event.duration: '${ !(record["coredns.duration"].nil?) ? record["coredns.duration"] * 1000000000 : nil }' - event.kind: "event" - event.category: "network" - event.type: "protocol" - event.outcome: '${ record["dns.response_code"] == "NOERROR" ? "success" : "failure" }' - event.protocol: "dns" - event.module: "coredns" - related.ip: '${ record["source.address"] }' - # for dashboard - fileset.name: "kubernetes" - coredns.query.name: '${ record["dns.question.name"] }' - remove_keys: "coredns.duration,coredns.dnssec_ok" - globalOutputRefs: - - "default" -{{- end }} - diff --git a/infra-logging/templates/flow/kube-system/klog.yaml b/infra-logging/templates/flow/kube-system/klog.yaml deleted file mode 100644 index 92e9568..0000000 --- a/infra-logging/templates/flow/kube-system/klog.yaml +++ /dev/null @@ -1,52 +0,0 @@ -{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }} ---- -apiVersion: logging.banzaicloud.io/v1beta1 -kind: Flow -metadata: - name: klog - namespace: kube-system -spec: - match: - - select: - labels: - k8s-app: "konnectivity-agent" - - select: - labels: - k8s-app: "kube-proxy" - - select: - labels: - app: "snapshot-validation-webhook" - filters: - - tag_normaliser: {} - - parser: - hash_value_field: "klog" - reserve_data: true - remove_key_name_field: true - parse: - type: "multi_format" - patterns: - - format: "regexp" - expression: '(?[A-Z])(?\d{2})(?\d{2})\s+(?