From 83624d37193bb678d423fdb250a822b38754a64e Mon Sep 17 00:00:00 2001 From: WrenIX Date: Sat, 25 Nov 2023 00:55:08 +0100 Subject: [PATCH] fix(infra-logging): init --- base-values/commons.yaml | 3 + base-values/infra.yaml | 9 + infra-logging/.helmignore | 23 ++ infra-logging/Chart.yaml | 6 + .../templates/configmap_init_crd.yaml | 17 ++ .../templates/flow/kube-system/coredns.yaml | 45 ++++ .../templates/flow/kube-system/klog.yaml | 52 +++++ .../logging-operator/flow/event-tailer.yaml | 29 +++ .../logging-operator/flow/fluentbit.yaml | 36 ++++ .../flow/logging-operator.yaml | 34 +++ .../templates/logging-operator/release.yaml | 201 ++++++++++++++++++ .../templates/logging-operator/repo.yaml | 9 + .../loki/configmap_grafana_datasource.yaml | 17 ++ infra-logging/templates/loki/release.yaml | 57 +++++ infra-logging/templates/loki/repo.yaml | 10 + infra-logging/values.yaml | 34 +++ 16 files changed, 582 insertions(+) create mode 100644 infra-logging/.helmignore create mode 100644 infra-logging/Chart.yaml create mode 100644 infra-logging/templates/configmap_init_crd.yaml create mode 100644 infra-logging/templates/flow/kube-system/coredns.yaml create mode 100644 infra-logging/templates/flow/kube-system/klog.yaml create mode 100644 infra-logging/templates/logging-operator/flow/event-tailer.yaml create mode 100644 infra-logging/templates/logging-operator/flow/fluentbit.yaml create mode 100644 infra-logging/templates/logging-operator/flow/logging-operator.yaml create mode 100644 infra-logging/templates/logging-operator/release.yaml create mode 100644 infra-logging/templates/logging-operator/repo.yaml create mode 100644 infra-logging/templates/loki/configmap_grafana_datasource.yaml create mode 100644 infra-logging/templates/loki/release.yaml create mode 100644 infra-logging/templates/loki/repo.yaml create mode 100644 infra-logging/values.yaml diff --git a/base-values/commons.yaml b/base-values/commons.yaml index de2258e..ce54d8c 100644 --- a/base-values/commons.yaml +++ b/base-values/commons.yaml @@ -16,6 +16,9 @@ commons: annotations: grafana: + datasource: + labels: + grafana_datasource: "1" dashboards: labels: grafana_dashboard: "1" diff --git a/base-values/infra.yaml b/base-values/infra.yaml index 16e8079..e194967 100644 --- a/base-values/infra.yaml +++ b/base-values/infra.yaml @@ -16,3 +16,12 @@ components: infra-certificates: enabled: true namespace: "certificates" + + infra-logging: + enabled: true + namespace: "logging" + values: + grafana: + dashboards: + annotations: + grafana.mon.local/dashboard-folder: "Logging" diff --git a/infra-logging/.helmignore b/infra-logging/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/infra-logging/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/infra-logging/Chart.yaml b/infra-logging/Chart.yaml new file mode 100644 index 0000000..663e8ee --- /dev/null +++ b/infra-logging/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: infra-logging +description: Install all logging related + +type: application +version: 0.1.0 diff --git a/infra-logging/templates/configmap_init_crd.yaml b/infra-logging/templates/configmap_init_crd.yaml new file mode 100644 index 0000000..f950f5e --- /dev/null +++ b/infra-logging/templates/configmap_init_crd.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-init + namespace: "{{ .Values.init.namespace }}" +data: + {{- if and + (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") + (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") + (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") + }} + init: "-1" + {{- else }} + init: "{{ add1 .Values.init.version }}" + {{- end }} + diff --git a/infra-logging/templates/flow/kube-system/coredns.yaml b/infra-logging/templates/flow/kube-system/coredns.yaml new file mode 100644 index 0000000..59f9065 --- /dev/null +++ b/infra-logging/templates/flow/kube-system/coredns.yaml @@ -0,0 +1,45 @@ +{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }} +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: coredns + namespace: kube-system +spec: + match: + - select: + labels: + k8s-app: "coredns" + filters: + - tag_normaliser: {} + - parser: + reserve_data: true + remove_key_name_field: true + parse: + type: "multi_format" + patterns: + - format: "regexp" + expression: '^\[(?.*)\] \[?(?.*)\]?:(?.*) - (?.*) "(?.*) (?.*) (?.*)\.? (?.*) (?.*) (?.*) (?.*)" (?.*) (?.*) (?.*) (?.*)s' + types: "source.port:integer,dns.id:integer,coredns.query.size:integer,coredns.dnssec_ok:bool,bufsize:integer,dns.header_flags:array,coredns.response.size:integer,coredns.duration:float" + - format: "none" + - record_transformer: + enable_ruby: true + records: + - source.ip: '${ record["source.address"] }' + dns.header_flags: '${ !(record["dns.header_flags"].nil?) ? record["dns.header_flags"].map(&:upcase) : nil }' + event.duration: '${ !(record["coredns.duration"].nil?) ? record["coredns.duration"] * 1000000000 : nil }' + event.kind: "event" + event.category: "network" + event.type: "protocol" + event.outcome: '${ record["dns.response_code"] == "NOERROR" ? "success" : "failure" }' + event.protocol: "dns" + event.module: "coredns" + related.ip: '${ record["source.address"] }' + # for dashboard + fileset.name: "kubernetes" + coredns.query.name: '${ record["dns.question.name"] }' + remove_keys: "coredns.duration,coredns.dnssec_ok" + globalOutputRefs: + - "default" +{{- end }} + diff --git a/infra-logging/templates/flow/kube-system/klog.yaml b/infra-logging/templates/flow/kube-system/klog.yaml new file mode 100644 index 0000000..92e9568 --- /dev/null +++ b/infra-logging/templates/flow/kube-system/klog.yaml @@ -0,0 +1,52 @@ +{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }} +--- +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: klog + namespace: kube-system +spec: + match: + - select: + labels: + k8s-app: "konnectivity-agent" + - select: + labels: + k8s-app: "kube-proxy" + - select: + labels: + app: "snapshot-validation-webhook" + filters: + - tag_normaliser: {} + - parser: + hash_value_field: "klog" + reserve_data: true + remove_key_name_field: true + parse: + type: "multi_format" + patterns: + - format: "regexp" + expression: '(?[A-Z])(?\d{2})(?\d{2})\s+(?