fix(infra-logging): init

base-values/commons.yaml

@@ -16,6 +16,9 @@ commons:
     annotations:
 
   grafana:
+    datasource:
+      labels:
+        grafana_datasource: "1"
     dashboards:
       labels:
         grafana_dashboard: "1"

base-values/infra.yaml

@@ -16,3 +16,12 @@ components:
   infra-certificates:
     enabled: true
     namespace: "certificates"
+
+  infra-logging:
+    enabled: true
+    namespace: "logging"
+    values:
+      grafana:
+        dashboards:
+          annotations:
+            grafana.mon.local/dashboard-folder: "Logging"

infra-logging/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

infra-logging/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: infra-logging
+description: Install all logging related
+
+type: application
+version: 0.1.0

infra-logging/templates/configmap_init_crd.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-init
+  namespace: "{{ .Values.init.namespace }}"
+data:
+  {{- if and
+    (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor")
+    (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule")
+    (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow")
+  }}
+  init: "-1"
+  {{- else }}
+  init: "{{ add1 .Values.init.version }}"
+  {{- end }}
+

infra-logging/templates/flow/kube-system/coredns.yaml

@@ -0,0 +1,45 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: coredns
+  namespace: kube-system
+spec:
+  match:
+    - select:
+        labels:
+          k8s-app: "coredns"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        reserve_data: true
+        remove_key_name_field: true
+        parse:
+          type: "multi_format"
+          patterns:
+            - format: "regexp"
+              expression: '^\[(?<log.level>.*)\] \[?(?<source.address>.*)\]?:(?<source.port>.*) - (?<dns.id>.*) "(?<dns.question.type>.*) (?<dns.question.class>.*) (?<dns.question.name>.*)\.? (?<network.transport>.*) (?<coredns.query.size>.*) (?<coredns.dnssec_ok>.*) (?<bufsize>.*)" (?<dns.response_code>.*) (?<dns.header_flags>.*) (?<coredns.response.size>.*) (?<coredns.duration>.*)s'
+              types: "source.port:integer,dns.id:integer,coredns.query.size:integer,coredns.dnssec_ok:bool,bufsize:integer,dns.header_flags:array,coredns.response.size:integer,coredns.duration:float"
+            - format: "none"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - source.ip: '${ record["source.address"] }'
+            dns.header_flags: '${ !(record["dns.header_flags"].nil?) ? record["dns.header_flags"].map(&:upcase) : nil }'
+            event.duration: '${ !(record["coredns.duration"].nil?) ? record["coredns.duration"] * 1000000000 : nil }'
+            event.kind: "event"
+            event.category: "network"
+            event.type: "protocol"
+            event.outcome: '${ record["dns.response_code"] == "NOERROR" ? "success" : "failure" }'
+            event.protocol: "dns"
+            event.module: "coredns"
+            related.ip: '${ record["source.address"] }'
+            # for dashboard
+            fileset.name: "kubernetes"
+            coredns.query.name: '${ record["dns.question.name"] }'
+        remove_keys: "coredns.duration,coredns.dnssec_ok"
+  globalOutputRefs:
+    - "default"
+{{- end }}
+

infra-logging/templates/flow/kube-system/klog.yaml

@@ -0,0 +1,52 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: klog
+  namespace: kube-system
+spec:
+  match:
+    - select:
+        labels:
+          k8s-app: "konnectivity-agent"
+    - select:
+        labels:
+          k8s-app: "kube-proxy"
+    - select:
+        labels:
+          app: "snapshot-validation-webhook"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        hash_value_field: "klog"
+        reserve_data: true
+        remove_key_name_field: true
+        parse:
+          type: "multi_format"
+          patterns:
+            - format: "regexp"
+              expression: '(?<log_level>[A-Z])(?<month>\d{2})(?<day>\d{2})\s+(?<time>\d{2}:\d{2}:\d{2}(|\.\d+))\s+(?<threadid>\d+)\s+(?<file>[^ ]*):(?<line>\d+)\]\s("(?<msg>([^"\\]*(?:\\.[^"\\]*)*))"(|\s+(?<kv>.*))|(?<greedy_msg>.*))$'
+              types: "month:integer,day:integer,threadid:integer"
+            - format: "none"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - timestamp: '${time.strftime("%Y")}-${ record["klog"]["month"] }-${ record["klog"]["day"] }T${ record["klog"]["time"] }Z'
+            message: '${ !(record["klog"]["greedy_msg"].nil?) ? record["klog"]["greedy_msg"] : record["klog"]["msg"] }'
+            log.level: '${ record["klog"]["log_level"].gsub("I", "info").gsub("W", "warn").gsub("E", "error").gsub("F", "fatal") }'
+            klog_kv: '${ !(record["klog"]["kv"].nil?) ? record["klog"]["kv"] : "" }'
+        remove_keys: "$['klog']['month'],$['klog']['day'],$['klog']['time'],$['klog']['log_level'],$['klog']['msg'],$['klog']['greedy_msg'],$['klog']['kv']"
+    - parser:
+        key_name: "klog_kv"
+        hash_value_field: "klog.fields"
+        reserve_data: true
+        remove_key_name_field: true
+        parse:
+          type: "multi_format"
+          patterns:
+            - format: "logfmt"
+            - format: "none"
+  globalOutputRefs:
+    - "default"
+{{- end }}

infra-logging/templates/logging-operator/flow/event-tailer.yaml

@@ -0,0 +1,29 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: event-tailer
+spec:
+  match:
+    - select:
+        labels:
+          "app.kubernetes.io/name": "event-tailer"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        hash_value_field: "kubernetes"
+        remove_key_name_field: true
+        reserve_data: true
+        parse:
+          type: "json"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - event.module: "kubernetes"
+            message: '${ record["kubernetes"]["event"]["message"] }'
+        remove_keys: "$['kubernetes']['event']['message']"
+  globalOutputRefs:
+    - "default"
+{{- end }}
+

infra-logging/templates/logging-operator/flow/fluentbit.yaml

@@ -0,0 +1,36 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: fluentbit
+spec:
+  match:
+    - select:
+        labels:
+          "app.kubernetes.io/name": "fluentbit"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        hash_value_field: "fluentbit"
+        reserve_data: true
+        remove_key_name_field: true
+        parse:
+          type: "regexp"
+          expression: '^\[(?<timestamp>.*)\] \[(?<log.level>.*)\] \[(?<component>.*)\] (?<message>.*)'
+          types: "timestamp:string,log.level:string,component:string,message:string"
+          time_key: "timestamp"
+          time_type: "string"
+          time_format: "%Y/%m/%d %H:%M:%S"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - event.kind: "event"
+            event.module: "fluentbit"
+            message: '${record["fluentbit"]["message"]}'
+            log.level: '${record["fluentbit"]["log.level"]}'
+        remove_keys: "$['fluentbit']['log']['level'],$['fluentbit']['message']"
+  globalOutputRefs:
+    - default
+{{- end }}
+

infra-logging/templates/logging-operator/flow/logging-operator.yaml

@@ -0,0 +1,34 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: logging-operator
+spec:
+  match:
+    - select:
+        labels:
+          "app.kubernetes.io/name": "logging-operator"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        hash_value_field: "logging-operator"
+        reserve_data: true
+        remove_key_name_field: true
+        parse:
+          type: "json"
+          time_key: "ts"
+          time_type: "string"
+          time_format: "%iso8601"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - event.kind: "event"
+            event.module: "logging-operator"
+            message: '${record["logging-operator"]["msg"]}'
+            log.level: '${record["logging-operator"]["level"]}'
+        remove_keys: "$['logging-operator']['level'],$['logging-operator']['msg']"
+  globalOutputRefs:
+    - default
+{{- end }}
+

infra-logging/templates/logging-operator/release.yaml

@@ -0,0 +1,201 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: logging-operator
+spec:
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: kube-logging
+      chart: logging-operator
+      interval: 10m
+  install:
+    {{- toYaml .Values.commons.helm.release.install | nindent 4 }}
+  test:
+    {{- toYaml .Values.commons.helm.release.test | nindent 4 }}
+  upgrade:
+    {{- toYaml .Values.commons.helm.release.upgrade | nindent 4 }}
+  interval: 10m
+  values:
+    monitoring:
+      serviceMonitor:
+        enabled: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+        additionalLabels:
+          {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 10 }}
+
+    # resources for logging-operator
+    resources:
+      limits:
+        memory: 3Gi
+      requests:
+        cpu: 100m
+        memory: 128Mi
+
+    logging:
+      enabled: true
+      # fluentbit is used to collect data on nodes (so it is usefull to use hostPath)
+      fluentbit:
+        bufferStorageVolume:
+          hostPath:
+            path: "/var/lib/kube-logging/fluentbit/buffer"
+        positiondb:
+          hostPath:
+            path: "/var/lib/kube-logging/fluentbit/positiondb"
+        metrics:
+          prometheusRules: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+          serviceMonitor: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+          serviceMonitorConfig:
+            additionalLabels:
+              {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 14 }}
+
+
+      # fluentd is used to recieve data from fluentbit, filter (e.g. parse, grep) and forward output (e.g. loki)
+      fluentd:
+        scaling:
+          replicas: {{ .Values.fluentd.replicas }}
+        # resources for fluentd
+        resources:
+          limits:
+            memory: "2400M"
+          requests:
+            cpu: "500m"
+            memory: "200M"
+        metrics:
+          prometheusRules: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+          serviceMonitor: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+          serviceMonitorConfig:
+            additionalLabels:
+              {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 14 }}
+        bufferVolumeMetrics:
+          prometheusRules: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+          serviceMonitor: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+          serviceMonitorConfig:
+            additionalLabels:
+              {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 14 }}
+
+      tls:
+        # make problems on reinstall (maybe try it sometime again)
+        enabled: false
+
+      # allow clusteroutput from flow in other namespace
+      allowClusterResourcesFromAllNamespaces: true
+
+      enableRecreateWorkloadOnImmutableFieldChange: true
+
+      # log kubernetes events
+      eventTailer:
+        name: "default"
+
+      # forward errors to output
+      errorOutputRef: "default"
+
+      # if no (cluster)flow exits for pods:
+      #  filter: drop log messages if they contains "debug"
+      #  send logs: clusterOutput "default"
+      defaultFlow:
+        filters:
+          - grep:
+              exclude:
+                - key: "message"
+                  pattern: /.*[Dd]ebug.*/
+          - prometheus:
+              metrics:
+                - name: "logs_defaultflow_count"
+                  desc: "The total number of message in namespace"
+                  type: "counter"
+              labels:
+                exported_namespace: "$.kubernetes.namespace_name"
+                exported_pod: "$.kubernetes.pod_name"
+                exported_container: "$.kubernetes.container_name"
+                image: "$.kubernetes.container_image"
+                app_kubernetes_io_name: "$['kubernetes']['labels']['app.kubernetes.io/name']"
+                app_kubernetes_io_instance: "$['kubernetes']['labels']['app.kubernetes.io/instance']"
+        globalOutputRefs:
+          - "default"
+
+      # usefull on elastic e.g. with dedot
+      globalFilters:
+        - prometheus:
+            metrics:
+              - name: "logs_all_count"
+                desc: "The total number of messages in namespace"
+                type: "counter"
+            labels:
+              exported_namespace: "$.kubernetes.namespace_name"
+              exported_pod: "$.kubernetes.pod_name"
+              exported_container: "$.kubernetes.container_name"
+              image: "$.kubernetes.container_image"
+              app_kubernetes_io_name: "$['kubernetes']['labels']['app.kubernetes.io/name']"
+              app_kubernetes_io_instance: "$['kubernetes']['labels']['app.kubernetes.io/instance']"
+
+      # deploy a clusteroutput (which all flows can use)
+      clusterOutputs:
+        - name: "default"
+          spec:
+            {{- if .Values.loki.enabled }}
+            # for loki:
+            # https://kube-logging.dev/docs/configuration/plugins/outputs/loki/
+            loki:
+              url: http://loki:3100
+              buffer:
+                timekey: 1m
+                timekey_wait: 30s
+                timekey_use_utc: true
+              # do not use configure_kubernetes_labels strip other kubernetes labels
+              extract_kubernetes_labels: true
+              labels:
+                # from configure_kubernetes_labels reimplement
+                host: $.kubernetes.host
+                namespace: $.kubernetes.namespace_name
+                pod: $.kubernetes.pod_name
+                pod_id: $.kubernetes.pod_id
+                container: $.kubernetes.container_name
+                container_id: $.kubernetes.docker_id
+            {{- else }}
+            nullout: {}
+            {{- end }}
+
+      # add some usefull default clusterFlows
+      clusterFlows:
+        # parse all data with logfmt of pod which contain label: kube_logging_parser=logfmt (and send to ClusterOutput default)
+        - name: logfmt
+          spec:
+            filters:
+              - parser:
+                  reserve_data: true
+                  remove_key_name_field: true
+                  hash_value_field: "logfmt"
+                  parse:
+                    type: "multi_format"
+                    patterns:
+                      - format: "logfmt"
+                      # fallback, just keep data unparsed
+                      - format: "none"
+            match:
+              - select:
+                  labels:
+                    "kube_logging_parser": "logfmt"
+            globalOutputRefs:
+              - "default"
+        # parse all data with json of pod which contain label: kube_logging_parser=json (and send to ClusterOutput default)
+        - name: json
+          spec:
+            filters:
+              - parser:
+                  reserve_data: true
+                  remove_key_name_field: true
+                  hash_value_field: "json"
+                  parse:
+                    type: "multi_format"
+                    patterns:
+                      - format: "json"
+                      # fallback, just keep data unparsed
+                      - format: "none"
+            match:
+              - select:
+                  labels:
+                    "kube_logging_parser": "json"
+            globalOutputRefs:
+              - "default"

infra-logging/templates/logging-operator/repo.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: kube-logging
+spec:
+  url: oci://ghcr.io/kube-logging/helm-charts
+  type: oci
+  interval: 10m0s

infra-logging/templates/loki/configmap_grafana_datasource.yaml

@@ -0,0 +1,17 @@
+{{- if .Values.loki.enabled }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-datasource-loki
+  labels:
+    {{- toYaml .Values.commons.grafana.datasource.labels | nindent 4 }}
+data:
+  datasource.yaml: |-
+    apiVersion: 1
+    datasources:
+      - name: "Loki"
+        type: "loki"
+        access: "proxy"
+        url: "http://loki.{{ .Release.Namespace }}.svc:3100"
+{{- end }}

infra-logging/templates/loki/release.yaml

@@ -0,0 +1,57 @@
+{{- if .Values.loki.enabled }}
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: loki
+spec:
+  chart:
+    spec:
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+      chart: loki
+      interval: 10m
+  install:
+    {{- toYaml .Values.commons.helm.release.install | nindent 4 }}
+  test:
+    {{- toYaml .Values.commons.helm.release.test | nindent 4 }}
+  upgrade:
+    {{- toYaml .Values.commons.helm.release.upgrade | nindent 4 }}
+  interval: 10m
+  values:
+    singleBinary:
+      replicas: 1
+    loki:
+      commonConfig:
+        replication_factor: 1
+      storage:
+        type: "filesystem"
+      auth_enabled: false
+    monitoring:
+      dashboards:
+        labels:
+          {{- toYaml .Values.commons.grafana.dashboards.labels | nindent 10 }}
+        annotations:
+          {{- toYaml .Values.grafana.dashboards.annotations | nindent 10 }}
+      rules:
+        enabled: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+        labels:
+          {{- toYaml .Values.commons.prometheus.rules.labels | nindent 10 }}
+      serviceMonitor:
+        enabled: {{ (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+        labels:
+          {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 10 }}
+      metricsInstance:
+        enabled: false
+      selfMonitoring:
+        enabled: false
+        grafanaAgent:
+          installOperator: false
+      lokiCanary:
+        enabled: false
+    test:
+      enabled: false
+    gateway:
+      enabled: false
+{{- end }}

infra-logging/templates/loki/repo.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.loki.enabled }}
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: grafana
+spec:
+  url: https://grafana.github.io/helm-charts
+  interval: 10m0s
+{{- end }}

infra-logging/values.yaml

@@ -0,0 +1,34 @@
+init:
+  version: 0
+  namespace: "bases"
+
+commons:
+  helm:
+    release:
+      install: {}
+      test: {}
+      upgrade: {}
+
+  grafana:
+    datasource:
+      labels:
+        grafana_datasource: "1"
+    dashboards:
+      labels:
+        grafana_dashboard: "1"
+
+  prometheus:
+    monitor:
+      labels: {}
+    rules:
+      labels: {}
+
+grafana:
+  dashboards:
+    annotations: {}
+
+fluentd:
+  replicas: 1
+
+loki:
+  enabled: true