feat(infra-fluxcd): init

infra-fluxcd/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

infra-fluxcd/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: infra-fluxcd
+description: Install resources for FluxCD (e.g. grafana dashboards, prometheus podmonitor)
+type: application
+
+version: 0.1.0

infra-fluxcd/grafana_dashboards/logs.json

@@ -0,0 +1,332 @@
+{
+  "__inputs": [
+    {
+      "name": "DS_LOKI",
+      "label": "Loki",
+      "description": "",
+      "type": "datasource",
+      "pluginId": "loki",
+      "pluginName": "Loki"
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      },
+      {
+        "datasource": {
+          "type": "datasource",
+          "uid": "grafana"
+        },
+        "enable": true,
+        "iconColor": "red",
+        "name": "flux events",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [
+            "flux"
+          ],
+          "type": "tags"
+        }
+      }
+    ]
+  },
+  "description": "Flux logs collected from Kubernetes, stored in Loki",
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": 29,
+  "iteration": 1653748775696,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "datasource": "${DS_LOKI}",
+      "description": "",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 4,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "hidden",
+          "placement": "bottom"
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": "${DS_LOKI}",
+          "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))",
+          "instant": false,
+          "legendFormat": "Log count",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "type": "timeseries"
+    },
+    {
+      "datasource": "${DS_LOKI}",
+      "description": "Logs from services running in Kubernetes",
+      "gridPos": {
+        "h": 25,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 2,
+      "options": {
+        "dedupStrategy": "numbers",
+        "enableLogDetails": false,
+        "prettifyLogMessage": true,
+        "showCommonLabels": false,
+        "showLabels": false,
+        "showTime": false,
+        "sortOrder": "Descending",
+        "wrapLogMessage": false
+      },
+      "targets": [
+        {
+          "datasource": "${DS_LOKI}",
+          "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"",
+          "refId": "A"
+        }
+      ],
+      "type": "logs"
+    }
+  ],
+  "refresh": "10s",
+  "schemaVersion": 36,
+  "style": "light",
+  "tags": [
+    "flux"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "",
+          "value": ""
+        },
+        "description": "String to search for",
+        "hide": 0,
+        "label": "Search Query",
+        "name": "query",
+        "options": [
+          {
+            "selected": true,
+            "text": "",
+            "value": ""
+          }
+        ],
+        "query": "",
+        "skipUrlSync": false,
+        "type": "textbox"
+      },
+      {
+        "allValue": "info|error",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "hide": 0,
+        "includeAll": true,
+        "multi": false,
+        "name": "level",
+        "options": [
+          {
+            "selected": true,
+            "text": "All",
+            "value": "$__all"
+          },
+          {
+            "selected": false,
+            "text": "info",
+            "value": "info"
+          },
+          {
+            "selected": false,
+            "text": "error",
+            "value": "error"
+          }
+        ],
+        "query": "info,error",
+        "queryValue": "",
+        "skipUrlSync": false,
+        "type": "custom"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(app)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "controller",
+        "options": [],
+        "query": "label_values(app)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": true,
+          "text": [
+            "flux-system"
+          ],
+          "value": [
+            "flux-system"
+          ]
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(namespace)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "namespace",
+        "options": [],
+        "query": "label_values(namespace)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "allValue": ".+",
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "datasource": "${DS_LOKI}",
+        "definition": "label_values(stream)",
+        "hide": 0,
+        "includeAll": true,
+        "multi": true,
+        "name": "stream",
+        "options": [],
+        "query": "label_values(stream)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "Loki",
+          "value": "Loki"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "DS_LOKI",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-6h",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "Flux Logs",
+  "uid": "flux-logs",
+  "version": 2
+}

infra-fluxcd/templates/configmap_grafana_dashboards.yaml

@@ -0,0 +1,13 @@
+{{- range $path, $bytes := .Files.Glob "grafana_dashboards/*.json" }}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: infra-fluxcd-grafana-db-{{ base $path }}
+  labels:
+    {{- toYaml $.Values.commons.grafana.dashboards.labels | nindent 4 }}
+  annotations:
+    {{- toYaml $.Values.grafana.dashboards.annotations | nindent 4 }}
+data:
+  {{- ($.Files.Glob $path ).AsConfig | nindent 2 }}
+{{- end }}

infra-fluxcd/templates/configmap_init_crd.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-init
+  namespace: "{{ .Values.init.namespace }}"
+data:
+  {{- if and
+    (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PodMonitor")
+    (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule")
+    (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow")
+  }}
+  init: "-1"
+  {{- else }}
+  init: "{{ add1 .Values.init.version }}"
+  {{- end }}
+

infra-fluxcd/templates/flow.yaml

@@ -0,0 +1,50 @@
+{{- if (.Capabilities.APIVersions.Has "logging.banzaicloud.io/v1beta1/Flow") }}
+---
+apiVersion: logging.banzaicloud.io/v1beta1
+kind: Flow
+metadata:
+  name: flux-system
+  namespace: flux-system
+spec:
+  match:
+    - select:
+        labels:
+          app: "helm-controller"
+    - select:
+        labels:
+          app: "kustomize-controller"
+    - select:
+        labels:
+          app: "notification-controller"
+    - select:
+        labels:
+          app: "source-controller"
+  filters:
+    - tag_normaliser: {}
+    - parser:
+        reserve_data: true
+        remove_key_name_field: true
+        hash_value_field: "klog"
+        parse:
+          type: "json"
+          time_key: "ts"
+          time_type: "string"
+          time_format: "%iso8601"
+    - record_transformer:
+        enable_ruby: true
+        records:
+          - event.module: "fluxcd"
+            message: '${ record["flux"]["msg"] ? record["flux"]["msg"] : record["flux"]["message"]}'
+            log.level: '${ record["flux"]["level"] ? record["flux"]["level"] : "unknown" }'
+        remove_keys: "$['flux']['msg'],$['flux']['message'],$['flux']['level']"
+    - prometheus:
+        labels:
+          controller: $.kubernetes.labels.app
+          level: $['log']['level']
+        metrics:
+        - desc: The total number of messages from apps
+          name: logs_fluxcd
+          type: counter
+  globalOutputRefs:
+    - "default"
+{{- end }}

infra-fluxcd/templates/pod-monitor.yaml

@@ -0,0 +1,31 @@
+{{- if (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PodMonitor") }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: flux-system
+  labels:
+    {{- toYaml .Values.commons.prometheus.monitor.labels | nindent 4 }}
+spec:
+  namespaceSelector:
+    matchNames:
+      - flux-system
+  selector:
+    matchExpressions:
+      - key: app
+        operator: In
+        values:
+          - helm-controller
+          - source-controller
+          - kustomize-controller
+          - notification-controller
+          - image-automation-controller
+          - image-reflector-controller
+  podMetricsEndpoints:
+    - port: http-prom
+      relabelings:
+        # https://github.com/prometheus-operator/prometheus-operator/issues/4816
+        - sourceLabels: [__meta_kubernetes_pod_phase]
+          action: keep
+          regex: Running
+{{- end }}

infra-fluxcd/templates/prometheus-rule.yaml

@@ -0,0 +1,34 @@
+{{- if (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/PrometheusRule") }}
+---
+apiVersion: "monitoring.coreos.com/v1"
+kind: "PrometheusRule"
+metadata:
+  name: flux-system
+  labels:
+    {{- toYaml .Values.commons.prometheus.rules.labels | nindent 4 }}
+spec:
+  groups:
+    - name: "FluxCD"
+      rules:
+        - alert: "FluxCD: source fetch failed"
+          expr: 'gotk_reconcile_condition{status="False",kind=~"GitRepository|HelmRepository|Bucket"} > 0'
+          for: "5m"
+          labels:
+            severity: "warning"
+          {{`
+          annotations:
+            summary: "FluxCD has not fetched a source in {{ $labels.exported_namespace }} correct"
+            description: "FluxCD has not fetched the source {{ $labels.name }} of {{ $labels.kind }} in {{ $labels.exported_namespace }}"
+          `}}
+
+        - alert: "FluxCD: install failed"
+          expr: 'gotk_reconcile_condition{status="False",kind=~"Kustomization|HelmRelease"} > 0'
+          for: "5m"
+          labels:
+            severity: "warning"
+          {{`
+          annotations:
+            summary: "FluxCD has not installed something in {{ $labels.exported_namespace }} correct"
+            description: "FluxCD has not installed {{ $labels.name }} of {{ $labels.kind }} in {{ $labels.exported_namespace }} correct"
+          `}}
+{{- end }}

infra-fluxcd/values.yaml

@@ -0,0 +1,18 @@
+init:
+  version: 0
+  namespace: "bases"
+
+commons:
+  grafana:
+    dashboards:
+      labels:
+        grafana_dashboard: "1"
+  prometheus:
+    monitor:
+      labels: {}
+    rules:
+      labels: {}
+
+grafana:
+  dashboards:
+    annotations: {}