From 092c943aef10ca07c39c972d0faff7a5f9a16cda Mon Sep 17 00:00:00 2001 From: WrenIX Date: Thu, 17 Oct 2024 20:46:47 +0200 Subject: [PATCH] fix(infra-monitoring): support multiple receivers --- infra-monitoring/README.adoc | 23 ++++++-- infra-monitoring/ci/ct-reciever-values.yaml | 38 +++++++++++++ .../alertmanager/alertmanager-config.yaml | 56 +++++++++---------- .../alertmanager/matrix/release.yaml | 4 +- infra-monitoring/values.yaml | 12 ++-- 5 files changed, 96 insertions(+), 37 deletions(-) create mode 100644 infra-monitoring/ci/ct-reciever-values.yaml diff --git a/infra-monitoring/README.adoc b/infra-monitoring/README.adoc index 978d90f..f9a9ec6 100644 --- a/infra-monitoring/README.adoc +++ b/infra-monitoring/README.adoc @@ -110,6 +110,16 @@ image::https://img.shields.io/badge/Version-application-informational?style=flat | `""` | +| alertmanager.receiver.customs +| object +| `{}` +| customs + +| alertmanager.receiver.inhibitRules +| list +| `[]` +| additional inhibitRules + | alertmanager.receiver.matrix.default | string | `"!example-room:matrix.org"` @@ -135,6 +145,11 @@ image::https://img.shields.io/badge/Version-application-informational?style=flat | `false` | +| alertmanager.receiver.matrix.token +| string +| `""` +| token of matrix reciever (use valuesFrom in flux for it) + | alertmanager.receiver.matrix.userID | string | `"@alert:matrix.org"` @@ -162,7 +177,7 @@ image::https://img.shields.io/badge/Version-application-informational?style=flat | alertmanager.receiver.ntfy.enabled | bool -| `true` +| `false` | | alertmanager.receiver.ntfy.ingress.annotations @@ -191,9 +206,9 @@ image::https://img.shields.io/badge/Version-application-informational?style=flat | repeat Interval | alertmanager.receiver.routes -| string -| `nil` -| would overwrite defaults (like ntfy or matrix) +| list +| `[]` +| would overwrite defaults (like ntfy or matrix) # send every alert to ntfy # (and continue to routing for that alert) - receiver: "ntfy-default" continue: true # send selected alerts to special matrix room # (and remove them for evaluation - no continue) - receiver: "matrix-room-name" matchers: - name: "team" matchType: "=" value: "room-name" # all reminig alerts to matrix default room - receiver: "matrix-default" | commons.auth.authentik.backend | string diff --git a/infra-monitoring/ci/ct-reciever-values.yaml b/infra-monitoring/ci/ct-reciever-values.yaml new file mode 100644 index 0000000..a47c0c3 --- /dev/null +++ b/infra-monitoring/ci/ct-reciever-values.yaml @@ -0,0 +1,38 @@ +alertmanager: + receiver: + customs: + own-receiver: + webhookConfigs: + - url: "own" + multiple-receiver: + webhookConfigs: + - url: "not exists" + other: "support" + ntfy: + enabled: true + matrix: + enabled: true + rooms: + multiple-receiver: + - room: "!multi-test:matrix.org" + sendResolved: false + only-matrix: + - room: "!only-test:matrix.org" + sendResolved: false + routes: + - receiver: "multiple-receiver" + continue: true + matchers: + - name: "team" + matchType: "=" + value: "multi" + - receiver: "own-receiver" + matchers: + - name: "namespace" + matchType: "=" + value: "own" + - receiver: "only-matrix" + matchers: + - name: "namespace" + matchType: "=~" + value: "matrix-.*" diff --git a/infra-monitoring/templates/alertmanager/alertmanager-config.yaml b/infra-monitoring/templates/alertmanager/alertmanager-config.yaml index 89f7a75..383d0fd 100644 --- a/infra-monitoring/templates/alertmanager/alertmanager-config.yaml +++ b/infra-monitoring/templates/alertmanager/alertmanager-config.yaml @@ -7,10 +7,28 @@ metadata: spec: receivers: - name: "null" - {{- with .Values.alertmanager.receiver.ntfy }} - {{- if .enabled }} - - name: "ntfy-default" + {{- range $i, $receiver := ( concat (keys .Values.alertmanager.receiver.customs) (keys .Values.alertmanager.receiver.matrix.rooms) | uniq) }} + - name: "{{ $receiver }}" webhookConfigs: + {{- with (get $.Values.alertmanager.receiver.matrix.rooms $receiver) }} + {{- range $item := . }} + - url: "http://alertmanager-matrix:4051/{{ $item.room }}" + sendResolved: {{ $item.sendResolved }} + {{- end }} + {{- end }} + {{- with (get $.Values.alertmanager.receiver.customs $receiver) }} + {{- with .webhookConfigs }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with (omit . "webhookConfigs") }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} + {{- end }} + - name: "default" + webhookConfigs: + {{- with .Values.alertmanager.receiver.ntfy }} + {{- if .enabled }} - url: "http://alertmanager-ntfy" sendResolved: {{ .sendResolved }} {{- if and .enabled (.config.user) }} @@ -23,22 +41,14 @@ spec: name: global-alertmanager-ntfy-auth key: password {{- end }} - {{- end }} - {{- end }} - {{- with .Values.alertmanager.receiver.matrix }} - {{- if .enabled }} - - name: "matrix-default" - webhookConfigs: + {{- end }} + {{- end }} + {{- with .Values.alertmanager.receiver.matrix }} + {{- if .enabled }} - url: "http://alertmanager-matrix:4051/{{ .default }}" sendResolved: {{ .sendResolved }} - {{- range $receiver, $conf := .rooms }} - - name: "matrix-{{ $receiver }}" - webhookConfigs: - - url: "http://alertmanager-matrix:4051/{{ $conf.room }}" - sendResolved: {{ $conf.sendResolved }} - {{- end }} - {{- end }} - {{- end }} + {{- end }} + {{- end }} inhibitRules: - sourceMatch: - name: "alertmanagerInhibitDisable" @@ -108,16 +118,6 @@ spec: value: "InfoInhibitor|Watchdog" {{- with .Values.alertmanager.receiver.routes }} {{- toYaml . | nindent 6 }} - {{- else }} - {{- with .Values.alertmanager.receiver }} - {{- if .ntfy.enabled }} - - receiver: "ntfy-default" - continue: true - {{- end }} - {{- if .matrix.enabled }} - - receiver: "matrix-default" - continue: true - {{- end }} - {{- end }}{{/* end-with receiver */}} {{- end }}{{/* end-with routes */}} + - receiver: "default" {{- end }} diff --git a/infra-monitoring/templates/alertmanager/matrix/release.yaml b/infra-monitoring/templates/alertmanager/matrix/release.yaml index 471417b..e9a549b 100644 --- a/infra-monitoring/templates/alertmanager/matrix/release.yaml +++ b/infra-monitoring/templates/alertmanager/matrix/release.yaml @@ -38,9 +38,11 @@ spec: userID: {{ .userID | quote }} rooms: - {{ .default | quote }} - {{- range $item := .rooms }} + {{- range $name, $conf := .rooms }} + {{- range $item := $conf }} - {{ $item.room | quote }} {{- end }} + {{- end }} alertmanager: "http://kube-prometheus-stack-alertmanager:9093" showLabels: true {{` diff --git a/infra-monitoring/values.yaml b/infra-monitoring/values.yaml index aa6ec9a..6b1d6bd 100644 --- a/infra-monitoring/values.yaml +++ b/infra-monitoring/values.yaml @@ -79,25 +79,29 @@ alertmanager: # value: "room-name" # # all reminig alerts to matrix default room # - receiver: "matrix-default" - routes: + routes: [] # -- additional inhibitRules inhibitRules: [] # -- repeat Interval repeatInterval: "24h" + # -- customs + customs: {} matrix: enabled: false sendResolved: false homeserver: "https://matrix.org" userID: "@alert:matrix.org" + # -- token of matrix reciever (use valuesFrom in flux for it) + token: "" default: "!example-room:matrix.org" # rooms: {} # room-name: - # sendResolved: false - # room: ""!example-room-name:matrix.org"" + # - room: "!example-room-name:matrix.org" + # sendResolved: false # ntfy: - enabled: true + enabled: false sendResolved: false ingress: enabled: false