admin
44d1904286
gates / gates (push) Successful in 12m24s
R-232 dated note + a dated check for tonight's sweep; R-934 closed (the disk was pulled 2026-10-23); R-1250 fixed on main, live proof on demo-hp with a real seed; 08 §9 item 5. Evidence audits/dooplex-alarms-2026-10-24/.
294 lines
10 KiB
Plaintext
294 lines
10 KiB
Plaintext
---
|
|
# ============================================================================
|
|
# alertmanager — alert routing and email delivery
|
|
#
|
|
# Brought into this repo on 2026-08-14. Until then it ran live with NO manifest
|
|
# anywhere: CLAUDE.md listed it as one of three such workloads, which meant any
|
|
# change to its routing survived only until something recreated it, and nobody
|
|
# could tell what the running config was without reading the cluster.
|
|
#
|
|
# Credentials are NOT here and must never be. smtp_from / auth username /
|
|
# password / recipient are ${VAR} placeholders resolved at pod start by the
|
|
# envsubst init container from the alertmanager-smtp Secret, which is created
|
|
# imperatively. Only the smarthost (smtp.gmail.com:587) and require_tls are
|
|
# literal, and neither is sensitive.
|
|
#
|
|
# Two deliberate fixes applied while adopting, both flagged rather than silent:
|
|
# * envsubst was running :latest. Pinned to the digest that was actually
|
|
# running, so a restart can no longer silently change the code.
|
|
# * the init container had no resource requests/limits, against house style.
|
|
# ============================================================================
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: alertmanager-config-template
|
|
namespace: mon-system
|
|
labels:
|
|
app: alertmanager
|
|
data:
|
|
alertmanager.yml.tmpl: |
|
|
global:
|
|
smtp_smarthost: 'smtp.gmail.com:587'
|
|
smtp_from: '${SMTP_AUTH_USERNAME}'
|
|
smtp_auth_username: '${SMTP_AUTH_USERNAME}'
|
|
smtp_auth_password: '${SMTP_AUTH_PASSWORD}'
|
|
smtp_require_tls: true
|
|
resolve_timeout: 5m
|
|
|
|
route:
|
|
receiver: 'email-notifications'
|
|
group_by: ['alertname', 'severity']
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: 4h
|
|
|
|
routes:
|
|
# --- disk health -------------------------------------------------------
|
|
# A failing drive does not improve between emails. The generic
|
|
# severity: critical route below repeats hourly, which produced ~24 mails
|
|
# a day for a single dying disk on 2026-08-13/14. Once you have seen it,
|
|
# a daily reminder is enough.
|
|
#
|
|
# The two *Growing alerts are deliberately kept frequent: they only fire
|
|
# while the SMART counters are actively climbing, i.e. something is
|
|
# degrading RIGHT NOW rather than sitting in a known-broken state.
|
|
- match:
|
|
alertname: DiskReportedUncorrectGrowing
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 4h
|
|
- match:
|
|
alertname: DiskReallocatedSectorsGrowing
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 4h
|
|
- match:
|
|
component: disk
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 24h
|
|
# --- workload health ---------------------------------------------------
|
|
# Same reasoning as the disk route: a crashlooping pod does not fix
|
|
# itself, so the generic critical route's hourly repeat would have sent
|
|
# 27 mails for the single external-dns-pihole failure that motivated
|
|
# these alerts. 6h still reaches you four times a day, which is far more
|
|
# than the zero notifications that outage actually produced.
|
|
#
|
|
# Must stay ABOVE the severity: critical route -- Alertmanager takes the
|
|
# first matching route, so ordering is what makes this override apply.
|
|
- match:
|
|
component: workload
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 6h
|
|
- match:
|
|
severity: critical
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 1h
|
|
- match:
|
|
severity: warning
|
|
receiver: 'email-notifications'
|
|
repeat_interval: 4h
|
|
|
|
receivers:
|
|
- name: 'email-notifications'
|
|
email_configs:
|
|
- to: '${SMTP_TO}'
|
|
send_resolved: true
|
|
headers:
|
|
Subject: '[{{ .Status | toUpper }}] {{ .GroupLabels.alertname }}'
|
|
html: |
|
|
{{ range .Alerts }}
|
|
<h2>{{ .Status | toUpper }}: {{ .Labels.alertname }}</h2>
|
|
<p><strong>Severity:</strong> {{ .Labels.severity }}</p>
|
|
<p><strong>Summary:</strong> {{ .Annotations.summary }}</p>
|
|
<p><strong>Description:</strong> {{ .Annotations.description }}</p>
|
|
<p><strong>Started:</strong> {{ .StartsAt.Format "2006-01-02 15:04:05" }}</p>
|
|
{{ if .EndsAt }}
|
|
<p><strong>Ended:</strong> {{ .EndsAt.Format "2006-01-02 15:04:05" }}</p>
|
|
{{ end }}
|
|
<hr>
|
|
{{ end }}
|
|
|
|
inhibit_rules:
|
|
- source_match:
|
|
severity: 'critical'
|
|
target_match:
|
|
severity: 'warning'
|
|
equal: ['alertname', 'volume']
|
|
---
|
|
apiVersion: v1
|
|
kind: PersistentVolumeClaim
|
|
metadata:
|
|
name: alertmanager-data
|
|
namespace: mon-system
|
|
labels:
|
|
app: alertmanager
|
|
# Explicit membership rather than relying on implicit default-group
|
|
# inclusion, which any stray recurring-job label would silently break.
|
|
recurring-job-group.longhorn.io/default: enabled
|
|
spec:
|
|
accessModes:
|
|
- ReadWriteOnce
|
|
# Explicit, not the cluster default: longhorn-ssd2 is the default but is the
|
|
# smallest disk on the box.
|
|
storageClassName: longhorn
|
|
resources:
|
|
requests:
|
|
storage: 1Gi
|
|
---
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: alertmanager
|
|
namespace: mon-system
|
|
labels:
|
|
app: alertmanager
|
|
spec:
|
|
replicas: 1
|
|
# Recreate, not RollingUpdate: the data volume is ReadWriteOnce, so a rolling
|
|
# update would deadlock with the new pod waiting for a volume the old pod
|
|
# still holds.
|
|
strategy:
|
|
type: Recreate
|
|
selector:
|
|
matchLabels:
|
|
app: alertmanager
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: alertmanager
|
|
spec:
|
|
# R-886 (operator yes 2026-10-12, felhom 09 §3 decision 217). The image runs as nobody (65534)
|
|
# and the Longhorn volume's root is root:root 0755, so since the 2026-10-05 Longhorn restart
|
|
# every nflog/silences write failed with "permission denied" and silences died at each restart.
|
|
# fsGroup makes the kubelet give the volume to group 65534 (group-writable) at mount.
|
|
securityContext:
|
|
fsGroup: 65534
|
|
initContainers:
|
|
# Renders the template into the real config, substituting the SMTP
|
|
# credentials from the Secret. The processed file lands in an emptyDir,
|
|
# which is why a ConfigMap edit needs a POD RESTART to take effect --
|
|
# POST /-/reload would just re-read the already-rendered file.
|
|
- name: envsubst
|
|
image: bhgedigital/envsubst@sha256:36a92e9344ef965e9318c8d2f70dd35cad654a4615b889c6a4dd483ca1ddec7c
|
|
command:
|
|
- sh
|
|
- -c
|
|
- envsubst < /template/alertmanager.yml.tmpl > /config/alertmanager.yml
|
|
env:
|
|
- name: SMTP_AUTH_USERNAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: alertmanager-smtp
|
|
key: SMTP_AUTH_USERNAME
|
|
- name: SMTP_AUTH_PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: alertmanager-smtp
|
|
key: SMTP_AUTH_PASSWORD
|
|
- name: SMTP_TO
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: alertmanager-smtp
|
|
key: SMTP_TO
|
|
volumeMounts:
|
|
- name: config-template
|
|
mountPath: /template
|
|
- name: config-processed
|
|
mountPath: /config
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 16Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 64Mi
|
|
containers:
|
|
- name: alertmanager
|
|
image: prom/alertmanager:v0.34.1
|
|
args:
|
|
- --config.file=/etc/alertmanager/alertmanager.yml
|
|
- --storage.path=/alertmanager
|
|
- --web.external-url=http://alertmanager.home
|
|
- --cluster.listen-address=
|
|
ports:
|
|
- name: http
|
|
containerPort: 9093
|
|
protocol: TCP
|
|
volumeMounts:
|
|
- name: config-processed
|
|
mountPath: /etc/alertmanager
|
|
- name: storage
|
|
mountPath: /alertmanager
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
cpu: 100m
|
|
memory: 128Mi
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /-/healthy
|
|
port: 9093
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /-/ready
|
|
port: 9093
|
|
initialDelaySeconds: 5
|
|
periodSeconds: 10
|
|
volumes:
|
|
- name: config-template
|
|
configMap:
|
|
name: alertmanager-config-template
|
|
- name: config-processed
|
|
emptyDir: {}
|
|
# Real storage, not emptyDir. This was an emptyDir until 2026-08-14,
|
|
# which meant SILENCES DID NOT SURVIVE A RESTART -- they live in this
|
|
# volume and nothing rebuilds them. Restarting alertmanager to pick up
|
|
# a config change therefore silently un-silenced everything and
|
|
# re-notified every firing alert, which is exactly what happened when
|
|
# the disk-alert routing was deployed.
|
|
- name: storage
|
|
persistentVolumeClaim:
|
|
claimName: alertmanager-data
|
|
---
|
|
apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: alertmanager
|
|
namespace: mon-system
|
|
labels:
|
|
app: alertmanager
|
|
spec:
|
|
type: ClusterIP
|
|
selector:
|
|
app: alertmanager
|
|
ports:
|
|
- name: http
|
|
port: 9093
|
|
targetPort: 9093
|
|
protocol: TCP
|
|
---
|
|
apiVersion: networking.k8s.io/v1
|
|
kind: Ingress
|
|
metadata:
|
|
name: alertmanager
|
|
namespace: mon-system
|
|
labels:
|
|
app: alertmanager
|
|
annotations:
|
|
external-dns.alpha.kubernetes.io/hostname: alertmanager.home
|
|
spec:
|
|
ingressClassName: nginx-internal
|
|
rules:
|
|
- host: alertmanager.home
|
|
http:
|
|
paths:
|
|
- path: /
|
|
pathType: Prefix
|
|
backend:
|
|
service:
|
|
name: alertmanager
|
|
port:
|
|
number: 9093
|