213 lines
9.2 KiB
YAML
213 lines
9.2 KiB
YAML
# =============================================================================
|
|
# Prometheus Alerting Rules for Longhorn
|
|
# =============================================================================
|
|
# =============================================================================
|
|
# Prometheus Alerting Rules for Longhorn
|
|
# Excludes prometheus-data PVC since it's designed to run at ~95% capacity
|
|
# =============================================================================
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-rules
|
|
namespace: mon-system
|
|
labels:
|
|
app: prometheus
|
|
data:
|
|
|
|
authentik-alerts.yml: |
|
|
groups:
|
|
- name: authentik-availability
|
|
rules:
|
|
# Fires if Prometheus cannot scrape the server metrics endpoint.
|
|
# Equivalent to "authentik-server pod is unreachable or crashed."
|
|
- alert: AuthentikServerDown
|
|
expr: up{job="authentik-server"} == 0
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik server is unreachable"
|
|
description: "authentik-server scrape has been failing for 3+ minutes. Logins will fail."
|
|
|
|
- alert: AuthentikWorkerDown
|
|
expr: up{job="authentik-worker"} == 0
|
|
for: 3m
|
|
labels:
|
|
severity: critical
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik worker is unreachable"
|
|
description: "authentik-worker scrape has been failing for 3+ minutes. Background tasks are not running."
|
|
|
|
# Per-outpost disconnect. Downgraded to warning because a single
|
|
# outpost failure only breaks a subset of apps (e.g. just the Arr
|
|
# stack if arr-outpost is down), not the entire IDP.
|
|
- alert: AuthentikOutpostDown
|
|
expr: up{job="authentik-outposts"} == 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik outpost {{ $labels.outpost }} is unreachable"
|
|
description: "Outpost {{ $labels.outpost }} has been down for 5+ minutes. Apps behind this outpost cannot authenticate."
|
|
|
|
- name: authentik-functional
|
|
rules:
|
|
# *** The 13-days-ago alert. ***
|
|
# Fires when the worker has tasks in progress but throughput
|
|
# has collapsed. Probes cannot catch this because the pod is
|
|
# technically alive — only the queue dynamics give it away.
|
|
- alert: AuthentikTaskQueueStuck
|
|
expr: |
|
|
sum(authentik_tasks_in_progress{job="authentik-worker"}) > 5
|
|
and
|
|
sum(rate(authentik_tasks_total{job="authentik-worker"}[5m])) < 0.01
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik worker queue is stuck"
|
|
description: "{{ $value }} tasks in progress with near-zero throughput for 10+ min. Worker is alive but not draining the queue — typically resolved by restarting the deployment."
|
|
|
|
# Softer version: queue is growing but still moving somewhat.
|
|
# Could indicate an expensive task blocking the workers, or
|
|
# a task that keeps retrying, or steady overload.
|
|
- alert: AuthentikTaskBacklog
|
|
expr: sum(authentik_tasks_in_progress{job="authentik-worker"}) > 20
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik task backlog >20 for 15 min"
|
|
description: "{{ $value }} tasks in progress for 15+ min. Likely overload or a single hanging task."
|
|
|
|
# User-visible error signal. Threshold is conservative —
|
|
# on a low-traffic homelab, 0.1 err/sec = ~6 errors/min which
|
|
# is already noticeable to users.
|
|
- alert: AuthentikHighErrorRate
|
|
expr: |
|
|
sum(rate(django_http_responses_total_by_status_total{job="authentik-server",status=~"5.."}[5m])) > 0.1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik is serving 5xx errors"
|
|
description: "{{ $value | printf \"%.2f\" }} 5xx responses/sec for 5+ min."
|
|
|
|
# p95 latency guard. 2s is high for modern auth flows — at this
|
|
# point users are visibly waiting on the login page.
|
|
- alert: AuthentikHighLatency
|
|
expr: |
|
|
histogram_quantile(0.95,
|
|
sum by (le) (rate(authentik_main_request_duration_seconds_bucket{job="authentik-server",dest="core"}[5m]))
|
|
) > 2
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
component: authentik
|
|
annotations:
|
|
summary: "Authentik p95 request latency > 2s"
|
|
description: "p95 latency {{ $value | printf \"%.2f\" }}s for 10+ min. Logins are slow."
|
|
|
|
postgresql-alerts.yml: |
|
|
groups:
|
|
- name: postgresql-availability
|
|
rules:
|
|
- alert: PostgresExporterDown
|
|
expr: up{job="cloudnativepg"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
component: postgresql
|
|
annotations:
|
|
summary: "CloudNativePG metrics endpoint unreachable"
|
|
description: "CNPG metrics exporter on {{ $labels.pod }} has been down for 2+ min. Postgres may be down or the sidecar may have crashed."
|
|
|
|
- name: postgresql-capacity
|
|
rules:
|
|
# Threshold of 80% of your 200 max_connections = 160.
|
|
# If you raise max_connections later, update the number.
|
|
- alert: PostgresHighConnections
|
|
expr: sum by (cluster) (cnpg_backends_total) > 160
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: postgresql
|
|
annotations:
|
|
summary: "Postgres cluster {{ $labels.cluster }} nearing connection limit"
|
|
description: "{{ $value }} active connections (>80% of max_connections=200). Check for connection leaks."
|
|
|
|
# Locks held long enough that other queries are waiting.
|
|
# Usually cleared fast; a sustained non-zero value is abnormal.
|
|
- alert: PostgresBackendsWaiting
|
|
expr: cnpg_backends_waiting_total > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
component: postgresql
|
|
annotations:
|
|
summary: "Postgres has queries blocked on locks"
|
|
description: "{{ $value }} backends waiting on locks for 5+ min. Investigate long-running transactions."
|
|
|
|
longhorn-alerts.yml: |
|
|
groups:
|
|
- name: longhorn-volume-alerts
|
|
rules:
|
|
# Critical: Volume at 95% capacity (excluding prometheus-data)
|
|
- alert: LonghornVolumeSpaceCritical
|
|
expr: |
|
|
(
|
|
(avg by (volume) (longhorn_volume_actual_size_bytes{volume!="pvc-81912547-5a18-4410-ac27-9c15251483a8"}))
|
|
/
|
|
(avg by (volume) (longhorn_volume_capacity_bytes{volume!="pvc-81912547-5a18-4410-ac27-9c15251483a8"}))
|
|
) * 100 > 95
|
|
for: 5m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Longhorn volume {{ $labels.volume }} is critically full"
|
|
description: "Volume {{ $labels.volume }} is at {{ printf \"%.1f\" $value }}% capacity. Immediate action required."
|
|
# Warning: Volume at 85% capacity (excluding prometheus-data)
|
|
- alert: LonghornVolumeSpaceWarning
|
|
expr: |
|
|
(
|
|
(avg by (volume) (longhorn_volume_actual_size_bytes{volume!="pvc-81912547-5a18-4410-ac27-9c15251483a8"}))
|
|
/
|
|
(avg by (volume) (longhorn_volume_capacity_bytes{volume!="pvc-81912547-5a18-4410-ac27-9c15251483a8"}))
|
|
) * 100 > 85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn volume {{ $labels.volume }} is running low on space"
|
|
description: "Volume {{ $labels.volume }} is at {{ printf \"%.1f\" $value }}% capacity. Consider expanding or cleaning up."
|
|
# Volume degraded
|
|
- alert: LonghornVolumeDegraded
|
|
expr: longhorn_volume_robustness != 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn volume {{ $labels.volume }} is degraded"
|
|
description: "Volume {{ $labels.volume }} robustness is not healthy. Check replica status."
|
|
|
|
# Node storage pressure
|
|
- alert: LonghornNodeStoragePressure
|
|
expr: |
|
|
(
|
|
longhorn_node_storage_usage_bytes
|
|
/
|
|
longhorn_node_storage_capacity_bytes
|
|
) * 100 > 90
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Longhorn node {{ $labels.node }} storage pressure"
|
|
description: "Node {{ $labels.node }} disk usage is at {{ printf \"%.1f\" $value }}%."
|