admin
afba622fb6
gates / gates (push) Successful in 59s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
53 lines
2.8 KiB
YAML
53 lines
2.8 KiB
YAML
rule_files: [bf.yml]
|
|
evaluation_interval: 1m
|
|
tests:
|
|
# 1. the push stops: last success at t=0, never again → fires once 26 h + 30 m have passed
|
|
- interval: 5m
|
|
input_series:
|
|
- series: 'felhom_hub_db_backup_last_success_timestamp_seconds{instance="dooplex"}'
|
|
values: '0+0x400'
|
|
- series: 'felhom_hub_db_restore_test_last_success_timestamp_seconds{instance="dooplex"}'
|
|
values: '0+0x400'
|
|
alert_rule_test:
|
|
- eval_time: 26h
|
|
alertname: HubDBBackupStale
|
|
exp_alerts: []
|
|
- eval_time: 26h40m
|
|
alertname: HubDBBackupStale
|
|
exp_alerts:
|
|
- exp_labels: {severity: critical, component: backup, instance: dooplex}
|
|
exp_annotations:
|
|
summary: "The Felhom hub database has not reached ep0 for 26 h (R-173)"
|
|
description: "No successful push of the hub DB snapshot to ep0 for >26h (daily at 02:30). Check `journalctl -u felhom-hub-db-backup.service`, the tunnel `systemctl status felhom-ep0-pbs-tunnel`, and that the hub logs `db snapshot written` at 02:00."
|
|
# 2. healthy: the push succeeds every 24 h → never fires
|
|
- interval: 1h
|
|
input_series:
|
|
- series: 'felhom_hub_db_backup_last_success_timestamp_seconds{instance="dooplex"}'
|
|
values: '0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 86400 172800 172800 172800'
|
|
- series: 'felhom_hub_db_restore_test_last_success_timestamp_seconds{instance="dooplex"}'
|
|
values: '0+0x50'
|
|
alert_rule_test:
|
|
- eval_time: 50h
|
|
alertname: HubDBBackupStale
|
|
exp_alerts: []
|
|
# 3. the metric never existed (script never ran) → fires on absent()
|
|
- interval: 5m
|
|
input_series:
|
|
- series: 'up{job="node"}'
|
|
values: '1+0x30'
|
|
alert_rule_test:
|
|
- eval_time: 40m
|
|
alertname: HubDBBackupStale
|
|
exp_alerts:
|
|
- exp_labels: {severity: critical, component: backup}
|
|
exp_annotations:
|
|
summary: "The Felhom hub database has not reached ep0 for 26 h (R-173)"
|
|
description: "No successful push of the hub DB snapshot to ep0 for >26h (daily at 02:30). Check `journalctl -u felhom-hub-db-backup.service`, the tunnel `systemctl status felhom-ep0-pbs-tunnel`, and that the hub logs `db snapshot written` at 02:00."
|
|
- eval_time: 2h
|
|
alertname: HubDBRestoreTestStale
|
|
exp_alerts:
|
|
- exp_labels: {severity: warning, component: backup}
|
|
exp_annotations:
|
|
summary: "The hub database copy on ep0 has not passed a restore test for 8 days (R-173)"
|
|
description: "The weekly restore test (Sun 04:30) has not succeeded for >8 days. Check `journalctl -u felhom-hub-db-restore-test.service`."
|