3c49dc8ea4
gates / gates (push) Successful in 12s
R-399: monitoring.integrity.read_data_subset defaults to 100%. A pack damaged without changing its size made plain `restic check` report "no errors were found" on demo-hp 2026-08-30; every read-data form caught it. Cost on that 134 MB store: 35.0s structure vs 39.2s at 100%. "off" (any case) is the off token; empty means not-configured, therefore the default; a malformed value falls back to the DEFAULT, never to structure. A completed check over 5 minutes logs a WARN naming the duration, the depth and R-401 — operator log only, no hub event, no depth change. The depth is now recorded with the verdict (LastIntegrityDepth; empty = NOT RECORDED, never "structure"). R-400: 24 debug-page references, 17 dispatched, 7 dead — three of which fetched on page LOAD, so those panels were permanently blank. backup/crossdrive implemented; backup/infra, hub/infra-push, dr/infra-status, storage/watchdog-status and both storage/simulate-* deleted with their panels and JavaScript. scripts/debug_route_gate.py fails in both directions and is registered after the seven were resolved. 18 referenced, 18 dispatched, none orphaned. Corrections: the dead-field warning in report/types.go said the controller runs no integrity check and the notifiers are called from nowhere — both false since v0.227.0. controller.yaml.example gains its missing integrity: block. integrityCheckTimeout's "ships OFF" comment rewritten.
140 lines
5.4 KiB
Go
140 lines
5.4 KiB
Go
package backup
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// ── R-399 / R-401 — the check tells the operator when it starts costing real time ────────────────
|
|
//
|
|
// The 100% default rests on ONE measurement, on ONE 134 MB store: 39.2 s. It will not stay true. The
|
|
// notice is what makes that fact reach a person from the product rather than from a customer.
|
|
//
|
|
// It is a LOG LINE and nothing else — no hub event, no customer alarm, and it never changes the depth
|
|
// by itself. TestR399_SlownessRaisesNoHubEvent in cmd/controller pins the first of those; the other
|
|
// two are pinned here.
|
|
|
|
// slowNoticeFragment is the ASCII-only fingerprint of the notice, chosen so no other line in this
|
|
// file's logs can match it.
|
|
const slowNoticeFragment = "R-401: the depth setting needs revisiting"
|
|
|
|
// withSlowThreshold lowers the notice threshold for one test and restores it.
|
|
//
|
|
// No test can make a check take five minutes, so the threshold is the seam. Everything else runs
|
|
// through the real CheckOffboxIntegrity.
|
|
func withSlowThreshold(t *testing.T, d time.Duration) {
|
|
t.Helper()
|
|
prev := integritySlowNoticeThreshold
|
|
integritySlowNoticeThreshold = d
|
|
t.Cleanup(func() { integritySlowNoticeThreshold = prev })
|
|
}
|
|
|
|
// slowReply makes the `check` call take measurable time so a lowered threshold is genuinely exceeded
|
|
// rather than merely equalled.
|
|
func slowReply(out []byte, err error) func(args []string) ([]byte, error) {
|
|
return okRepo(func(args []string) ([]byte, error) {
|
|
time.Sleep(3 * time.Millisecond)
|
|
return out, err
|
|
})
|
|
}
|
|
|
|
// B1 — a slow check warns, and the warning names the duration and the depth.
|
|
func TestR399_SlowCheckWarns(t *testing.T) {
|
|
withSlowThreshold(t, time.Millisecond)
|
|
m, cap := newIntegrityManager(t, slowReply(nil, nil))
|
|
res := m.CheckOffboxIntegrity(context.Background())
|
|
|
|
if !res.OK {
|
|
t.Fatalf("fixture: this run should pass; got %+v", res)
|
|
}
|
|
log := cap.logBuf.String()
|
|
if !strings.Contains(log, slowNoticeFragment) {
|
|
t.Fatalf("a check over the threshold produced no notice — the operator would keep spending a "+
|
|
"customer's bandwidth every week and hear about it from the customer. Log:\n%s", log)
|
|
}
|
|
if !strings.Contains(log, "WARN") {
|
|
t.Errorf("the notice is not at WARN level:\n%s", log)
|
|
}
|
|
// It must name BOTH facts: how long, and how deep. Either alone is unactionable.
|
|
if !strings.Contains(log, "100%") {
|
|
t.Errorf("the notice does not name the depth that was slow:\n%s", log)
|
|
}
|
|
if !strings.Contains(log, "the check took") {
|
|
t.Errorf("the notice does not name the duration:\n%s", log)
|
|
}
|
|
// And it changes NOTHING by itself. A notice that silently reconfigured the box would be a
|
|
// behaviour change wearing a notice's clothes.
|
|
if m.integrityReadDataSubset() != "100%" {
|
|
t.Error("the notice altered the configured depth — it must only report")
|
|
}
|
|
}
|
|
|
|
// B2 — a fast check is silent. A notice that fires every week is not a notice.
|
|
func TestR399_FastCheckIsSilent(t *testing.T) {
|
|
withSlowThreshold(t, time.Hour)
|
|
m, cap := newIntegrityManager(t, okRepo(nil))
|
|
m.CheckOffboxIntegrity(context.Background())
|
|
|
|
if strings.Contains(cap.logBuf.String(), slowNoticeFragment) {
|
|
t.Fatalf("a check under the threshold warned anyway:\n%s", cap.logBuf.String())
|
|
}
|
|
}
|
|
|
|
// B3 — a SKIP has no duration to judge.
|
|
func TestR399_SkipNeverWarns(t *testing.T) {
|
|
withSlowThreshold(t, time.Nanosecond) // every non-zero duration would qualify
|
|
m, cap := newIntegrityManager(t, okRepo(nil))
|
|
if err := m.AcquireRunningForTest(); err != nil {
|
|
t.Fatalf("fixture: %v", err)
|
|
}
|
|
defer m.ReleaseRunningForTest()
|
|
|
|
res := m.CheckOffboxIntegrity(context.Background())
|
|
if !res.Skipped {
|
|
t.Fatalf("fixture: expected a skip, got %+v", res)
|
|
}
|
|
if strings.Contains(cap.logBuf.String(), slowNoticeFragment) {
|
|
t.Fatalf("a skipped check produced a slowness notice — it never ran, so there is no duration "+
|
|
"to judge:\n%s", cap.logBuf.String())
|
|
}
|
|
}
|
|
|
|
// B4 — "I could not look" is not "I looked and it was slow".
|
|
func TestR399_UnreachableNeverWarns(t *testing.T) {
|
|
withSlowThreshold(t, time.Nanosecond)
|
|
m, cap := newIntegrityManager(t, func(args []string) ([]byte, error) {
|
|
time.Sleep(3 * time.Millisecond)
|
|
return []byte("ssh: connect to host nas.local port 22: Connection refused"), errors.New("exit 1")
|
|
})
|
|
res := m.CheckOffboxIntegrity(context.Background())
|
|
if !res.Unreachable {
|
|
t.Fatalf("fixture: expected unreachable, got %+v", res)
|
|
}
|
|
if strings.Contains(cap.logBuf.String(), slowNoticeFragment) {
|
|
t.Fatalf("an unreachable store produced a slowness notice:\n%s", cap.logBuf.String())
|
|
}
|
|
}
|
|
|
|
// B5 — the notice and the failure verdict are INDEPENDENT facts. Neither suppresses the other.
|
|
func TestR399_SlowAndFailedProducesBoth(t *testing.T) {
|
|
withSlowThreshold(t, time.Millisecond)
|
|
const damaged = "pack 5b1f2c3d: not found in index\nrepository contains errors"
|
|
m, cap := newIntegrityManager(t, slowReply([]byte(damaged), errFake))
|
|
res := m.CheckOffboxIntegrity(context.Background())
|
|
|
|
if res.OK || res.Unreachable {
|
|
t.Fatalf("fixture: expected a readable-and-damaged verdict, got %+v", res)
|
|
}
|
|
log := cap.logBuf.String()
|
|
if !strings.Contains(log, "check FAILED") {
|
|
t.Fatalf("the failure was not reported:\n%s", log)
|
|
}
|
|
if !strings.Contains(log, slowNoticeFragment) {
|
|
t.Fatalf("a slow FAILING check lost its slowness notice — suppressing one fact because the "+
|
|
"other fired is how the second fact stops existing:\n%s", log)
|
|
}
|
|
}
|