ee71abd1d4
Operator ruling 09 §3 decision 139. One exact sudoers rule FELHOM_FSTRIM
(`/usr/sbin/pct ^fstrim [0-9]+$`) + manifest entry guest-fstrim; new
internal/fstrim job: due Wednesday from 10:00 host-local, starts only
10:00-20:59, holds backup.InFlight (busy -> deferred to the next hourly
tick), failed trim retried at most 3x per week, bytes parsed from the
"(N bytes) trimmed" lines, last result per guest persisted in
<state_dir>/guest-disk-trim.json and reported as guest_disk_trim.
Config opt-out: "disk_trim": {"disable": true}.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
268 lines
9.7 KiB
Go
268 lines
9.7 KiB
Go
package fstrim
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"encoding/json"
|
||
"errors"
|
||
"log/slog"
|
||
"path/filepath"
|
||
"reflect"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/backup"
|
||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||
)
|
||
|
||
// The real `pct fstrim 9201` output measured on demo-hp 2026-10-06 (audits/ten-answers-2026-10-06/r444-measure.txt).
|
||
const measuredOut = "/var/lib/lxc/9201/rootfs/: 30.1 GiB (32277680128 bytes) trimmed\n" +
|
||
"/var/lib/lxc/9201/rootfs/var/lib/felhom: 53.9 GiB (57865633792 bytes) trimmed\n"
|
||
|
||
const measuredBytes = int64(32277680128 + 57865633792)
|
||
|
||
type fakeRunner struct {
|
||
mu sync.Mutex
|
||
calls [][]string
|
||
out string
|
||
err error
|
||
onRun func()
|
||
}
|
||
|
||
func (f *fakeRunner) Run(_ context.Context, name string, args ...string) ([]byte, []byte, error) {
|
||
f.mu.Lock()
|
||
f.calls = append(f.calls, append([]string{name}, args...))
|
||
f.mu.Unlock()
|
||
if f.onRun != nil {
|
||
f.onRun()
|
||
}
|
||
if f.err != nil {
|
||
return nil, []byte("mount busy"), f.err
|
||
}
|
||
return []byte(f.out), nil, nil
|
||
}
|
||
|
||
type fakeGuests struct {
|
||
g []proxmox.Guest
|
||
err error
|
||
}
|
||
|
||
func (f fakeGuests) Guests(context.Context) ([]proxmox.Guest, error) { return f.g, f.err }
|
||
|
||
// A Wednesday 10:30 in a fixed zone (CEST-like), so the tests do not depend on the machine's zone.
|
||
var zone = time.FixedZone("CEST", 2*3600)
|
||
|
||
func at(day, hour, min int) time.Time { return time.Date(2026, 10, day, hour, min, 0, 0, zone) } // 2026-10-07 = Wednesday
|
||
|
||
func newT(t *testing.T, r Runner, g GuestSource, gate Gate, now *time.Time) (*Trimmer, *bytes.Buffer, string) {
|
||
t.Helper()
|
||
var logs bytes.Buffer
|
||
path := filepath.Join(t.TempDir(), "guest-disk-trim.json")
|
||
tr := New(r, g, gate, path, slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
|
||
tr.loc = zone
|
||
tr.now = func() time.Time { return *now }
|
||
return tr, &logs, path
|
||
}
|
||
|
||
func running(ids ...int) fakeGuests {
|
||
var g []proxmox.Guest
|
||
for _, id := range ids {
|
||
g = append(g, proxmox.Guest{VMID: id, Status: "running", Type: "lxc"})
|
||
}
|
||
return fakeGuests{g: g}
|
||
}
|
||
|
||
func TestParseTrimmedTheMeasuredOutput(t *testing.T) {
|
||
b, m := ParseTrimmed(measuredOut)
|
||
if b != measuredBytes || m != 2 {
|
||
t.Fatalf("ParseTrimmed = %d bytes over %d mounts, want %d over 2", b, m, measuredBytes)
|
||
}
|
||
if b, m := ParseTrimmed("something else\n"); b != 0 || m != 0 {
|
||
t.Fatalf("unrelated output parsed as %d/%d", b, m)
|
||
}
|
||
if got := GiB(measuredBytes); got != "84.0 GiB" {
|
||
t.Fatalf("GiB = %q", got)
|
||
}
|
||
}
|
||
|
||
// The night window (01:00–06:59) must never be eligible, and the daytime window is exactly 10:00–20:59.
|
||
func TestEligibleHourNeverInTheNight(t *testing.T) {
|
||
for h := 0; h < 24; h++ {
|
||
lt := time.Date(2026, 10, 7, h, 30, 0, 0, zone)
|
||
got := EligibleHour(lt)
|
||
if h >= 1 && h <= 6 && got {
|
||
t.Errorf("hour %02d is in the night window and must not be eligible", h)
|
||
}
|
||
if want := h >= 10 && h <= 20; got != want {
|
||
t.Errorf("EligibleHour(%02d:30) = %v, want %v", h, got, want)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestWeekAnchorIsTheLastWednesdayTen(t *testing.T) {
|
||
cases := map[time.Time]time.Time{
|
||
at(7, 10, 0): at(7, 10, 0), // Wednesday 10:00 itself
|
||
at(7, 9, 59): time.Date(2026, 9, 30, 10, 0, 0, 0, zone), // before 10:00 Wednesday → the previous week
|
||
at(8, 15, 0): at(7, 10, 0), // Thursday
|
||
at(13, 20, 0): at(7, 10, 0), // next Tuesday
|
||
at(14, 11, 0): at(14, 10, 0), // next Wednesday
|
||
}
|
||
for in, want := range cases {
|
||
if got := weekAnchor(in); !got.Equal(want) {
|
||
t.Errorf("weekAnchor(%s) = %s, want %s", in.Format("Mon 01-02 15:04"), got.Format("Mon 01-02 15:04"), want.Format("Mon 01-02 15:04"))
|
||
}
|
||
}
|
||
}
|
||
|
||
// The consequence: on Wednesday 10:30 a running owned guest is trimmed with the ONE exact argv, the bytes are parsed,
|
||
// the positive log line is written, the result is persisted, and the host report carries it.
|
||
func TestPassTrimsADueGuestAndReportsIt(t *testing.T) {
|
||
now := at(7, 10, 30)
|
||
r := &fakeRunner{out: measuredOut}
|
||
tr, logs, path := newT(t, r, running(9201), &backup.InFlight{}, &now)
|
||
tr.Pass(context.Background())
|
||
|
||
if want := [][]string{{"pct", "fstrim", "9201"}}; !reflect.DeepEqual(r.calls, want) {
|
||
t.Fatalf("runner calls = %q, want %q", r.calls, want)
|
||
}
|
||
if !strings.Contains(logs.String(), "fstrim: guest 9201 trimmed 84.0 GiB in ") {
|
||
t.Fatalf("no positive per-guest log line:\n%s", logs.String())
|
||
}
|
||
st := tr.GuestDiskTrimStatus(context.Background())
|
||
if st == nil || st.Schedule != ScheduleText || len(st.Guests) != 1 {
|
||
t.Fatalf("report stanza = %+v", st)
|
||
}
|
||
g := st.Guests[0]
|
||
if g.VMID != 9201 || !g.OK || g.BytesTrimmed != measuredBytes || g.Mounts != 2 || g.LastOKAt == "" || g.LastAttemptAt == "" {
|
||
t.Fatalf("report guest = %+v", g)
|
||
}
|
||
// Persisted: a NEW Trimmer over the same file (an agent restart) still has it and does not trim again this week.
|
||
now = at(8, 11, 0)
|
||
r2 := &fakeRunner{out: measuredOut}
|
||
tr2 := New(r2, running(9201), &backup.InFlight{}, path, slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)))
|
||
tr2.loc, tr2.now = zone, func() time.Time { return now }
|
||
if st2 := tr2.GuestDiskTrimStatus(context.Background()); len(st2.Guests) != 1 || st2.Guests[0].BytesTrimmed != measuredBytes {
|
||
t.Fatalf("result lost over a restart: %+v", st2)
|
||
}
|
||
tr2.Pass(context.Background())
|
||
if len(r2.calls) != 0 {
|
||
t.Fatalf("trimmed again in the same week after a restart: %q", r2.calls)
|
||
}
|
||
// Next week it is due again.
|
||
now = at(14, 10, 5)
|
||
tr2.Pass(context.Background())
|
||
if len(r2.calls) != 1 {
|
||
t.Fatalf("not trimmed in the next week: %q", r2.calls)
|
||
}
|
||
}
|
||
|
||
func TestPassNeverRunsInTheNight(t *testing.T) {
|
||
for _, h := range []int{1, 3, 6, 9, 21, 23} {
|
||
now := at(7, h, 15)
|
||
r := &fakeRunner{out: measuredOut}
|
||
tr, _, _ := newT(t, r, running(9201), &backup.InFlight{}, &now)
|
||
tr.Pass(context.Background())
|
||
if len(r.calls) != 0 {
|
||
t.Errorf("trimmed at %02d:15: %q", h, r.calls)
|
||
}
|
||
}
|
||
}
|
||
|
||
// A backup (or restore-test) holding the heavy-op gate DEFERS the trim; the next hour, gate free, it runs. And while
|
||
// a trim runs, the gate is held, so a backup cannot start beside it.
|
||
func TestPassDefersToAHeavyOperationAndRetriesNextHour(t *testing.T) {
|
||
now := at(7, 10, 30)
|
||
gate := &backup.InFlight{}
|
||
release, _, _ := gate.TryAcquire("backup:9201")
|
||
var busyDuringTrim string
|
||
r := &fakeRunner{out: measuredOut}
|
||
r.onRun = func() { busyDuringTrim = gate.Busy() }
|
||
tr, logs, _ := newT(t, r, running(9201), gate, &now)
|
||
|
||
tr.Pass(context.Background())
|
||
if len(r.calls) != 0 {
|
||
t.Fatalf("trimmed beside a running backup: %q", r.calls)
|
||
}
|
||
if !strings.Contains(logs.String(), "fstrim: deferred") || !strings.Contains(logs.String(), "backup:9201") {
|
||
t.Fatalf("the deferral is not logged with what holds the gate:\n%s", logs.String())
|
||
}
|
||
release()
|
||
now = now.Add(time.Hour)
|
||
tr.Pass(context.Background())
|
||
if len(r.calls) != 1 {
|
||
t.Fatalf("not retried the next hour: %q", r.calls)
|
||
}
|
||
if busyDuringTrim != GateName {
|
||
t.Fatalf("the heavy-op gate was %q during the trim, want %q", busyDuringTrim, GateName)
|
||
}
|
||
if gate.Busy() != "" {
|
||
t.Fatalf("the gate was not released after the pass: %q", gate.Busy())
|
||
}
|
||
}
|
||
|
||
func TestFailedTrimWarnsIsRecordedAndRetriedAtMostThreeTimes(t *testing.T) {
|
||
now := at(7, 10, 30)
|
||
r := &fakeRunner{err: errors.New("exit status 255")}
|
||
tr, logs, _ := newT(t, r, running(9201), &backup.InFlight{}, &now)
|
||
for i := 0; i < 6; i++ {
|
||
tr.Pass(context.Background())
|
||
now = now.Add(time.Hour)
|
||
}
|
||
if len(r.calls) != MaxAttemptsPerWeek {
|
||
t.Fatalf("attempts in one week = %d, want %d", len(r.calls), MaxAttemptsPerWeek)
|
||
}
|
||
if !strings.Contains(logs.String(), "level=WARN") || !strings.Contains(logs.String(), "fstrim: guest 9201 trim FAILED") {
|
||
t.Fatalf("no WARN for the failure:\n%s", logs.String())
|
||
}
|
||
g := tr.GuestDiskTrimStatus(context.Background()).Guests[0]
|
||
if g.OK || g.LastOKAt != "" || !strings.Contains(g.Error, "exit status 255") || !strings.Contains(g.Error, "mount busy") {
|
||
t.Fatalf("failed result not recorded as a failure: %+v", g)
|
||
}
|
||
// A success later keeps a clean record.
|
||
r.err = nil
|
||
r.out = measuredOut
|
||
now = at(14, 10, 10)
|
||
tr.Pass(context.Background())
|
||
if g := tr.GuestDiskTrimStatus(context.Background()).Guests[0]; !g.OK || g.Error != "" || g.BytesTrimmed != measuredBytes {
|
||
t.Fatalf("success after failure: %+v", g)
|
||
}
|
||
}
|
||
|
||
func TestOnlyRunningOwnedGuestsAndAFailedListActsOnNothing(t *testing.T) {
|
||
now := at(7, 10, 30)
|
||
r := &fakeRunner{out: measuredOut}
|
||
g := fakeGuests{g: []proxmox.Guest{{VMID: 9201, Status: "stopped"}, {VMID: 9202, Status: "running"}}}
|
||
tr, _, _ := newT(t, r, g, &backup.InFlight{}, &now)
|
||
tr.Pass(context.Background())
|
||
if want := [][]string{{"pct", "fstrim", "9202"}}; !reflect.DeepEqual(r.calls, want) {
|
||
t.Fatalf("calls = %q, want only the running guest", r.calls)
|
||
}
|
||
r2 := &fakeRunner{out: measuredOut}
|
||
tr2, logs, _ := newT(t, r2, fakeGuests{err: errors.New("pool read 403")}, &backup.InFlight{}, &now)
|
||
tr2.Pass(context.Background())
|
||
if len(r2.calls) != 0 || !strings.Contains(logs.String(), "owned-guest list unavailable") {
|
||
t.Fatalf("a failed ownership read must act on nothing: calls %q", r2.calls)
|
||
}
|
||
}
|
||
|
||
func TestReportJSONShape(t *testing.T) {
|
||
now := at(7, 10, 30)
|
||
tr, _, _ := newT(t, &fakeRunner{out: measuredOut}, running(9201), &backup.InFlight{}, &now)
|
||
tr.Pass(context.Background())
|
||
b, err := json.Marshal(tr.GuestDiskTrimStatus(context.Background()))
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
for _, k := range []string{`"schedule":`, `"guests":[{"vmid":9201`, `"last_attempt_at":"2026-10-07T08:30:00Z"`, `"ok":true`,
|
||
`"bytes_trimmed":90143313920`, `"mounts":2`, `"duration_seconds":`, `"last_ok_at":"2026-10-07T08:30:00Z"`} {
|
||
if !strings.Contains(string(b), k) {
|
||
t.Errorf("report JSON lacks %s: %s", k, b)
|
||
}
|
||
}
|
||
if strings.Contains(string(b), `"error"`) {
|
||
t.Errorf("an ok result must omit error: %s", b)
|
||
}
|
||
}
|