Docker engine slow lane (live-restore once by reload; ring-0 pending-docker under a root-owned ring-0 mark; ring 1 and undo only by a signed os_docker_step the wrapper re-verifies against a root-owned signers file; same-container-id health), the version report (facts mode -> host report system stanza, R-852), guest restart scan every pass (R-849), the crash guard (kernel.panic=10, the 3rd unclean stop in 60 min stays off, 24 h re-arm)
gates / gates (push) Successful in 19s
gates / gates (push) Successful in 19s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_0159rPz1ZhFKsS53msqPYxtS
This commit is contained in:
+147
-40
@@ -44,11 +44,11 @@ import (
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/localapi"
|
||||
applog "gitea.dooplex.hu/admin/felhom-agent/internal/log"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/mgmtplane"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/osupdate"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pbs"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/pbsdr"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/poke"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/provision"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/osupdate"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/proxmox"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/reconcile"
|
||||
"gitea.dooplex.hu/admin/felhom-agent/internal/restorespace"
|
||||
@@ -245,6 +245,10 @@ func main() {
|
||||
os.Exit(runSelftestRestoreTestDue(context.Background(), cfg, logger))
|
||||
case "os-update":
|
||||
os.Exit(runSelftestOSUpdate(context.Background(), cfg, logger, vmid))
|
||||
case "os-facts":
|
||||
os.Exit(runSelftestFacts(context.Background(), cfg, logger, vmid))
|
||||
case "live-restore":
|
||||
os.Exit(runSelftestLiveRestore(context.Background(), cfg, logger, vmid))
|
||||
case "pbs-verify":
|
||||
os.Exit(runSelftestPBSVerify(context.Background(), cfg, logger))
|
||||
case "lanresolver":
|
||||
@@ -845,6 +849,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
// OS updates, guest fast lane (agent v0.140.0, `11-os-updates.md` §8 step 2): the leg consumes the hub's
|
||||
// os_update block and runs after each successful primary whole-guest backup (wired on the local API below).
|
||||
osLeg := newOSLeg(cfg, client, px, logger)
|
||||
collector.SetSystemReporter(&factsReporter{leg: osLeg, guest: firstGuest(px)}) // R-852: the versions
|
||||
desiredSyncer.AddConsumer(osLeg)
|
||||
// S5: consume a host_loss restore_directive into an inspectable restore PLAN (derive + surface,
|
||||
// execute nothing). The recipe is fetched on-demand (rare directive) via a fresh Collect.
|
||||
@@ -1087,7 +1092,16 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
// recurring clobber before it locks the box out. Port 22 for G1 (H1 passes the felhom-sshd port).
|
||||
collector.SetMgmtPlaneReporter(mgmtplane.NewReporter(mgmtplane.DefaultPrivsepDir, mgmtplane.DefaultHealMarker, mgmtplane.DefaultSshdPort))
|
||||
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec}, cfg.Hub.HostID, logger)
|
||||
// Agent v0.142.0: a signed Docker engine step (`11` §5.8) — ring 1 and every undo; under the heavy-op gate.
|
||||
dockerExec := osupdate.DockerStepExecutor{Leg: osLeg, Guest: firstGuest(px),
|
||||
Gate: func(ctx context.Context) (func(), error) {
|
||||
release, busy, ok := heavyOps.TryAcquire("os-docker-step")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("busy: %s", busy)
|
||||
}
|
||||
return release, nil
|
||||
}}
|
||||
jobsRunner := signedjobs.NewRunner(client, gate, signedjobs.ExecutorChain{wipeExec, decommExec, updateExec, dockerExec}, cfg.Hub.HostID, logger)
|
||||
loop.SetEnvelopeObserver(hub.MultiObserver(desiredSyncer, jobsRunner))
|
||||
|
||||
// Controller-driven escrow ceremony (v0.88.0): static config facts + the LATE-BOUND DR gate —
|
||||
@@ -1127,7 +1141,7 @@ func runDaemon(cfg config.Config, logger *slog.Logger, logRing *applog.Ring) int
|
||||
return
|
||||
case <-time.After(90 * time.Second):
|
||||
}
|
||||
_, _ = osLeg.Run(ctx, vmid, "night")
|
||||
_ = osLeg.Run(ctx, vmid, "night")
|
||||
})
|
||||
}
|
||||
if localTokens != nil {
|
||||
@@ -1869,7 +1883,7 @@ func buildLocalAPIServer(cfg config.Config, px *proxmox.Client, store *backup.St
|
||||
ConfigPath: cfg.SourcePath,
|
||||
StateDir: cfg.WGTunnel.WithDefaults().StateDir,
|
||||
SmbCredsDir: cfg.Privileged.SmbCredsDir,
|
||||
ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap
|
||||
ControllerSwap: guestBinder, // Phase 1: agentic controller update — in-guest image swap
|
||||
GuestsStateDir: "/var/lib/felhom-agent/guests", // R-523: <vmid>/bootstrap + controller-parked marker
|
||||
// F2-b: recover a guest left with a stale vzdump lock by a reboot-during-backup. Reads + start
|
||||
// go through the API client; the `pct unlock` is the one fenced root-CLI op (no API equivalent).
|
||||
@@ -3498,10 +3512,12 @@ func (f *selftestFlag) Set(v string) error {
|
||||
f.mode = "controller-swap"
|
||||
case "os-update":
|
||||
f.mode = "os-update"
|
||||
case "os-facts", "live-restore": // agent v0.142.0
|
||||
f.mode = v
|
||||
case "wgtunnel": // dispatched since S3 but refused here until 2026-10-04 (TestSelftestFlag_AcceptsEveryDispatchedMode)
|
||||
f.mode = "wgtunnel"
|
||||
default:
|
||||
return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|lanresolver|wgtunnel|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap|os-update)", v)
|
||||
return fmt.Errorf("invalid --selftest value %q (want read|task|hub|storage|backup|restore-test|restore-test-due|pbs-verify|lanresolver|wgtunnel|bring-up|provision|escrow-create|escrow-consume|identity-consume|controller-swap|os-update|os-facts|live-restore)", v)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -3558,29 +3574,71 @@ func runSelftestOSUpdate(ctx context.Context, cfg config.Config, logger *slog.Lo
|
||||
fmt.Printf("=== felhom-agent %s selftest=os-update vmid=%d ring=%d enabled=%v guest-release=%v host-release=%v appliance=%v ===\n",
|
||||
version, vmid, b.Ring, b.Enabled, b.Release != nil, b.HostRelease != nil, leg.Appliance)
|
||||
start := time.Now()
|
||||
g, h := leg.Run(ctx, vmid, "debug")
|
||||
for _, rep := range []osupdate.Report{g, h} {
|
||||
pass := leg.Run(ctx, vmid, "debug")
|
||||
worst := pass.Guest
|
||||
for i, rep := range []osupdate.Report{pass.Guest, pass.Host, pass.Docker} {
|
||||
if rep.Layer == "" {
|
||||
fmt.Println(" host step: skipped (see the log line above)")
|
||||
fmt.Printf(" %s step: skipped (see the log line above)\n", []string{"guest", "host", "docker"}[i])
|
||||
continue
|
||||
}
|
||||
printJSON("os-update report ("+rep.Layer+")", map[string]any{"run_id": rep.RunID, "ring": rep.Ring, "release_id": rep.ReleaseID,
|
||||
"mode": rep.Mode, "outcome": rep.Outcome, "healthy": rep.Healthy, "health_reason": rep.HealthReason,
|
||||
"upgraded": rep.Upgraded, "pending": len(rep.Pending), "not_covered": rep.NotCovered,
|
||||
"restart_needed": rep.RestartNeeded, "reboot_needed": rep.RebootNeeded, "wrapper_seconds": rep.PassSeconds, "refused": rep.Refused})
|
||||
"restart_needed": rep.RestartNeeded, "reboot_needed": rep.RebootNeeded, "wrapper_seconds": rep.PassSeconds,
|
||||
"refused": rep.Refused, "docker_engine": rep.DockerEngine, "authority": rep.Authority})
|
||||
if !(rep.Outcome == "applied" || rep.Outcome == "nothing" || rep.Outcome == "inventory" || rep.Outcome == "skipped") {
|
||||
worst = rep
|
||||
}
|
||||
}
|
||||
fmt.Printf(" pass took %s\n", time.Since(start).Round(100*time.Millisecond))
|
||||
rep := g
|
||||
if h.Layer != "" && !(h.Outcome == "applied" || h.Outcome == "nothing" || h.Outcome == "inventory") {
|
||||
rep = h
|
||||
}
|
||||
switch rep.Outcome {
|
||||
switch worst.Outcome {
|
||||
case "applied", "nothing", "inventory", "skipped":
|
||||
return 0
|
||||
}
|
||||
return 1
|
||||
}
|
||||
|
||||
// runSelftestFacts prints the versions the host report carries (R-852, agent v0.142.0) — read-only.
|
||||
//
|
||||
// sudo -u felhom-agent felhom-agent --config … --selftest=os-facts -vmid 9201
|
||||
func runSelftestFacts(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
||||
if vmid <= 0 {
|
||||
fmt.Fprintln(os.Stderr, "selftest=os-facts: -vmid is required")
|
||||
return 2
|
||||
}
|
||||
px, _ := newProxmoxClient(cfg)
|
||||
leg := newOSLeg(cfg, nil, px, logger)
|
||||
start := time.Now()
|
||||
f, err := leg.Facts(ctx, vmid)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "selftest=os-facts:", err)
|
||||
return 1
|
||||
}
|
||||
var v any
|
||||
_ = json.Unmarshal(f, &v)
|
||||
printJSON(fmt.Sprintf("facts (vmid %d, %s)", vmid, time.Since(start).Round(100*time.Millisecond)), v)
|
||||
return 0
|
||||
}
|
||||
|
||||
// runSelftestLiveRestore is the ONE-TIME live-restore step (`09` decision 87) as a debug action — the night leg does
|
||||
// the same before a ring-0 Docker step. It prints the container ids before and after (they must not change).
|
||||
//
|
||||
// sudo -u felhom-agent felhom-agent --config … --selftest=live-restore -vmid 9202
|
||||
func runSelftestLiveRestore(ctx context.Context, cfg config.Config, logger *slog.Logger, vmid int) int {
|
||||
if vmid <= 0 {
|
||||
fmt.Fprintln(os.Stderr, "selftest=live-restore: -vmid is required")
|
||||
return 2
|
||||
}
|
||||
px, _ := newProxmoxClient(cfg)
|
||||
leg := newOSLeg(cfg, nil, px, logger)
|
||||
if err := leg.EnsureLiveRestore(ctx, time.Now().UTC().Format("20060102T150405Z"), vmid); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "selftest=live-restore:", err)
|
||||
return 1
|
||||
}
|
||||
fmt.Println("live-restore: on (see the wrapper's LIVE-RESTORE line above for the container ids)")
|
||||
return 0
|
||||
}
|
||||
|
||||
// newTunnelProber reads the box's REAL tunnel (R-841, agent v0.141.0): the cloudflared container in each running
|
||||
// customer guest — a guest that binds /mnt/felhom-drives, the same rule the OS wrapper's R10 uses — through the
|
||||
// existing `pct exec [0-9]* -- docker inspect -f *` sudoers line.
|
||||
@@ -3591,31 +3649,80 @@ func newTunnelProber(cfg config.Config, px *proxmox.Client) hub.CloudflaredProbe
|
||||
}
|
||||
return hub.GuestTunnelProber{
|
||||
Runner: &proxmox.ExecRunner{Mode: mode, SudoPath: cfg.Privileged.SudoPath},
|
||||
Guests: func(ctx context.Context) ([]int, error) {
|
||||
if px == nil {
|
||||
return nil, fmt.Errorf("no proxmox client")
|
||||
}
|
||||
gs, err := px.ListLXC(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []int
|
||||
for _, g := range gs {
|
||||
if g.Status != "running" {
|
||||
continue
|
||||
}
|
||||
gc, err := px.GuestConfig(ctx, g.VMID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range gc.MountPoints() {
|
||||
if src, _, _ := strings.Cut(v, ","); src == "/mnt/felhom-drives" {
|
||||
out = append(out, g.VMID)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
},
|
||||
Guests: customerGuests(px),
|
||||
}
|
||||
}
|
||||
|
||||
// customerGuests lists the running guests that bind /mnt/felhom-drives — the box's customer guest(s), the same rule the
|
||||
// OS wrapper's R10 uses. Shared by the tunnel probe, the facts read and the signed Docker step.
|
||||
func customerGuests(px *proxmox.Client) func(ctx context.Context) ([]int, error) {
|
||||
return func(ctx context.Context) ([]int, error) {
|
||||
if px == nil {
|
||||
return nil, fmt.Errorf("no proxmox client")
|
||||
}
|
||||
gs, err := px.ListLXC(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []int
|
||||
for _, g := range gs {
|
||||
if g.Status != "running" {
|
||||
continue
|
||||
}
|
||||
gc, err := px.GuestConfig(ctx, g.VMID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, v := range gc.MountPoints() {
|
||||
if src, _, _ := strings.Cut(v, ","); src == "/mnt/felhom-drives" {
|
||||
out = append(out, g.VMID)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
}
|
||||
|
||||
// firstGuest is the single customer guest (an error when there is none).
|
||||
func firstGuest(px *proxmox.Client) func(ctx context.Context) (int, error) {
|
||||
f := customerGuests(px)
|
||||
return func(ctx context.Context) (int, error) {
|
||||
v, err := f(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(v) == 0 {
|
||||
return 0, fmt.Errorf("no running customer guest")
|
||||
}
|
||||
return v[0], nil
|
||||
}
|
||||
}
|
||||
|
||||
// factsReporter feeds the host report's `system` stanza (R-852, agent v0.142.0) from the wrapper's read-only facts
|
||||
// mode, at most every 10 minutes (each read is ~2 s of pct exec; the host reports every 15 min).
|
||||
type factsReporter struct {
|
||||
leg *osupdate.Leg
|
||||
guest func(ctx context.Context) (int, error)
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
vmid int
|
||||
facts json.RawMessage
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *factsReporter) SystemFacts(ctx context.Context) (int, json.RawMessage, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if !f.at.IsZero() && time.Since(f.at) < 10*time.Minute {
|
||||
return f.vmid, f.facts, f.err
|
||||
}
|
||||
f.at = time.Now()
|
||||
f.vmid, f.err = f.guest(ctx)
|
||||
if f.err != nil {
|
||||
f.facts = nil
|
||||
return 0, nil, f.err
|
||||
}
|
||||
f.facts, f.err = f.leg.Facts(ctx, f.vmid)
|
||||
return f.vmid, f.facts, f.err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user